From 36f03d4864761ef0c6afae135934147c2cc59b93 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:48:47 -0700 Subject: [PATCH 001/521] test(tui): expose stale server upgrade failure --- cmux-tui/crates/cmux-tui/tests/cli.rs | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 957a0e2f8133..c08da4d949f4 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -189,6 +189,61 @@ fn plain_launch_attaches_to_existing_local_session() { panic!("plain launch never attached as a TUI client"); } +#[test] +fn plain_launch_explains_how_to_replace_an_incompatible_local_server() { + let dir = unique_temp_dir("incompatible-server"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = transport::listen(&socket).unwrap(); + let server = std::thread::spawn(move || { + for connection in 0..2 { + let mut stream = listener.accept().unwrap(); + if connection == 0 { + let mut request = String::new(); + BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); + let request: serde_json::Value = serde_json::from_str(&request).unwrap(); + let response = serde_json::json!({ + "id": request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": 4242, + "version": "0.0.0-stale", + "protocol": 8, + }, + }); + writeln!(stream, "{response}").unwrap(); + } + } + }); + + let output = Command::new(bin()) + .args(["--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + server.join().unwrap(); + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8(output.stderr).unwrap(); + assert!(stderr.contains("server: v0.0.0-stale protocol 8"), "{stderr}"); + assert!( + stderr.contains(&format!( + "client: v{} protocol {}", + env!("CARGO_PKG_VERSION"), + cmux_tui_core::server::PROTOCOL_VERSION + )), + "{stderr}" + ); + assert!(stderr.contains("Stopping exits pane processes."), "{stderr}"); + assert!(stderr.contains("cmux-tui server stop"), "{stderr}"); + assert!(!stderr.contains("already in use"), "{stderr}"); +} + #[cfg(unix)] #[test] fn configured_websocket_server_does_not_attach_to_existing_session() { From 9c7874259300588c5adf174f3333d2dc9cafd2bd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:51:03 -0700 Subject: [PATCH 002/521] fix(tui): report and stop stale servers --- .github/workflows/cmux-tui-build-package.yml | 9 +- cmux-tui/crates/cmux-tui-core/src/lib.rs | 1 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 14 +- cmux-tui/crates/cmux-tui-core/src/release.rs | 103 ++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 110 ++++-- cmux-tui/crates/cmux-tui/src/app.rs | 33 +- cmux-tui/crates/cmux-tui/src/cli.rs | 210 ++++++++--- cmux-tui/crates/cmux-tui/src/localization.rs | 57 +++ cmux-tui/crates/cmux-tui/src/main.rs | 69 ++-- .../crates/cmux-tui/src/server_lifecycle.rs | 343 ++++++++++++++++++ cmux-tui/crates/cmux-tui/src/session/mod.rs | 8 + .../crates/cmux-tui/src/session/remote.rs | 25 +- cmux-tui/crates/cmux-tui/tests/cli.rs | 185 +++++++++- cmux-tui/dist/RELEASING-TUI.md | 8 +- cmux-tui/docs/getting-started.md | 9 + cmux-tui/docs/protocol.md | 6 +- cmux-tui/spec/commands.md | 32 +- 17 files changed, 1064 insertions(+), 158 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/release.rs create mode 100644 cmux-tui/crates/cmux-tui/src/server_lifecycle.rs diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index f1a3c1f6b4f8..7600a3a84447 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -107,9 +107,10 @@ jobs: # Package artifacts must build and stamp the checked-out submodule, # independent of any self-hosted runner source override. unset CMUX_GHOSTTY_SRC + CMUX_TUI_DISTRIBUTION_VERSION="${{ inputs.version }}" CMUX_TUI_BUILD_COMMIT="$(git -C .. rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ../ghostty rev-parse HEAD)" - export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT + export CMUX_TUI_DISTRIBUTION_VERSION CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT cargo build -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.target }} - name: Build cmux-tui (cross) @@ -120,9 +121,10 @@ jobs: # Package artifacts must build and stamp the checked-out submodule, # independent of any self-hosted runner source override. unset CMUX_GHOSTTY_SRC + CMUX_TUI_DISTRIBUTION_VERSION="${{ inputs.version }}" CMUX_TUI_BUILD_COMMIT="$(git -C .. rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ../ghostty rev-parse HEAD)" - export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT + export CMUX_TUI_DISTRIBUTION_VERSION CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT cargo zigbuild -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.target }} - name: Stage binary @@ -181,9 +183,10 @@ jobs: # Keep the manual Zig build and Cargo's build script on the same # checked-out Ghostty source whose revision is stamped below. unset CMUX_GHOSTTY_SRC + CMUX_TUI_DISTRIBUTION_VERSION="${{ inputs.version }}" CMUX_TUI_BUILD_COMMIT="$(git rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ghostty rev-parse HEAD)" - export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT + export CMUX_TUI_DISTRIBUTION_VERSION CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT cd ghostty zig build -Demit-lib-vt=true -Demit-xcframework=false -Doptimize=ReleaseFast -Dtarget=x86_64-windows-gnu --prefix "$RUNNER_TEMP/ghostty-vt-win-gnu" cd ../cmux-tui diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 98e5b10ef8b6..d7871b155f10 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -19,6 +19,7 @@ mod surface; pub mod layout; pub mod platform; +pub mod release; pub mod server; pub use browser::{TRANSPORT_SAFE_CAPTURE_MEGAPIXELS, normalize_url}; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 3ba745957e1d..bc63c19b8b5b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3,9 +3,7 @@ use std::collections::{HashMap, HashSet}; use std::fmt; -#[cfg(test)] -use std::sync::atomic::AtomicBool; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; use std::sync::{Arc, Mutex, MutexGuard, Weak}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -641,6 +639,7 @@ impl ClientSizingState { pub struct Mux { state: Mutex, subscribers: MuxEventBroadcaster, + shutdown_requested: AtomicBool, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -760,6 +759,7 @@ impl Mux { split_screens: HashMap::new(), }), subscribers: MuxEventBroadcaster::default(), + shutdown_requested: AtomicBool::new(false), next_id: AtomicU64::new(1), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -2088,6 +2088,14 @@ impl Mux { } } + pub fn request_shutdown(&self) { + self.shutdown_requested.store(true, Ordering::Release); + } + + pub fn shutdown_requested(&self) -> bool { + self.shutdown_requested.load(Ordering::Acquire) + } + /// Update options used for future surface/browser launches. pub fn update_surface_options(&self, update: impl FnOnce(&mut SurfaceOptions)) { let mut options = self.surface_options.lock().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/release.rs b/cmux-tui/crates/cmux-tui-core/src/release.rs new file mode 100644 index 000000000000..ccd5c9eca88d --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/release.rs @@ -0,0 +1,103 @@ +use serde_json::Value; + +/// The build identity that must match between a local client and server. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReleaseIdentity { + pub version: String, + pub build_commit: Option, + pub ghostty_commit: Option, + pub protocol: u32, +} + +impl ReleaseIdentity { + pub fn current(protocol: u32) -> Self { + Self { + version: distribution_version().to_string(), + build_commit: stamped_build_commit().map(str::to_string), + ghostty_commit: stamped_ghostty_commit().map(str::to_string), + protocol, + } + } + + pub fn from_protocol_data(data: &Value) -> Self { + Self { + version: data + .get("version") + .and_then(Value::as_str) + .filter(|version| !version.is_empty()) + .unwrap_or("unknown") + .to_string(), + build_commit: optional_string(data, "build_commit"), + ghostty_commit: optional_string(data, "ghostty_commit"), + protocol: data + .get("protocol") + .and_then(Value::as_u64) + .and_then(|protocol| u32::try_from(protocol).ok()) + .unwrap_or(0), + } + } + + pub fn exactly_matches(&self, other: &Self) -> bool { + self == other + } + + pub fn version_with_build_metadata(&self) -> String { + match (&self.build_commit, &self.ghostty_commit) { + (Some(commit), Some(ghostty)) => { + format!("{} ({commit}; ghostty {ghostty})", self.version) + } + (Some(commit), None) => format!("{} ({commit})", self.version), + (None, _) => self.version.clone(), + } + } +} + +pub fn distribution_version() -> &'static str { + option_env!("CMUX_TUI_DISTRIBUTION_VERSION") + .filter(|version| !version.is_empty()) + .unwrap_or(env!("CARGO_PKG_VERSION")) +} + +pub fn stamped_build_commit() -> Option<&'static str> { + option_env!("CMUX_TUI_BUILD_COMMIT") + .or(option_env!("CMUX_MUX_BUILD_COMMIT")) + .filter(|commit| !commit.is_empty()) +} + +pub fn stamped_ghostty_commit() -> Option<&'static str> { + option_env!("CMUX_TUI_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()) +} + +fn optional_string(data: &Value, key: &str) -> Option { + data.get(key).and_then(Value::as_str).filter(|value| !value.is_empty()).map(str::to_string) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn protocol_data_defaults_missing_identity_fields_conservatively() { + assert_eq!( + ReleaseIdentity::from_protocol_data(&json!({"protocol": 9})), + ReleaseIdentity { + version: "unknown".to_string(), + build_commit: None, + ghostty_commit: None, + protocol: 9, + } + ); + } + + #[test] + fn exact_matching_includes_source_identities() { + let current = ReleaseIdentity::current(9); + let mut other = current.clone(); + assert!(current.exactly_matches(&other)); + + other.build_commit = Some("different".to_string()); + assert!(!current.exactly_matches(&other)); + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index f442ccdc0743..c2c6163426ed 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -43,6 +43,7 @@ use zeroize::Zeroize; use crate::model::{Screen, State, Workspace}; use crate::mux::clamp_terminal_size; use crate::platform::{self, transport}; +use crate::release::ReleaseIdentity; use crate::surface::AttachLifecycle; use crate::{ AgentRecord, AgentSource, AgentState, AttachFrame, DefaultColors, Direction, LayoutLeafSpec, @@ -78,6 +79,7 @@ struct Request { enum Command { Identify, Ping, + Shutdown, SetClientInfo { #[serde(default)] name: Option, @@ -1753,11 +1755,13 @@ pub fn detach_control_client(mux: &Mux, client: u64) -> bool { fn handle_message(mux: &Arc, client: u64, message: &str, writer: &MessageWriter) -> bool { let mut detach_self = false; + let mut shutdown = false; let response = match serde_json::from_str::(message) { Ok(req) => { let id = req.id.clone(); detach_self = matches!(&req.cmd, Command::DetachClient { client: target } if *target == client); + shutdown = matches!(&req.cmd, Command::Shutdown); match handle_command(mux, client, req.cmd, writer) { Ok(data) => Response { id, ok: true, data: Some(data), error: None }, Err(e) => Response { id, ok: false, data: None, error: Some(e.to_string()) }, @@ -1774,6 +1778,10 @@ fn handle_message(mux: &Arc, client: u64, message: &str, writer: &MessageWr disconnect_client(mux, client, true); return false; } + if shutdown && response_ok && sent { + mux.request_shutdown(); + return true; + } sent } @@ -2744,27 +2752,39 @@ fn handle_command( writer: &MessageWriter, ) -> anyhow::Result { match cmd { - Command::Identify => Ok(json!({ - "app": "cmux-tui", - "version": env!("CARGO_PKG_VERSION"), - "build_commit": stamped_build_commit(), - "ghostty_commit": stamped_ghostty_commit(), - "protocol": PROTOCOL_VERSION, - "capabilities": [ - ATTACH_INITIAL_SIZE_CAPABILITY, - WORKSPACE_REGISTRY_CAPABILITY, - PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY - ], - "session": mux.session, - "pid": std::process::id(), - })), - Command::Ping => Ok(json!({ - "ok": true, - "version": env!("CARGO_PKG_VERSION"), - "build_commit": stamped_build_commit(), - "ghostty_commit": stamped_ghostty_commit(), - "protocol": PROTOCOL_VERSION, - })), + Command::Identify => { + let release = ReleaseIdentity::current(PROTOCOL_VERSION); + Ok(json!({ + "app": "cmux-tui", + "version": release.version, + "build_commit": release.build_commit, + "ghostty_commit": release.ghostty_commit, + "protocol": release.protocol, + "capabilities": [ + ATTACH_INITIAL_SIZE_CAPABILITY, + WORKSPACE_REGISTRY_CAPABILITY, + PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY + ], + "session": mux.session, + "pid": std::process::id(), + })) + } + Command::Ping => { + let release = ReleaseIdentity::current(PROTOCOL_VERSION); + Ok(json!({ + "ok": true, + "version": release.version, + "build_commit": release.build_commit, + "ghostty_commit": release.ghostty_commit, + "protocol": release.protocol, + })) + } + Command::Shutdown => { + if !mux.control_clients.is_unix(client) { + anyhow::bail!("shutdown is only available over the local session socket"); + } + Ok(json!({})) + } Command::SetClientInfo { name, kind } => { let (name, kind) = mux.control_clients.set_info(client, name, kind)?; mux.emit(MuxEvent::ClientChanged { client, name, kind }); @@ -3949,16 +3969,6 @@ fn handle_command( } } -fn stamped_build_commit() -> Option<&'static str> { - option_env!("CMUX_TUI_BUILD_COMMIT") - .or(option_env!("CMUX_MUX_BUILD_COMMIT")) - .filter(|commit| !commit.is_empty()) -} - -fn stamped_ghostty_commit() -> Option<&'static str> { - option_env!("CMUX_TUI_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()) -} - fn subscribed_event_json(event: &MuxEvent) -> Value { match event { MuxEvent::SurfaceOutput(id) => json!({"event": "surface-output", "surface": id}), @@ -4449,24 +4459,50 @@ mod tests { #[test] fn identify_and_ping_return_build_metadata() { let mux = test_mux(); + let release = ReleaseIdentity::current(PROTOCOL_VERSION); let identity = handle_command(&mux, 0, Command::Identify, &test_writer()).unwrap(); assert_eq!(identity["app"].as_str(), Some("cmux-tui")); - assert_eq!(identity["version"].as_str(), Some(env!("CARGO_PKG_VERSION"))); + assert_eq!(identity["version"].as_str(), Some(release.version.as_str())); assert_eq!(identity["protocol"].as_u64(), Some(PROTOCOL_VERSION as u64)); - assert_eq!(identity["build_commit"].as_str(), stamped_build_commit()); - assert_eq!(identity["ghostty_commit"].as_str(), stamped_ghostty_commit()); + assert_eq!(identity["build_commit"].as_str(), release.build_commit.as_deref()); + assert_eq!(identity["ghostty_commit"].as_str(), release.ghostty_commit.as_deref()); let data = handle_command(&mux, 0, Command::Ping, &test_writer()).unwrap(); assert_eq!(data["ok"].as_bool(), Some(true)); - assert_eq!(data["version"].as_str(), Some(env!("CARGO_PKG_VERSION"))); - assert_eq!(data["build_commit"].as_str(), stamped_build_commit()); - assert_eq!(data["ghostty_commit"].as_str(), stamped_ghostty_commit()); + assert_eq!(data["version"].as_str(), Some(release.version.as_str())); + assert_eq!(data["build_commit"].as_str(), release.build_commit.as_deref()); + assert_eq!(data["ghostty_commit"].as_str(), release.ghostty_commit.as_deref()); assert_eq!(data["protocol"].as_u64(), Some(PROTOCOL_VERSION as u64)); assert_eq!(STABLE_SPLIT_IDS_PROTOCOL_VERSION, 8); assert_eq!(STACK_LAYOUT_PROTOCOL_VERSION, 9); assert_eq!(PROTOCOL_VERSION, 9); } + #[test] + fn local_shutdown_responds_before_requesting_mux_exit() { + let mux = test_mux(); + let outbound = Arc::new(BoundedOutbound::default()); + let writer = MessageWriter::new(QueuedSink { outbound: outbound.clone(), control: None }); + let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + + assert!(handle_message(&mux, client, r#"{"id":7,"cmd":"shutdown"}"#, &writer)); + assert!(mux.shutdown_requested()); + let response: Value = serde_json::from_str(&outbound.try_pop().unwrap()).unwrap(); + assert_eq!(response, json!({"id": 7, "ok": true, "data": {}})); + } + + #[test] + fn websocket_clients_cannot_shutdown_the_server() { + let mux = test_mux(); + let writer = test_writer(); + let client = mux.control_clients.register(ClientTransport::WebSocket, writer.clone()); + + let error = handle_command(&mux, client, Command::Shutdown, &writer).unwrap_err(); + + assert!(error.to_string().contains("local session socket")); + assert!(!mux.shutdown_requested()); + } + #[test] fn split_ids_serialize_stably_and_both_ratio_commands_work() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index d1d40a7adb19..63da4406187f 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -17,8 +17,8 @@ use std::time::{Duration, Instant}; use base64::Engine; use cmux_tui_core::{ - BrowserSource, BrowserStatus, Direction, MuxEvent, PairingChallenge, PaneId, Rect, SplitDir, - SplitEdge, SplitId, SurfaceId, SurfaceKind, WorkspaceId, exact_split_for_pane_edge, + BrowserSource, BrowserStatus, Direction, Mux, MuxEvent, PairingChallenge, PaneId, Rect, + SplitDir, SplitEdge, SplitId, SurfaceId, SurfaceKind, WorkspaceId, exact_split_for_pane_edge, layout_screen, split_sides, zellij_default_pane_layout, }; use crossterm::ExecutableCommand; @@ -923,6 +923,10 @@ impl OrderedSession { self.inner.tree() } + fn shutdown_requested(&self) -> bool { + self.inner.shutdown_requested() + } + fn respond_pairing(&self, request: u64, approve: bool) -> anyhow::Result<()> { self.inner.respond_pairing(request, approve) } @@ -2885,6 +2889,7 @@ impl PaneFocusHistory { pub struct App { pub session: OrderedSession, + host_mux: Option>, session_event_worker: Option, session_generation: u64, app_events: SyncSender, @@ -3673,6 +3678,7 @@ pub fn run_with_machine_updates( default_colors: cmux_tui_core::DefaultColors, machine_ui: Option, machine_controller: Option>, + host_mux: Option>, ) -> anyhow::Result { let mut config = crate::config::load(); let chrome = ChromeTheme::for_defaults(config.chrome, default_colors); @@ -3782,6 +3788,7 @@ pub fn run_with_machine_updates( let initial_machine_notice = machine_ui.as_ref().and_then(|machine| machine.notice.clone()); let mut app = App { session, + host_mux, session_event_worker: Some(session_event_worker), session_generation, app_events: tx, @@ -4024,7 +4031,7 @@ impl App { self.draw_terminal(terminal)?; self.emit_graphics()?; - while !self.quit && !crate::shutdown_requested() { + while !self.quit && !self.shutdown_requested() { // Block for the first event, then drain whatever queued so a // torrent of pty output coalesces into one frame. let timeout = if self.shake_frames > 0 @@ -4114,6 +4121,12 @@ impl App { Ok(()) } + fn shutdown_requested(&self) -> bool { + crate::shutdown_requested() + || self.session.shutdown_requested() + || self.host_mux.as_ref().is_some_and(|mux| mux.shutdown_requested()) + } + fn restart_machine_updates(&mut self) -> anyhow::Result<()> { let Some(worker) = self.machine_action_worker.as_ref() else { return Ok(()); @@ -15874,6 +15887,19 @@ mod tests { worker.stop_and_join(); } + #[test] + fn host_mux_owns_shutdown_after_the_rendered_session_changes() { + let rendered_mux = Mux::new("rendered", SurfaceOptions::default()); + let host_mux = Mux::new("host", SurfaceOptions::default()); + let mut app = test_app(Session::Local(rendered_mux)); + app.host_mux = Some(host_mux.clone()); + assert!(!app.shutdown_requested()); + + host_mux.request_shutdown(); + + assert!(app.shutdown_requested()); + } + fn test_app(session: Session) -> App { test_app_with_events(session).0 } @@ -15884,6 +15910,7 @@ mod tests { let session = OrderedSession::new(session, pty_input.sender(), events.clone()); let app = App { session, + host_mux: None, session_event_worker: None, session_generation: 1, app_events: events, diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index c0c6a6e5d211..8d78d20f432e 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1,13 +1,14 @@ use std::collections::BTreeMap; use std::io::{self, BufRead, BufReader, Read, Write}; use std::path::PathBuf; -use std::time::{Duration, Instant}; +use std::time::Duration; use cmux_tui_core::platform::transport; +use cmux_tui_core::release::ReleaseIdentity; +use cmux_tui_core::server::PROTOCOL_VERSION; use serde_json::{Value, json}; const REQUEST_ID: u64 = 1; -const CAPABILITY_REQUEST_ID: u64 = 0; const ATTACH_INITIAL_SIZE_CAPABILITY: &str = "attach-initial-size"; type BuildFn = fn(&FlagMap) -> Result; @@ -398,6 +399,12 @@ const VERBS: &[VerbSpec] = &[ allowed: &["name", "force", "builtin"], kind: VerbKind::Local(run_plugin), }, + VerbSpec { + name: "server", + help: "Inspect or stop the local server.", + allowed: &[], + kind: VerbKind::Local(run_server_lifecycle), + }, ]; const fn socket(build: BuildFn, print: PrintFn, stream: bool) -> VerbKind { @@ -427,7 +434,12 @@ pub fn print_help(usage: &str) { println!(); println!("VERB HELP"); for verb in VERBS { - println!(" {:<18} {}", verb.name, verb.help); + let help = if verb.name == "server" { + crate::localization::catalog().server.help + } else { + verb.help + }; + println!(" {:<18} {help}", verb.name); } } @@ -581,20 +593,30 @@ fn run_command(args: CliArgs) -> i32 { let _ = stream.set_read_timeout(Some(Duration::from_secs(10))); } let mut reader = BufReader::new(stream); - if request.get("cmd").and_then(Value::as_str) == Some("attach-surface") - && request.get("cols").is_some() - { - match server_supports_capability(&mut reader, ATTACH_INITIAL_SIZE_CAPABILITY) { - Ok(true) => {} - Ok(false) => { - eprintln!("initial attach sizing is not supported by this server"); - return 1; - } - Err(err) => { - eprintln!("{err}"); + let probe = if args.verb.name == "identify" { + None + } else { + let probe = match crate::server_lifecycle::ServerProbe::inspect(&mut reader) { + Ok(probe) => probe, + Err(error) => { + eprintln!("{error}"); return 3; } + }; + if let Err(error) = probe.require_compatible(&socket_path) { + eprintln!("{error}"); + return 1; } + Some(probe) + }; + if request.get("cmd").and_then(Value::as_str) == Some("attach-surface") + && request.get("cols").is_some() + && !probe + .as_ref() + .is_some_and(|probe| probe.identity.supports(ATTACH_INITIAL_SIZE_CAPABILITY)) + { + eprintln!("initial attach sizing is not supported by this server"); + return 1; } if let Err(err) = write_json_line(reader.get_mut(), &request) { eprintln!("transport error: {err}"); @@ -612,52 +634,14 @@ fn write_json_line(writer: &mut dyn Write, value: &Value) -> io::Result<()> { writer.write_all(b"\n") } +#[cfg(test)] fn server_supports_capability( reader: &mut BufReader>, capability: &str, ) -> Result { - write_json_line(reader.get_mut(), &json!({"id": CAPABILITY_REQUEST_ID, "cmd": "identify"})) - .map_err(|err| format!("transport error: {err}"))?; - let deadline = Instant::now() + Duration::from_secs(10); - let mut line = String::new(); - - loop { - match reader.read_line(&mut line) { - Ok(0) => return Err("transport closed before identify response".to_string()), - Ok(_) => {} - Err(err) - if matches!(err.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) - && Instant::now() < deadline => - { - continue; - } - Err(err) - if matches!(err.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) => - { - return Err("timed out waiting for identify response".to_string()); - } - Err(err) => return Err(format!("transport error: {err}")), - } - let value: Value = - serde_json::from_str(&line).map_err(|err| format!("bad identify response: {err}"))?; - if value.get("event").is_some() - || value.get("id").and_then(Value::as_u64) != Some(CAPABILITY_REQUEST_ID) - { - line.clear(); - continue; - } - if value.get("ok").and_then(Value::as_bool) != Some(true) { - return Err(value - .get("error") - .and_then(Value::as_str) - .unwrap_or("identify failed") - .to_string()); - } - return Ok(value - .pointer("/data/capabilities") - .and_then(Value::as_array) - .is_some_and(|values| values.iter().any(|value| value.as_str() == Some(capability)))); - } + crate::server_lifecycle::ServerProbe::inspect(reader) + .map(|probe| probe.identity.supports(capability)) + .map_err(|error| error.to_string()) } fn is_boolean_flag(spec: &VerbSpec, name: &str) -> bool { @@ -680,6 +664,111 @@ fn run_plugin(global: &GlobalArgs, flags: &FlagMap) -> i32 { ) } +fn run_server_lifecycle(global: &GlobalArgs, flags: &FlagMap) -> i32 { + let messages = &crate::localization::catalog().server; + let action = match flags.positionals.as_slice() { + [action] => action.as_str(), + [] => { + eprintln!("cmux-tui: {}", messages.missing_action); + return 2; + } + _ => { + eprintln!("cmux-tui: {} {:?}", messages.unknown_action, flags.positionals); + return 2; + } + }; + if !flags.values.is_empty() { + eprintln!("cmux-tui: {} {:?}", messages.unknown_action, flags.values); + return 2; + } + if !matches!(action, "status" | "stop") { + eprintln!("cmux-tui: {} {action:?}", messages.unknown_action); + return 2; + } + let socket_path = resolve_socket(global); + let lifecycle = match crate::server_lifecycle::ServerLifecycle::connect(socket_path) { + Ok(lifecycle) => lifecycle, + Err(error) => { + eprintln!("{error}"); + return 3; + } + }; + + match action { + "status" => print_server_status(global.json, lifecycle.probe()), + "stop" => match lifecycle.stop() { + Ok(()) => { + println!("{}", messages.stopped); + 0 + } + Err(error) => { + eprintln!("{error}"); + 1 + } + }, + _ => unreachable!("server action was validated before connecting"), + } +} + +fn print_server_status(json_output: bool, probe: &crate::server_lifecycle::ServerProbe) -> i32 { + let messages = &crate::localization::catalog().server; + let server = &probe.identity; + let client = ReleaseIdentity::current(PROTOCOL_VERSION); + let compatible = probe.is_compatible(); + if json_output { + let value = json!({ + "running": true, + "compatible": compatible, + "server": { + "version": server.release.version, + "build_commit": server.release.build_commit, + "ghostty_commit": server.release.ghostty_commit, + "protocol": server.release.protocol, + "session": server.session, + "pid": server.pid, + }, + "client": { + "version": client.version, + "build_commit": client.build_commit, + "ghostty_commit": client.ghostty_commit, + "protocol": client.protocol, + }, + }); + let mut stdout = io::stdout(); + match serde_json::to_writer(&mut stdout, &value) + .map_err(io::Error::other) + .and_then(|_| stdout.write_all(b"\n")) + { + Ok(()) => 0, + Err(error) => { + eprintln!("stdout error: {error}"); + 3 + } + } + } else { + let server_version = server.release.version_with_build_metadata(); + let client_version = client.version_with_build_metadata(); + println!( + "{}: v{} {} {} pid={}", + messages.server_label, + server_version, + messages.protocol_label, + server.release.protocol, + server.pid, + ); + println!( + "{}: v{} {} {}", + messages.client_label, client_version, messages.protocol_label, client.protocol, + ); + println!( + "{}: {}", + messages.status_label, + if compatible { messages.compatible } else { messages.incompatible }, + ); + 0 + } +} + fn resolve_socket(global: &GlobalArgs) -> PathBuf { if let Some(path) = &global.socket { return path.clone(); @@ -1334,7 +1423,8 @@ fn print_empty(_: &Value, _: &mut dyn Write) -> io::Result<()> { fn print_identify(data: &Value, out: &mut dyn Write) -> io::Result<()> { writeln!( out, - "cmux-tui session={} protocol={} pid={}", + "cmux-tui version={} session={} protocol={} pid={}", + data.get("version").and_then(Value::as_str).unwrap_or(""), data.get("session").and_then(Value::as_str).unwrap_or(""), data.get("protocol").and_then(Value::as_u64).unwrap_or(0), data.get("pid").and_then(Value::as_u64).unwrap_or(0) @@ -1674,7 +1764,7 @@ mod tests { reads: VecDeque::from([ Err(io::ErrorKind::WouldBlock), Err(io::ErrorKind::TimedOut), - Ok(b"{\"id\":0,\"ok\":true,\"data\":{\"capabilities\":[\"attach-initial-size\"]}}\n" + Ok(b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",\"capabilities\":[\"attach-initial-size\"]}}\n" .to_vec()), ]), current: io::Cursor::new(Vec::new()), @@ -1692,9 +1782,9 @@ mod tests { fn capability_probe_preserves_partial_line_across_timeout() { let stream = ScriptedStream { reads: VecDeque::from([ - Ok(b"{\"id\":0,\"ok\":true,\"data\":".to_vec()), + Ok(b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",".to_vec()), Err(io::ErrorKind::TimedOut), - Ok(b"{\"capabilities\":[\"attach-initial-size\"]}}\n".to_vec()), + Ok(b"\"capabilities\":[\"attach-initial-size\"]}}\n".to_vec()), ]), current: io::Cursor::new(Vec::new()), writes: Vec::new(), diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 69715033940e..aef978db6856 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -19,6 +19,26 @@ pub(crate) struct ForeignViewportMessages { pub terminal_grid: &'static str, } +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct ServerMessages { + pub help: &'static str, + pub incompatible_local_server: &'static str, + pub server_label: &'static str, + pub client_label: &'static str, + pub protocol_label: &'static str, + pub status_label: &'static str, + pub compatible: &'static str, + pub incompatible: &'static str, + pub stop_to_use: &'static str, + pub stopping_exits_panes: &'static str, + pub restart_instruction: &'static str, + pub stopped: &'static str, + pub missing_action: &'static str, + pub unknown_action: &'static str, + pub shutdown_unsupported: &'static str, + pub shutdown_timed_out: &'static str, +} + #[derive(Debug, PartialEq, Eq)] pub(crate) struct SidebarMessages { pub machines: &'static str, @@ -132,6 +152,7 @@ const fn decimal_width(mut value: u16) -> usize { pub(crate) struct Catalog { pub pairing: PairingMessages, pub foreign_viewport: ForeignViewportMessages, + pub server: ServerMessages, pub sidebar: SidebarMessages, } @@ -144,6 +165,24 @@ static ENGLISH: Catalog = Catalog { approve: "[ Approve enter ]", }, foreign_viewport: ForeignViewportMessages { terminal_grid: "terminal grid" }, + server: ServerMessages { + help: "Inspect or stop the local server.", + incompatible_local_server: "This session is running a different cmux-tui release.", + server_label: "server", + client_label: "client", + protocol_label: "protocol", + status_label: "status", + compatible: "compatible", + incompatible: "incompatible", + stop_to_use: "Stop the old server to use this cmux-tui version.", + stopping_exits_panes: "Stopping exits pane processes.", + restart_instruction: "Run `{command}`, then run `cmux-tui` again.", + stopped: "Stopped the cmux-tui server.", + missing_action: "server needs an action: status or stop", + unknown_action: "unknown server action", + shutdown_unsupported: "this server cannot be stopped by this client", + shutdown_timed_out: "the server did not stop within 10 seconds", + }, sidebar: SidebarMessages { machines: "machines", workspaces: "workspaces", @@ -224,6 +263,24 @@ static JAPANESE: Catalog = Catalog { approve: "[ 承認 enter ]", }, foreign_viewport: ForeignViewportMessages { terminal_grid: "端末グリッド" }, + server: ServerMessages { + help: "ローカルサーバーの確認または停止。", + incompatible_local_server: "このセッションは別の cmux-tui リリースで実行されています。", + server_label: "サーバー", + client_label: "クライアント", + protocol_label: "プロトコル", + status_label: "状態", + compatible: "互換", + incompatible: "非互換", + stop_to_use: "この cmux-tui バージョンを使うには、古いサーバーを停止してください。", + stopping_exits_panes: "停止するとペインのプロセスが終了します。", + restart_instruction: "`{command}` を実行してから、`cmux-tui` をもう一度実行してください。", + stopped: "cmux-tui サーバーを停止しました。", + missing_action: "server には status または stop の操作が必要です", + unknown_action: "不明な server 操作", + shutdown_unsupported: "このクライアントからこのサーバーを停止できません", + shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", + }, sidebar: SidebarMessages { machines: "マシン", workspaces: "ワークスペース", diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 1a2bf7b4ebb0..2d97cb9dce38 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -23,6 +23,7 @@ mod process_diagnostics; #[cfg(target_os = "linux")] mod provider_authority; mod pty_input; +mod server_lifecycle; mod session; mod sidebar_files; mod ui; @@ -459,20 +460,8 @@ fn parse_args_result(args: impl IntoIterator) -> Result String { - // Packaged builds stamp both source identities so artifact validation can - // reject a cmux binary built against a different Ghostty checkout before - // it enters an app bundle. Local builds report the crate version alone. - let commit = option_env!("CMUX_TUI_BUILD_COMMIT") - .or(option_env!("CMUX_MUX_BUILD_COMMIT")) - .filter(|commit| !commit.is_empty()); - let ghostty = option_env!("CMUX_TUI_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()); - match (commit, ghostty) { - (Some(commit), Some(ghostty)) => { - format!("{} ({commit}; ghostty {ghostty})", env!("CARGO_PKG_VERSION")) - } - (Some(commit), None) => format!("{} ({commit})", env!("CARGO_PKG_VERSION")), - (None, _) => env!("CARGO_PKG_VERSION").to_string(), - } + cmux_tui_core::release::ReleaseIdentity::current(cmux_tui_core::server::PROTOCOL_VERSION) + .version_with_build_metadata() } impl Args { @@ -840,16 +829,21 @@ fn run_server( .socket .clone() .unwrap_or_else(|| cmux_tui_core::server::default_socket_path(&args.session)); - if args.should_attach_existing(&ws_addr, &ws_token) - && socket_path.exists() - && let Ok(remote) = RemoteSession::connect(&socket_path) - { - return run_connected_session_client( - socket_path, - args.session, - config, - Session::Remote(remote), - ); + if args.should_attach_existing(&ws_addr, &ws_token) && socket_path.exists() { + match RemoteSession::connect(&socket_path) { + Ok(remote) => { + return run_connected_session_client( + socket_path, + args.session, + config, + Session::Remote(remote), + ); + } + Err(error) if cmux_tui_core::platform::transport::connect(&socket_path).is_ok() => { + return Err(error); + } + Err(_) => {} + } } let mut surface_options = SurfaceOptions::default(); @@ -904,7 +898,7 @@ fn run_server( let result = if args.headless { run_headless(&mux, &socket_path) } else if let Some(runtime) = machine_runtime { - run_machine_client(runtime) + run_machine_client(runtime, mux.clone()) } else { run_tui(Session::Local(mux.clone()), args.session) }; @@ -915,7 +909,7 @@ fn run_server( } fn run_tui(session: Session, session_label: String) -> anyhow::Result<()> { - match run_tui_once(session, session_label, None, None)? { + match run_tui_once(session, session_label, None, None, None)? { app::RunOutcome::Quit => Ok(()), app::RunOutcome::Machine(_) => { anyhow::bail!("machine request returned without a machine runtime") @@ -947,27 +941,28 @@ fn run_connected_session_client( SessionClientMode::Plain => run_tui(session, session_label), SessionClientMode::Machines => { let runtime = MachineRuntime::new(socket_path, config.machines); - run_machine_client_with_initial(runtime, session) + run_machine_client_with_initial(runtime, session, None) } } } -fn run_machine_client(mut runtime: MachineRuntime) -> anyhow::Result<()> { +fn run_machine_client(mut runtime: MachineRuntime, host_mux: Arc) -> anyhow::Result<()> { let active = runtime.initial_key(); let session = runtime.connect(active)?; - run_machine_client_with_initial(runtime, session) + run_machine_client_with_initial(runtime, session, Some(host_mux)) } fn run_machine_client_with_initial( runtime: MachineRuntime, session: Session, + host_mux: Option>, ) -> anyhow::Result<()> { let active = runtime.initial_key(); let label = runtime.name(active).unwrap_or("machine").to_string(); let machine_ui = MachineUiState::new(runtime.snapshot(active)); let controller: Box = Box::new(StaticMachineController { runtime, active, pending_active: None }); - match run_tui_once(session, label, Some(machine_ui), Some(controller))? { + match run_tui_once(session, label, Some(machine_ui), Some(controller), host_mux)? { app::RunOutcome::Quit => Ok(()), app::RunOutcome::Machine(_) => { anyhow::bail!("machine request escaped its in-place controller") @@ -1057,7 +1052,7 @@ fn run_provider_machine_client( )), }; let controller: Box = Box::new(runtime); - match run_tui_once(session, label, Some(machine_ui), Some(controller))? { + match run_tui_once(session, label, Some(machine_ui), Some(controller), None)? { app::RunOutcome::Quit => Ok(()), app::RunOutcome::Machine(_) => { anyhow::bail!("provider request escaped its in-place controller") @@ -1077,6 +1072,7 @@ fn run_tui_once( session_label: String, machine_ui: Option, machine_controller: Option>, + host_mux: Option>, ) -> anyhow::Result { crossterm::terminal::enable_raw_mode()?; let config = config::load(); @@ -1094,7 +1090,14 @@ fn run_tui_once( eprintln!("cmux-tui: failed to set default colors: {err}"); } raw_result?; - app::run_with_machine_updates(session, session_label, colors, machine_ui, machine_controller) + app::run_with_machine_updates( + session, + session_label, + colors, + machine_ui, + machine_controller, + host_mux, + ) } fn run_headless(mux: &Arc, socket_path: &std::path::Path) -> anyhow::Result<()> { @@ -1103,7 +1106,7 @@ fn run_headless(mux: &Arc, socket_path: &std::path::Path) -> anyhow::Result // the mux reaps exited surfaces itself. let events = mux.subscribe(); loop { - if shutdown_requested() { + if shutdown_requested() || mux.shutdown_requested() { break; } match events.recv_timeout(std::time::Duration::from_millis(250)) { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs new file mode 100644 index 000000000000..30578d29ff16 --- /dev/null +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -0,0 +1,343 @@ +use std::collections::HashSet; +use std::io::{BufRead, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::Context; +use cmux_tui_core::platform::transport; +use cmux_tui_core::release::ReleaseIdentity; +use cmux_tui_core::server::PROTOCOL_VERSION; +use serde_json::{Value, json}; + +const PROBE_REQUEST_ID: u64 = 0; +const SHUTDOWN_REQUEST_ID: u64 = 1; +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); + +#[derive(Clone, Debug)] +pub(crate) struct ServerIdentity { + pub release: ReleaseIdentity, + pub session: String, + pub pid: u32, + capabilities: HashSet, +} + +impl ServerIdentity { + fn from_protocol_data(data: &Value) -> anyhow::Result { + if data.get("app").and_then(Value::as_str) != Some("cmux-tui") { + anyhow::bail!("socket endpoint is not a cmux-tui session"); + } + let pid = data + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| u32::try_from(pid).ok()) + .unwrap_or(0); + let capabilities = data + .get("capabilities") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + Ok(Self { + release: ReleaseIdentity::from_protocol_data(data), + session: data.get("session").and_then(Value::as_str).unwrap_or("").to_string(), + pid, + capabilities, + }) + } + + pub(crate) fn supports(&self, capability: &str) -> bool { + self.capabilities.contains(capability) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ServerProbe { + pub identity: ServerIdentity, +} + +impl ServerProbe { + pub(crate) fn inspect( + reader: &mut BufReader>, + ) -> anyhow::Result { + write_json_line(reader.get_mut(), &json!({"id": PROBE_REQUEST_ID, "cmd": "identify"})) + .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + let response = read_response(reader, PROBE_REQUEST_ID)?; + if response.get("ok").and_then(Value::as_bool) != Some(true) { + anyhow::bail!( + "{}", + response.get("error").and_then(Value::as_str).unwrap_or("identify failed") + ); + } + let data = response.get("data").unwrap_or(&Value::Null); + Ok(Self { identity: ServerIdentity::from_protocol_data(data)? }) + } + + pub(crate) fn connect( + path: &Path, + ) -> anyhow::Result<(Self, BufReader>)> { + let stream = transport::connect(path).map_err(|error| { + anyhow::anyhow!("cannot connect to session socket {}: {error}", path.display()) + })?; + stream.set_read_timeout(Some(RESPONSE_TIMEOUT))?; + let mut reader = BufReader::new(stream); + let probe = Self::inspect(&mut reader)?; + Ok((probe, reader)) + } + + pub(crate) fn is_compatible(&self) -> bool { + self.identity.release.exactly_matches(&ReleaseIdentity::current(PROTOCOL_VERSION)) + } + + pub(crate) fn require_compatible(&self, path: &Path) -> anyhow::Result<()> { + if self.is_compatible() { + return Ok(()); + } + Err(IncompatibleLocalServer { message: incompatible_server_message(&self.identity, path) } + .into()) + } +} + +#[derive(Debug)] +pub(crate) struct IncompatibleLocalServer { + message: String, +} + +impl std::fmt::Display for IncompatibleLocalServer { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for IncompatibleLocalServer {} + +pub(crate) struct ServerLifecycle { + path: PathBuf, + probe: ServerProbe, + reader: BufReader>, +} + +impl ServerLifecycle { + pub(crate) fn connect(path: PathBuf) -> anyhow::Result { + let (probe, reader) = ServerProbe::connect(&path)?; + Ok(Self { path, probe, reader }) + } + + pub(crate) fn probe(&self) -> &ServerProbe { + &self.probe + } + + pub(crate) fn stop(mut self) -> anyhow::Result<()> { + write_json_line( + self.reader.get_mut(), + &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), + ) + .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + + let accepted = match read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID) { + Ok(response) => response.get("ok").and_then(Value::as_bool) == Some(true), + Err(_error) if connection_is_gone(&self.path) => return Ok(()), + Err(error) => return Err(error), + }; + if !accepted { + self.stop_legacy_server().context("failed to signal the older server")?; + } + wait_for_disconnect(&mut self.reader).context("failed while waiting for the server to stop") + } + + #[cfg(unix)] + fn stop_legacy_server(&self) -> anyhow::Result<()> { + terminate_legacy_server(self.probe.identity.pid) + } + + #[cfg(not(unix))] + fn stop_legacy_server(&self) -> anyhow::Result<()> { + anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported) + } +} + +#[cfg(unix)] +fn terminate_legacy_server(pid: u32) -> anyhow::Result<()> { + let current_pid = libc::pid_t::try_from(std::process::id()).ok(); + let pid = libc::pid_t::try_from(pid) + .ok() + .filter(|pid| *pid > 1 && Some(*pid) != current_pid) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) + })?; + if unsafe { libc::kill(pid, libc::SIGTERM) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + Ok(()) +} + +pub(crate) fn validate_local_identity(data: &Value, path: &Path) -> anyhow::Result<()> { + let probe = ServerProbe { identity: ServerIdentity::from_protocol_data(data)? }; + probe.require_compatible(path) +} + +pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path) -> String { + let messages = &crate::localization::catalog().server; + let client = ReleaseIdentity::current(PROTOCOL_VERSION); + let (server_version, client_version) = if identity.release.version == client.version { + (identity.release.version_with_build_metadata(), client.version_with_build_metadata()) + } else { + (identity.release.version.clone(), client.version.clone()) + }; + let command = format!("cmux-tui server stop --socket {}", shell_quote(path)); + let restart = messages.restart_instruction.replace("{command}", &command); + format!( + "{}\n\n{}: v{} {} {}\n{}: v{} {} {}\n\n{}\n{}\n{}", + messages.incompatible_local_server, + messages.server_label, + server_version, + messages.protocol_label, + identity.release.protocol, + messages.client_label, + client_version, + messages.protocol_label, + client.protocol, + messages.stop_to_use, + messages.stopping_exits_panes, + restart, + ) +} + +pub(crate) fn write_json_line(writer: &mut dyn Write, value: &Value) -> std::io::Result<()> { + serde_json::to_writer(&mut *writer, value).map_err(std::io::Error::other)?; + writer.write_all(b"\n") +} + +fn read_response( + reader: &mut BufReader>, + request_id: u64, +) -> anyhow::Result { + read_matching_response(reader, request_id, false) +} + +fn read_shutdown_response( + reader: &mut BufReader>, + request_id: u64, +) -> anyhow::Result { + read_matching_response(reader, request_id, true) +} + +fn read_matching_response( + reader: &mut BufReader>, + request_id: u64, + accept_unidentified_error: bool, +) -> anyhow::Result { + let deadline = Instant::now() + RESPONSE_TIMEOUT; + let mut line = String::new(); + loop { + match reader.read_line(&mut line) { + Ok(0) => anyhow::bail!("transport closed before response"), + Ok(_) => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) && Instant::now() < deadline => + { + continue; + } + Err(error) => return Err(error.into()), + } + let value: Value = serde_json::from_str(&line) + .map_err(|error| anyhow::anyhow!("bad response: {error}"))?; + line.clear(); + if value.get("event").is_some() { + continue; + } + let response_id = value.get("id").and_then(Value::as_u64); + if response_id == Some(request_id) + || (accept_unidentified_error + && response_id.is_none() + && value.get("ok").and_then(Value::as_bool) == Some(false)) + { + return Ok(value); + } + } +} + +fn wait_for_disconnect(reader: &mut BufReader>) -> anyhow::Result<()> { + let deadline = Instant::now() + RESPONSE_TIMEOUT; + let mut line = String::new(); + loop { + match reader.read_line(&mut line) { + Ok(0) => return Ok(()), + Ok(_) => line.clear(), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) && Instant::now() < deadline => {} + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out) + } + Err(error) => return Err(error.into()), + } + } +} + +fn connection_is_gone(path: &Path) -> bool { + transport::connect(path).is_err() +} + +fn shell_quote(path: &Path) -> String { + let value = path.display().to_string(); + if value.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '/' | '.' | '_' | '-') + }) { + value + } else { + format!("'{}'", value.replace('\'', "'\"'\"'")) + } +} + +#[cfg(test)] +mod tests { + #[cfg(unix)] + use std::process::{Command, Stdio}; + + use super::*; + + #[test] + fn mismatch_message_contains_both_release_identities_and_stop_command() { + let identity = ServerIdentity { + release: ReleaseIdentity { + version: "0.1.0-old".to_string(), + build_commit: None, + ghostty_commit: None, + protocol: PROTOCOL_VERSION - 1, + }, + session: "test".to_string(), + pid: 42, + capabilities: HashSet::new(), + }; + + let message = incompatible_server_message(&identity, Path::new("/tmp/test socket")); + + assert!(message.contains("server: v0.1.0-old protocol 8")); + assert!(message.contains("client: v")); + assert!(message.contains("cmux-tui server stop --socket '/tmp/test socket'")); + assert!(message.contains("Stopping exits pane processes.")); + } + + #[cfg(unix)] + #[test] + fn legacy_stop_terminates_only_the_identified_process() { + let mut child = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + + terminate_legacy_server(child.id()).unwrap(); + + assert!(!child.wait().unwrap().success()); + } +} diff --git a/cmux-tui/crates/cmux-tui/src/session/mod.rs b/cmux-tui/crates/cmux-tui/src/session/mod.rs index 2e5c7c21002e..a5ea4c3f349c 100644 --- a/cmux-tui/crates/cmux-tui/src/session/mod.rs +++ b/cmux-tui/crates/cmux-tui/src/session/mod.rs @@ -210,6 +210,14 @@ impl Session { remote.begin_shutdown(); } } + + pub fn shutdown_requested(&self) -> bool { + match self { + Session::Local(mux) => mux.shutdown_requested(), + Session::Remote(_) => false, + } + } + pub fn invalidate_remote_tree(&self) { if let Session::Remote(remote) = self { remote.invalidate_tree(); diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index 0b136595a5e3..c38f213f83f1 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -15,7 +15,7 @@ use base64::Engine; use cmux_tui_core::{ BrowserFrame, BrowserSource, BrowserStatus, DefaultColors, MuxEvent, MuxEventBroadcaster, MuxEventReceiver, NotificationEvent, NotificationLevel, PairingChallenge, Rgb, SurfaceId, - SurfaceKind, platform::transport, + SurfaceKind, platform::transport, server::PROTOCOL_VERSION, }; use cmux_tui_machine_protocol::BearerToken; use ghostty_vt::{Callbacks, MouseEncoders, MouseInput, RenderState, Terminal}; @@ -24,7 +24,7 @@ use zeroize::Zeroize; use super::tree::{TreeView, parse_tree}; -const SUPPORTED_PROTOCOL_VERSION: u64 = 9; +const SUPPORTED_PROTOCOL_VERSION: u64 = PROTOCOL_VERSION as u64; const SURFACE_OVERFLOW_RETRY_DELAYS: [Duration; 3] = [Duration::from_millis(250), Duration::from_millis(500), Duration::from_secs(1)]; const SURFACE_OVERFLOW_STABLE: Duration = Duration::from_secs(5); @@ -496,7 +496,10 @@ impl RemoteSession { let stream = transport::connect(path).map_err(|e| { anyhow::anyhow!("cannot connect to session socket {}: {e}", path.display()) })?; - Self::connect_stream(stream) + let transport = RemoteTransport::json_lines(stream).map_err(|error| { + anyhow::anyhow!("cannot configure JSON-lines session transport: {error}") + })?; + Self::connect_transport_with_provider_authority(transport, None, Some(path)) } /// Connect over an already-established full-duplex byte stream. @@ -505,6 +508,7 @@ impl RemoteSession { /// establishment outside `RemoteSession` lets clients use a local socket, /// an SSH relay, or another authenticated tunnel without teaching the /// session and rendering layers about those transports. + #[cfg_attr(not(test), allow(dead_code))] pub fn connect_stream(stream: Box) -> anyhow::Result> { let transport = RemoteTransport::json_lines(stream).map_err(|error| { anyhow::anyhow!("cannot configure JSON-lines session transport: {error}") @@ -513,19 +517,20 @@ impl RemoteSession { } pub fn connect_transport(transport: RemoteTransport) -> anyhow::Result> { - Self::connect_transport_with_provider_authority(transport, None) + Self::connect_transport_with_provider_authority(transport, None, None) } pub fn connect_provider_transport( transport: RemoteTransport, authority: BearerToken, ) -> anyhow::Result> { - Self::connect_transport_with_provider_authority(transport, Some(authority)) + Self::connect_transport_with_provider_authority(transport, Some(authority), None) } fn connect_transport_with_provider_authority( transport: RemoteTransport, provider_workspace_authority: Option, + local_path: Option<&Path>, ) -> anyhow::Result> { let RemoteTransport { mut reader, writer } = transport; let session = Arc::new(RemoteSession { @@ -561,17 +566,21 @@ impl RemoteSession { } })?; - if let Err(error) = session.initialize() { + if let Err(error) = session.initialize(local_path) { session.disconnect_transport(); return Err(error); } Ok(session) } - fn initialize(&self) -> anyhow::Result<()> { + fn initialize(&self, local_path: Option<&Path>) -> anyhow::Result<()> { // Identify (validates the endpoint) and subscribe to events. let ident = self.request(json!({"cmd": "identify"}))?; - validate_remote_identity(&ident)?; + if let Some(path) = local_path { + crate::server_lifecycle::validate_local_identity(&ident, path)?; + } else { + validate_remote_identity(&ident)?; + } *self.capabilities.lock().unwrap() = ident .get("capabilities") .and_then(Value::as_array) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index c08da4d949f4..0d0bd6c06aec 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -244,6 +244,154 @@ fn plain_launch_explains_how_to_replace_an_incompatible_local_server() { assert!(!stderr.contains("already in use"), "{stderr}"); } +#[test] +fn incompatible_server_allows_identity_and_status_but_rejects_ping() { + let dir = unique_temp_dir("incompatible-cli"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = transport::listen(&socket).unwrap(); + let server = std::thread::spawn(move || { + let mut commands = Vec::new(); + for _ in 0..3 { + let mut stream = listener.accept().unwrap(); + let mut request = String::new(); + BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); + let request: serde_json::Value = serde_json::from_str(&request).unwrap(); + commands.push(request["cmd"].as_str().unwrap().to_string()); + let response = serde_json::json!({ + "id": request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": 4242, + "version": "0.0.0-stale", + "protocol": cmux_tui_core::server::PROTOCOL_VERSION, + }, + }); + writeln!(stream, "{response}").unwrap(); + } + commands + }); + + let status = Command::new(bin()) + .args(["server", "status", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_success(&status); + assert!(String::from_utf8_lossy(&status.stdout).contains("status: incompatible")); + + let identify = Command::new(bin()) + .args(["identify", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_success(&identify); + assert!(String::from_utf8_lossy(&identify.stdout).contains("version=0.0.0-stale")); + + let ping = Command::new(bin()) + .args(["ping", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_eq!(ping.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&ping.stderr).contains("cmux-tui server stop")); + + assert_eq!(server.join().unwrap(), ["identify", "identify", "identify"]); + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); +} + +#[test] +fn server_status_and_stop_control_a_compatible_headless_session() { + let mut server = HeadlessServer::start("server-lifecycle"); + let status = cli(&server, &["--json", "server", "status"]); + assert_success(&status); + let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); + assert_eq!(status["running"].as_bool(), Some(true)); + assert_eq!(status["compatible"].as_bool(), Some(true)); + assert_eq!( + status["server"]["version"].as_str(), + Some(cmux_tui_core::release::distribution_version()) + ); + + let stop = cli(&server, &["server", "stop"]); + assert_success(&stop); + assert!(String::from_utf8_lossy(&stop.stdout).contains("Stopped the cmux-tui server.")); + assert!(server.child.wait().unwrap().success()); + assert!(!server.socket.exists()); +} + +#[cfg(unix)] +#[test] +fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { + let dir = unique_temp_dir("legacy-server-stop"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = transport::listen(&socket).unwrap(); + let mut pane_process = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let pane_pid = pane_process.id(); + let server = std::thread::spawn(move || { + let mut stream = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); + let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(identify_request["cmd"].as_str(), Some("identify")); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": identify_request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": pane_pid, + "version": "0.0.0-stale", + "protocol": 8, + }, + }) + ) + .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); + assert_eq!( + serde_json::from_str::(&request).unwrap()["cmd"].as_str(), + Some("shutdown") + ); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": serde_json::Value::Null, + "ok": false, + "error": "bad request", + }) + ) + .unwrap(); + }); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + assert!(!pane_process.wait().unwrap().success()); + server.join().unwrap(); + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); +} + #[cfg(unix)] #[test] fn configured_websocket_server_does_not_attach_to_existing_session() { @@ -273,7 +421,7 @@ fn cli_verbs_cover_command_output_errors_and_streams() { let identify = cli(&server, &["identify"]); assert_success(&identify); - assert!(String::from_utf8_lossy(&identify.stdout).starts_with("cmux-tui session=")); + assert!(String::from_utf8_lossy(&identify.stdout).starts_with("cmux-tui version=")); let identify_json = cli(&server, &["--json", "identify"]); assert_success(&identify_json); @@ -562,12 +710,37 @@ fn stream_preserves_partial_line_across_read_timeout() { let listener = transport::listen(&socket).unwrap(); let server = std::thread::spawn(move || { let mut stream = listener.accept().unwrap(); + let read_half = stream.try_clone_box().unwrap(); + let mut reader = BufReader::new(read_half); let mut request = String::new(); - { - let read_half = stream.try_clone_box().unwrap(); - let mut reader = BufReader::new(read_half); - reader.read_line(&mut request).unwrap(); - } + reader.read_line(&mut request).unwrap(); + let request_json: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(request_json["cmd"].as_str(), Some("identify")); + let release = cmux_tui_core::release::ReleaseIdentity::current( + cmux_tui_core::server::PROTOCOL_VERSION, + ); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": request_json["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "version": release.version, + "build_commit": release.build_commit, + "ghostty_commit": release.ghostty_commit, + "protocol": release.protocol, + "capabilities": [], + "session": "test", + "pid": std::process::id(), + }, + }) + ) + .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); assert!(request.contains("\"cmd\":\"subscribe\"")); stream.write_all(br#"{"event":"status","message":""#).unwrap(); diff --git a/cmux-tui/dist/RELEASING-TUI.md b/cmux-tui/dist/RELEASING-TUI.md index 5199c0fd7764..42641be0f57d 100644 --- a/cmux-tui/dist/RELEASING-TUI.md +++ b/cmux-tui/dist/RELEASING-TUI.md @@ -8,9 +8,11 @@ TUI distribution versions are independent of the SDK version. The SDK package relocation to `cmux-sdk` is tracked separately and is not part of this release path. -The TUI does not store its version in a checked-in manifest. The packaging -scripts receive `--version`, so cutting a stable TUI release is just creating a -`cmux-tui-vX.Y.Z` tag on `main`. +The TUI does not store its distribution version in a checked-in manifest. The +packaging workflow stamps its `version` input into every binary so local +clients can distinguish releases even though the Cargo crate version remains +fixed. Cutting a stable TUI release is just creating a `cmux-tui-vX.Y.Z` tag on +`main`. ## Packages diff --git a/cmux-tui/docs/getting-started.md b/cmux-tui/docs/getting-started.md index fd7ca580df2e..31e5f5697a8e 100644 --- a/cmux-tui/docs/getting-started.md +++ b/cmux-tui/docs/getting-started.md @@ -41,6 +41,15 @@ cargo run -p cmux-tui -- attach --session agents Detach from an attached TUI with prefix `d`. With default keys, that is `Ctrl-b d`. The server keeps running, and another `attach` reconnects to the same tree. PTY tabs attach with a Ghostty VT-state replay followed by a live output stream. +Local clients require the running server to have the same release identity. Inspect a session without attaching, or stop it before starting an upgraded binary: + +```bash +cmux-tui server status --session agents +cmux-tui server stop --session agents +``` + +Stopping the server exits its pane processes. `identify`, `server status`, and `server stop` remain available when client and server releases differ. + ## Remote machines The optional machine rail keeps rendering local while it connects individual session transports through Unix sockets or SSH. It is disabled for the default local run and activates when `machine_sidebar.enabled` is true or `machines` contains a valid entry in `cmux-tui.json`. Start a headless cmux session on each remote machine, and make the remote `cmux-tui` or `cmux` executable available to noninteractive SSH. The SSH connector runs its `relay` mode and does not nest a second TUI. diff --git a/cmux-tui/docs/protocol.md b/cmux-tui/docs/protocol.md index f253d90511a2..f328aa213a2e 100644 --- a/cmux-tui/docs/protocol.md +++ b/cmux-tui/docs/protocol.md @@ -36,6 +36,8 @@ The server command set in this branch is: ```text identify +ping +shutdown list-workspaces send read-screen @@ -166,7 +168,9 @@ When the stream ends, it sends: ## Client Compatibility -The remote TUI requires protocol v9. It rejects protocol-v8 servers before loading their workspace tree because v8 does not define stack layout nodes or `new-pane`. +Direct local clients require the server's distribution version, build commit, Ghostty commit, and protocol to match exactly. `identify`, `cmux-tui server status`, and `cmux-tui server stop` bypass that check so an upgraded client can inspect and stop an older server. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. + +Remote transports require protocol v9 but do not require the same release build. They reject protocol-v8 servers before loading their workspace tree because v8 does not define stack layout nodes or `new-pane`. Existing `set-ratio` clients remain source-compatible and the server keeps the pane-and-direction command unchanged. Protocol-v8 and newer frontends should read `layout.split` and send `set-split-ratio` so nested same-direction dividers are addressed exactly. Protocol v9 adds stack layout nodes and `new-pane`; clients must not send `new-pane` to a protocol-v8 server. diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 5c134ef35313..518cbdf701de 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -172,7 +172,7 @@ CLI mapping: | --- | --- | | Verb | `identify` | | Flags | none | -| Plain stdout | `cmux-tui session= protocol= pid=` | +| Plain stdout | `cmux-tui version= session= protocol= pid=` | | JSON stdout | exact result object | | Exit codes | common | @@ -216,6 +216,36 @@ Example: {"id":2,"ok":true,"data":{"ok":true,"version":"0.1.0","build_commit":"abc123","ghostty_commit":"def456","protocol":9}} ``` +### shutdown + +| Field | Value | +| --- | --- | +| name | `shutdown` | +| status | implemented | +| since | protocol 9 additive extension | + +Requests an orderly server exit after the success response is queued. The command is accepted only on the local Unix-domain socket. Stopping the mux exits every pane process. + +Params: none. + +Result: `object{}`. + +Errors: + +| Error | Condition | +| --- | --- | +| `shutdown is only available over the local session socket` | Request arrived over WebSocket | +| `bad request: ...` | Malformed request envelope | + +CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. A newer Unix client falls back to signaling the PID reported by `identify` when an older server does not implement `shutdown`. + +Example: + +```json +{"id":3,"cmd":"shutdown"} +{"id":3,"ok":true,"data":{}} +``` + ### set-client-info | Field | Value | From 5f56b49577aaa7c15490cb6093b708389115e93d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:41:04 -0700 Subject: [PATCH 003/521] fix(tui): drain panes before legacy shutdown --- cmux-tui/crates/cmux-tui/src/localization.rs | 6 + .../crates/cmux-tui/src/server_lifecycle.rs | 87 ++++++++++++- cmux-tui/crates/cmux-tui/tests/cli.rs | 123 ++++++++++++++++++ cmux-tui/spec/commands.md | 2 +- 4 files changed, 214 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index aef978db6856..569ffafea627 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -35,6 +35,8 @@ pub(crate) struct ServerMessages { pub stopped: &'static str, pub missing_action: &'static str, pub unknown_action: &'static str, + #[cfg(unix)] + pub legacy_cleanup_failed: &'static str, pub shutdown_unsupported: &'static str, pub shutdown_timed_out: &'static str, } @@ -180,6 +182,8 @@ static ENGLISH: Catalog = Catalog { stopped: "Stopped the cmux-tui server.", missing_action: "server needs an action: status or stop", unknown_action: "unknown server action", + #[cfg(unix)] + legacy_cleanup_failed: "could not close pane processes before stopping the older server", shutdown_unsupported: "this server cannot be stopped by this client", shutdown_timed_out: "the server did not stop within 10 seconds", }, @@ -278,6 +282,8 @@ static JAPANESE: Catalog = Catalog { stopped: "cmux-tui サーバーを停止しました。", missing_action: "server には status または stop の操作が必要です", unknown_action: "不明な server 操作", + #[cfg(unix)] + legacy_cleanup_failed: "古いサーバーを停止する前にペインのプロセスを終了できませんでした", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", }, diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 30578d29ff16..964223991c21 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -11,6 +11,10 @@ use serde_json::{Value, json}; const PROBE_REQUEST_ID: u64 = 0; const SHUTDOWN_REQUEST_ID: u64 = 1; +#[cfg(unix)] +const LEGACY_LIST_REQUEST_ID: u64 = 2; +#[cfg(unix)] +const LEGACY_CLOSE_REQUEST_ID_START: u64 = 3; const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); #[derive(Clone, Debug)] @@ -141,20 +145,55 @@ impl ServerLifecycle { Err(error) => return Err(error), }; if !accepted { - self.stop_legacy_server().context("failed to signal the older server")?; + self.stop_legacy_server()?; } wait_for_disconnect(&mut self.reader).context("failed while waiting for the server to stop") } #[cfg(unix)] - fn stop_legacy_server(&self) -> anyhow::Result<()> { + fn stop_legacy_server(&mut self) -> anyhow::Result<()> { + self.close_legacy_surfaces().with_context(|| { + crate::localization::catalog().server.legacy_cleanup_failed.to_string() + })?; terminate_legacy_server(self.probe.identity.pid) + .context("failed to signal the older server") } #[cfg(not(unix))] - fn stop_legacy_server(&self) -> anyhow::Result<()> { + fn stop_legacy_server(&mut self) -> anyhow::Result<()> { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported) } + + #[cfg(unix)] + fn close_legacy_surfaces(&mut self) -> anyhow::Result<()> { + write_json_line( + self.reader.get_mut(), + &json!({"id": LEGACY_LIST_REQUEST_ID, "cmd": "list-workspaces"}), + ) + .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + let response = read_response(&mut self.reader, LEGACY_LIST_REQUEST_ID)?; + let data = response_data(&response, "list-workspaces")?; + + for (index, surface) in legacy_surface_ids(data).into_iter().enumerate() { + let request_id = LEGACY_CLOSE_REQUEST_ID_START + .checked_add(u64::try_from(index)?) + .ok_or_else(|| anyhow::anyhow!("too many surfaces to close"))?; + write_json_line( + self.reader.get_mut(), + &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), + ) + .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + let response = read_response(&mut self.reader, request_id)?; + if response.get("ok").and_then(Value::as_bool) != Some(true) { + let error = + response.get("error").and_then(Value::as_str).unwrap_or("close-surface failed"); + if !error.starts_with("unknown surface ") { + anyhow::bail!("{error}"); + } + } + } + Ok(()) + } } #[cfg(unix)] @@ -261,6 +300,31 @@ fn read_matching_response( } } +#[cfg(unix)] +fn response_data<'a>(response: &'a Value, command: &str) -> anyhow::Result<&'a Value> { + if response.get("ok").and_then(Value::as_bool) != Some(true) { + anyhow::bail!("{}", response.get("error").and_then(Value::as_str).unwrap_or(command)); + } + Ok(response.get("data").unwrap_or(&Value::Null)) +} + +#[cfg(unix)] +fn legacy_surface_ids(data: &Value) -> Vec { + let mut surfaces = Vec::new(); + for workspace in data.get("workspaces").and_then(Value::as_array).into_iter().flatten() { + for screen in workspace.get("screens").and_then(Value::as_array).into_iter().flatten() { + for pane in screen.get("panes").and_then(Value::as_array).into_iter().flatten() { + for tab in pane.get("tabs").and_then(Value::as_array).into_iter().flatten() { + if let Some(surface) = tab.get("surface").and_then(Value::as_u64) { + surfaces.push(surface); + } + } + } + } + } + surfaces +} + fn wait_for_disconnect(reader: &mut BufReader>) -> anyhow::Result<()> { let deadline = Instant::now() + RESPONSE_TIMEOUT; let mut line = String::new(); @@ -340,4 +404,21 @@ mod tests { assert!(!child.wait().unwrap().success()); } + + #[cfg(unix)] + #[test] + fn legacy_surface_ids_follow_the_workspace_tree() { + let data = json!({ + "workspaces": [{ + "screens": [{ + "panes": [ + {"tabs": [{"surface": 11}, {"surface": 12}]}, + {"tabs": [{"surface": 13}]}, + ], + }], + }], + }); + + assert_eq!(legacy_surface_ids(&data), [11, 12, 13]); + } } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 0d0bd6c06aec..b2daec050761 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -376,6 +376,47 @@ fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { }) ) .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); + let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": true, + "data": { + "workspaces": [{ + "screens": [{ + "panes": [{ + "tabs": [{"surface": 41}, {"surface": 42}], + }], + }], + }], + }, + }) + ) + .unwrap(); + + for expected_surface in [41, 42] { + request.clear(); + reader.read_line(&mut request).unwrap(); + let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); + assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": close_request["id"], + "ok": true, + "data": {}, + }) + ) + .unwrap(); + } }); let output = Command::new(bin()) @@ -392,6 +433,88 @@ fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { let _ = fs::remove_dir_all(&dir); } +#[cfg(unix)] +#[test] +fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { + let dir = unique_temp_dir("legacy-server-cleanup-failure"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = transport::listen(&socket).unwrap(); + let mut server_process = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let server_pid = server_process.id(); + let server = std::thread::spawn(move || { + let mut stream = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); + let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": identify_request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": server_pid, + "version": "0.0.0-stale", + "protocol": 8, + }, + }) + ) + .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": serde_json::Value::Null, + "ok": false, + "error": "bad request", + }) + ) + .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); + let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": false, + "error": "cleanup unavailable", + }) + ) + .unwrap(); + }); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(server_process.try_wait().unwrap().is_none()); + server_process.kill().unwrap(); + server_process.wait().unwrap(); + server.join().unwrap(); + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); +} + #[cfg(unix)] #[test] fn configured_websocket_server_does_not_attach_to_existing_session() { diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 518cbdf701de..1a2058aba9c2 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -237,7 +237,7 @@ Errors: | `shutdown is only available over the local session socket` | Request arrived over WebSocket | | `bad request: ...` | Malformed request envelope | -CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. A newer Unix client falls back to signaling the PID reported by `identify` when an older server does not implement `shutdown`. +CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client closes every surface reported by `list-workspaces` before signaling the PID reported by `identify`. If orderly pane cleanup fails, the client leaves the server running and reports the failure. Example: From 9ef5a64b58c336bcabb98e6920cddf597e447b94 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 02:00:14 -0700 Subject: [PATCH 004/521] fix(tui): harden server replacement --- cmux-tui/crates/cmux-tui-core/build.rs | 133 ++++++ cmux-tui/crates/cmux-tui-core/src/platform.rs | 79 ++++ cmux-tui/crates/cmux-tui-core/src/release.rs | 7 +- cmux-tui/crates/cmux-tui/src/cli.rs | 20 +- cmux-tui/crates/cmux-tui/src/localization.rs | 66 +++ .../crates/cmux-tui/src/server_lifecycle.rs | 181 +++++--- cmux-tui/crates/cmux-tui/tests/cli.rs | 390 ++++++++++-------- cmux-tui/dist/RELEASING-TUI.md | 5 + cmux-tui/docs/protocol.md | 4 +- cmux-tui/spec/commands.md | 18 +- 10 files changed, 684 insertions(+), 219 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/build.rs diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs new file mode 100644 index 000000000000..de434ba7407d --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -0,0 +1,133 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +const FNV_OFFSET: u64 = 0xcbf29ce484222325; +const FNV_PRIME: u64 = 0x100000001b3; + +fn main() { + for name in [ + "CMUX_TUI_BUILD_COMMIT", + "CMUX_MUX_BUILD_COMMIT", + "CMUX_TUI_GHOSTTY_COMMIT", + "CMUX_GHOSTTY_SRC", + ] { + println!("cargo:rerun-if-env-changed={name}"); + } + + let manifest_dir = + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("Cargo sets CARGO_MANIFEST_DIR")); + let workspace_root = + manifest_dir.ancestors().nth(2).expect("cmux-tui-core is inside the workspace"); + let repository_root = + workspace_root.parent().expect("the cmux-tui workspace is inside the repository"); + let ghostty_root = env::var_os("CMUX_GHOSTTY_SRC") + .map(PathBuf::from) + .unwrap_or_else(|| repository_root.join("ghostty")); + + println!("cargo:rerun-if-changed={}", workspace_root.join("Cargo.toml").display()); + println!("cargo:rerun-if-changed={}", workspace_root.join("Cargo.lock").display()); + println!("cargo:rerun-if-changed={}", workspace_root.join("crates").display()); + println!("cargo:rerun-if-changed={}", ghostty_root.join("build.zig").display()); + println!("cargo:rerun-if-changed={}", ghostty_root.join("include").display()); + println!("cargo:rerun-if-changed={}", ghostty_root.join("src").display()); + track_git_identity(repository_root); + track_git_identity(&ghostty_root); + + if env::var_os("CMUX_TUI_BUILD_COMMIT").is_none() + && env::var_os("CMUX_MUX_BUILD_COMMIT").is_none() + && let Some(identity) = source_identity(workspace_root, workspace_root) + { + println!("cargo:rustc-env=CMUX_TUI_SOURCE_COMMIT={identity}"); + } + if env::var_os("CMUX_TUI_GHOSTTY_COMMIT").is_none() + && let Some(identity) = source_identity(&ghostty_root, &ghostty_root) + { + println!("cargo:rustc-env=CMUX_TUI_SOURCE_GHOSTTY_COMMIT={identity}"); + } +} + +fn source_identity(git_root: &Path, fallback_root: &Path) -> Option { + if let Some(commit) = git_text(git_root, &["rev-parse", "HEAD"]) { + let status = + git_bytes(git_root, &["status", "--porcelain=v1", "--untracked-files=all", "--", "."])?; + if status.is_empty() { + return Some(commit); + } + return Some(format!("{commit}-dirty-{:016x}", dirty_fingerprint(git_root)?)); + } + directory_fingerprint(fallback_root) + .ok() + .map(|fingerprint| format!("source-{fingerprint:016x}")) +} + +fn dirty_fingerprint(root: &Path) -> Option { + let mut hash = FNV_OFFSET; + hash_bytes(&mut hash, &git_bytes(root, &["diff", "--binary", "HEAD", "--", "."])?); + let untracked = + git_bytes(root, &["ls-files", "-z", "--others", "--exclude-standard", "--", "."])?; + for path in untracked.split(|byte| *byte == 0).filter(|path| !path.is_empty()) { + hash_bytes(&mut hash, path); + let path = std::str::from_utf8(path).ok()?; + hash_bytes(&mut hash, &fs::read(root.join(path)).ok()?); + } + Some(hash) +} + +fn directory_fingerprint(root: &Path) -> std::io::Result { + fn visit(root: &Path, path: &Path, hash: &mut u64) -> std::io::Result<()> { + let mut entries = fs::read_dir(path)?.collect::, _>>()?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let name = entry.file_name(); + if matches!(name.to_str(), Some(".git" | "target" | "zig-cache" | "zig-out")) { + continue; + } + let path = entry.path(); + if path.is_dir() { + visit(root, &path, hash)?; + } else if path.is_file() { + hash_bytes( + hash, + path.strip_prefix(root).unwrap_or(&path).as_os_str().as_encoded_bytes(), + ); + hash_bytes(hash, &fs::read(path)?); + } + } + Ok(()) + } + + let mut hash = FNV_OFFSET; + visit(root, root, &mut hash)?; + Ok(hash) +} + +fn hash_bytes(hash: &mut u64, bytes: &[u8]) { + for byte in bytes { + *hash ^= u64::from(*byte); + *hash = hash.wrapping_mul(FNV_PRIME); + } +} + +fn track_git_identity(root: &Path) { + let Some(head) = git_text(root, &["rev-parse", "--git-path", "HEAD"]) else { return }; + println!("cargo:rerun-if-changed={head}"); + if let Some(reference) = git_text(root, &["symbolic-ref", "-q", "HEAD"]) + && let Some(reference_path) = + git_text(root, &["rev-parse", "--git-path", reference.as_str()]) + { + println!("cargo:rerun-if-changed={reference_path}"); + } +} + +fn git_text(root: &Path, args: &[&str]) -> Option { + let output = git_bytes(root, args)?; + let value = std::str::from_utf8(&output).ok()?.trim(); + (!value.is_empty()).then(|| value.to_string()) +} + +fn git_bytes(root: &Path, args: &[&str]) -> Option> { + let output = Command::new("git").arg("-C").arg(root).args(args).output().ok()?; + output.status.success().then_some(output.stdout) +} diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 4fa1af761bee..32d183590e12 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -13,6 +13,9 @@ pub mod transport { fn set_read_timeout(&self, timeout: Option) -> io::Result<()>; fn set_write_timeout(&self, timeout: Option) -> io::Result<()>; fn shutdown(&self, how: Shutdown) -> io::Result<()>; + fn peer_process_id(&self) -> io::Result> { + Ok(None) + } } pub struct Listener { @@ -77,6 +80,69 @@ pub mod transport { fn shutdown(&self, how: std::net::Shutdown) -> io::Result<()> { UnixStream::shutdown(self, how) } + + fn peer_process_id(&self) -> io::Result> { + peer_process_id(self) + } + } + + #[cfg(target_os = "macos")] + fn peer_process_id(stream: &UnixStream) -> io::Result> { + use std::mem::size_of; + use std::os::fd::AsRawFd; + + let mut pid: libc::pid_t = 0; + let mut length = size_of::() as libc::socklen_t; + let result = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_LOCAL, + libc::LOCAL_PEERPID, + (&raw mut pid).cast(), + &raw mut length, + ) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + if length as usize != size_of::() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid peer process id")); + } + u32::try_from(pid) + .map(Some) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid peer process id")) + } + + #[cfg(target_os = "linux")] + fn peer_process_id(stream: &UnixStream) -> io::Result> { + use std::mem::{size_of, zeroed}; + use std::os::fd::AsRawFd; + + let mut credentials = unsafe { zeroed::() }; + let mut length = size_of::() as libc::socklen_t; + let result = unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_PEERCRED, + (&raw mut credentials).cast(), + &raw mut length, + ) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + if length as usize != size_of::() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid peer credentials")); + } + u32::try_from(credentials.pid) + .map(Some) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid peer process id")) + } + + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + fn peer_process_id(_stream: &UnixStream) -> io::Result> { + Ok(None) } } @@ -724,6 +790,19 @@ fn restrict_permissions(_path: &Path, _mode: u32) -> std::io::Result<()> { mod tests { use super::*; + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn unix_transport_reports_the_kernel_peer_process() { + use std::os::unix::net::UnixStream; + + use transport::Stream as _; + + let (client, server) = UnixStream::pair().unwrap(); + + assert_eq!(client.peer_process_id().unwrap(), Some(std::process::id())); + assert_eq!(server.peer_process_id().unwrap(), Some(std::process::id())); + } + fn position(candidates: &[GhosttyInstallation], expected: impl AsRef) -> usize { let expected = expected.as_ref(); candidates diff --git a/cmux-tui/crates/cmux-tui-core/src/release.rs b/cmux-tui/crates/cmux-tui-core/src/release.rs index ccd5c9eca88d..2b201b77af1e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/release.rs +++ b/cmux-tui/crates/cmux-tui-core/src/release.rs @@ -61,11 +61,14 @@ pub fn distribution_version() -> &'static str { pub fn stamped_build_commit() -> Option<&'static str> { option_env!("CMUX_TUI_BUILD_COMMIT") .or(option_env!("CMUX_MUX_BUILD_COMMIT")) + .or(option_env!("CMUX_TUI_SOURCE_COMMIT")) .filter(|commit| !commit.is_empty()) } pub fn stamped_ghostty_commit() -> Option<&'static str> { - option_env!("CMUX_TUI_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()) + option_env!("CMUX_TUI_GHOSTTY_COMMIT") + .or(option_env!("CMUX_TUI_SOURCE_GHOSTTY_COMMIT")) + .filter(|commit| !commit.is_empty()) } fn optional_string(data: &Value, key: &str) -> Option { @@ -94,6 +97,8 @@ mod tests { #[test] fn exact_matching_includes_source_identities() { let current = ReleaseIdentity::current(9); + assert!(current.build_commit.is_some()); + assert!(current.ghostty_commit.is_some()); let mut other = current.clone(); assert!(current.exactly_matches(&other)); diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index db8d6416b589..65d952853795 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -698,7 +698,11 @@ fn run_server_lifecycle(global: &GlobalArgs, flags: &FlagMap) -> i32 { "status" => print_server_status(global.json, lifecycle.probe()), "stop" => match lifecycle.stop() { Ok(()) => { - println!("{}", messages.stopped); + if global.json { + println!("{}", json!({"stopped": true})); + } else { + println!("{}", messages.stopped); + } 0 } Err(error) => { @@ -714,11 +718,15 @@ fn print_server_status(json_output: bool, probe: &crate::server_lifecycle::Serve let messages = &crate::localization::catalog().server; let server = &probe.identity; let client = ReleaseIdentity::current(PROTOCOL_VERSION); - let compatible = probe.is_compatible(); + let mismatches = probe.mismatches(); + let compatible = mismatches.is_empty(); if json_output { + let mismatch_reasons = + mismatches.iter().map(|mismatch| mismatch.code()).collect::>(); let value = json!({ "running": true, "compatible": compatible, + "mismatch_reasons": mismatch_reasons, "server": { "version": server.release.version, "protocol": server.release.protocol, @@ -756,6 +764,14 @@ fn print_server_status(json_output: bool, probe: &crate::server_lifecycle::Serve messages.status_label, if compatible { messages.compatible } else { messages.incompatible }, ); + if !mismatches.is_empty() { + let reasons = mismatches + .into_iter() + .map(|mismatch| mismatch.message(messages)) + .collect::>() + .join(messages.reason_separator); + println!("{}: {}", messages.reason_label, reasons); + } 0 } } diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 4e141fb69784..28858b8859de 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -27,8 +27,14 @@ pub(crate) struct ServerMessages { pub client_label: &'static str, pub protocol_label: &'static str, pub status_label: &'static str, + pub reason_label: &'static str, + pub reason_separator: &'static str, pub compatible: &'static str, pub incompatible: &'static str, + pub reason_version: &'static str, + pub reason_source: &'static str, + pub reason_terminal_engine: &'static str, + pub reason_protocol: &'static str, pub stop_to_use: &'static str, pub stopping_exits_panes: &'static str, pub restart_before_command: &'static str, @@ -37,9 +43,25 @@ pub(crate) struct ServerMessages { pub missing_action: &'static str, pub unknown_action: &'static str, pub connect_failed: &'static str, + pub endpoint_invalid: &'static str, pub identity_failed: &'static str, + pub transport_failed: &'static str, + pub response_closed: &'static str, + pub response_invalid: &'static str, #[cfg(unix)] pub legacy_cleanup_failed: &'static str, + #[cfg(unix)] + pub legacy_list_failed: &'static str, + #[cfg(unix)] + pub legacy_close_failed: &'static str, + #[cfg(unix)] + pub legacy_too_many_surfaces: &'static str, + #[cfg(unix)] + pub legacy_peer_unavailable: &'static str, + #[cfg(unix)] + pub legacy_peer_mismatch: &'static str, + #[cfg(unix)] + pub legacy_signal_failed: &'static str, pub shutdown_unsupported: &'static str, pub shutdown_timed_out: &'static str, } @@ -177,8 +199,14 @@ static ENGLISH: Catalog = Catalog { client_label: "client", protocol_label: "protocol", status_label: "status", + reason_label: "reason", + reason_separator: ", ", compatible: "compatible", incompatible: "incompatible", + reason_version: "distribution version differs", + reason_source: "source build differs", + reason_terminal_engine: "terminal engine build differs", + reason_protocol: "protocol differs", stop_to_use: "Stop the old server to use this cmux-tui version.", stopping_exits_panes: "Stopping exits pane processes.", restart_before_command: "Run `", @@ -187,9 +215,25 @@ static ENGLISH: Catalog = Catalog { missing_action: "server needs an action: status or stop", unknown_action: "unknown server action", connect_failed: "Could not connect to the cmux-tui server.", + endpoint_invalid: "The socket is not a cmux-tui server.", identity_failed: "The cmux-tui server did not return its identity.", + transport_failed: "Communication with the cmux-tui server failed.", + response_closed: "The cmux-tui server closed the connection before responding.", + response_invalid: "The cmux-tui server returned an invalid response.", #[cfg(unix)] legacy_cleanup_failed: "could not close pane processes before stopping the older server", + #[cfg(unix)] + legacy_list_failed: "The older server could not list its pane processes.", + #[cfg(unix)] + legacy_close_failed: "The older server could not close a pane process.", + #[cfg(unix)] + legacy_too_many_surfaces: "The older server reported too many pane processes.", + #[cfg(unix)] + legacy_peer_unavailable: "The older server process could not be verified.", + #[cfg(unix)] + legacy_peer_mismatch: "The older server's reported process does not own its socket.", + #[cfg(unix)] + legacy_signal_failed: "The verified older server process could not be stopped.", shutdown_unsupported: "this server cannot be stopped by this client", shutdown_timed_out: "the server did not stop within 10 seconds", }, @@ -280,8 +324,14 @@ static JAPANESE: Catalog = Catalog { client_label: "クライアント", protocol_label: "プロトコル", status_label: "状態", + reason_label: "理由", + reason_separator: "、", compatible: "互換", incompatible: "非互換", + reason_version: "配布バージョンが異なります", + reason_source: "ソースビルドが異なります", + reason_terminal_engine: "ターミナルエンジンのビルドが異なります", + reason_protocol: "プロトコルが異なります", stop_to_use: "この cmux-tui バージョンを使うには、古いサーバーを停止してください。", stopping_exits_panes: "停止するとペインのプロセスが終了します。", restart_before_command: "`", @@ -290,9 +340,25 @@ static JAPANESE: Catalog = Catalog { missing_action: "server には status または stop の操作が必要です", unknown_action: "不明な server 操作", connect_failed: "cmux-tui サーバーに接続できませんでした。", + endpoint_invalid: "このソケットは cmux-tui サーバーではありません。", identity_failed: "cmux-tui サーバーから識別情報を取得できませんでした。", + transport_failed: "cmux-tui サーバーとの通信に失敗しました。", + response_closed: "cmux-tui サーバーが応答前に接続を終了しました。", + response_invalid: "cmux-tui サーバーから無効な応答が返されました。", #[cfg(unix)] legacy_cleanup_failed: "古いサーバーを停止する前にペインのプロセスを終了できませんでした", + #[cfg(unix)] + legacy_list_failed: "古いサーバーからペインのプロセス一覧を取得できませんでした。", + #[cfg(unix)] + legacy_close_failed: "古いサーバーでペインのプロセスを終了できませんでした。", + #[cfg(unix)] + legacy_too_many_surfaces: "古いサーバーから報告されたペインのプロセスが多すぎます。", + #[cfg(unix)] + legacy_peer_unavailable: "古いサーバーのプロセスを検証できませんでした。", + #[cfg(unix)] + legacy_peer_mismatch: "古いサーバーが報告したプロセスはソケットを所有していません。", + #[cfg(unix)] + legacy_signal_failed: "検証済みの古いサーバープロセスを停止できませんでした。", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", }, diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 74c04a418ffc..82fd525f8730 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -3,7 +3,6 @@ use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; -use anyhow::Context; use cmux_tui_core::platform::transport; use cmux_tui_core::release::ReleaseIdentity; use cmux_tui_core::server::PROTOCOL_VERSION; @@ -17,6 +16,53 @@ const LEGACY_LIST_REQUEST_ID: u64 = 2; const LEGACY_CLOSE_REQUEST_ID_START: u64 = 3; const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); +type TransportReader = BufReader>; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ReleaseMismatch { + DistributionVersion, + SourceBuild, + TerminalEngine, + Protocol, +} + +impl ReleaseMismatch { + pub(crate) fn code(self) -> &'static str { + match self { + Self::DistributionVersion => "distribution-version", + Self::SourceBuild => "source-build", + Self::TerminalEngine => "terminal-engine", + Self::Protocol => "protocol", + } + } + + pub(crate) fn message(self, messages: &crate::localization::ServerMessages) -> &'static str { + match self { + Self::DistributionVersion => messages.reason_version, + Self::SourceBuild => messages.reason_source, + Self::TerminalEngine => messages.reason_terminal_engine, + Self::Protocol => messages.reason_protocol, + } + } +} + +fn release_mismatches(server: &ReleaseIdentity, client: &ReleaseIdentity) -> Vec { + let mut mismatches = Vec::new(); + if server.version != client.version { + mismatches.push(ReleaseMismatch::DistributionVersion); + } + if server.build_commit != client.build_commit { + mismatches.push(ReleaseMismatch::SourceBuild); + } + if server.ghostty_commit != client.ghostty_commit { + mismatches.push(ReleaseMismatch::TerminalEngine); + } + if server.protocol != client.protocol { + mismatches.push(ReleaseMismatch::Protocol); + } + mismatches +} + #[derive(Clone, Debug)] pub(crate) struct ServerIdentity { pub release: ReleaseIdentity, @@ -27,7 +73,7 @@ pub(crate) struct ServerIdentity { impl ServerIdentity { fn from_protocol_data(data: &Value) -> anyhow::Result { if data.get("app").and_then(Value::as_str) != Some("cmux-tui") { - anyhow::bail!("socket endpoint is not a cmux-tui session"); + anyhow::bail!(crate::localization::catalog().server.endpoint_invalid); } let pid = data .get("pid") @@ -56,11 +102,9 @@ pub(crate) struct ServerProbe { } impl ServerProbe { - pub(crate) fn inspect( - reader: &mut BufReader>, - ) -> anyhow::Result { + pub(crate) fn inspect(reader: &mut TransportReader) -> anyhow::Result { write_json_line(reader.get_mut(), &json!({"id": PROBE_REQUEST_ID, "cmd": "identify"})) - .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; let response = read_response(reader, PROBE_REQUEST_ID)?; if response.get("ok").and_then(Value::as_bool) != Some(true) { anyhow::bail!(crate::localization::catalog().server.identity_failed); @@ -69,19 +113,24 @@ impl ServerProbe { Ok(Self { identity: ServerIdentity::from_protocol_data(data)? }) } - pub(crate) fn connect( - path: &Path, - ) -> anyhow::Result<(Self, BufReader>)> { + pub(crate) fn connect(path: &Path) -> anyhow::Result<(Self, TransportReader, Option)> { let stream = transport::connect(path) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.connect_failed))?; - stream.set_read_timeout(Some(RESPONSE_TIMEOUT))?; + let peer_process_id = stream.peer_process_id().ok().flatten(); + stream + .set_read_timeout(Some(RESPONSE_TIMEOUT)) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; let mut reader = BufReader::new(stream); let probe = Self::inspect(&mut reader)?; - Ok((probe, reader)) + Ok((probe, reader, peer_process_id)) } pub(crate) fn is_compatible(&self) -> bool { - self.identity.release.exactly_matches(&ReleaseIdentity::current(PROTOCOL_VERSION)) + self.mismatches().is_empty() + } + + pub(crate) fn mismatches(&self) -> Vec { + release_mismatches(&self.identity.release, &ReleaseIdentity::current(PROTOCOL_VERSION)) } pub(crate) fn require_compatible(&self, path: &Path) -> anyhow::Result<()> { @@ -109,13 +158,14 @@ impl std::error::Error for IncompatibleLocalServer {} pub(crate) struct ServerLifecycle { path: PathBuf, probe: ServerProbe, - reader: BufReader>, + reader: TransportReader, + peer_process_id: Option, } impl ServerLifecycle { pub(crate) fn connect(path: PathBuf) -> anyhow::Result { - let (probe, reader) = ServerProbe::connect(&path)?; - Ok(Self { path, probe, reader }) + let (probe, reader, peer_process_id) = ServerProbe::connect(&path)?; + Ok(Self { path, probe, reader, peer_process_id }) } pub(crate) fn probe(&self) -> &ServerProbe { @@ -127,7 +177,7 @@ impl ServerLifecycle { self.reader.get_mut(), &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), ) - .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; let accepted = match read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID) { Ok(response) => response.get("ok").and_then(Value::as_bool) == Some(true), @@ -137,16 +187,16 @@ impl ServerLifecycle { if !accepted { self.stop_legacy_server()?; } - wait_for_disconnect(&mut self.reader).context("failed while waiting for the server to stop") + wait_for_disconnect(&mut self.reader) } #[cfg(unix)] fn stop_legacy_server(&mut self) -> anyhow::Result<()> { - self.close_legacy_surfaces().with_context(|| { - crate::localization::catalog().server.legacy_cleanup_failed.to_string() + let pid = verified_legacy_pid(self.probe.identity.pid, self.peer_process_id)?; + self.close_legacy_surfaces().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - terminate_legacy_server(self.probe.identity.pid) - .context("failed to signal the older server") + terminate_legacy_server(pid) } #[cfg(not(unix))] @@ -160,25 +210,29 @@ impl ServerLifecycle { self.reader.get_mut(), &json!({"id": LEGACY_LIST_REQUEST_ID, "cmd": "list-workspaces"}), ) - .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; let response = read_response(&mut self.reader, LEGACY_LIST_REQUEST_ID)?; - let data = response_data(&response, "list-workspaces")?; + let data = response_data(&response)?; for (index, surface) in legacy_surface_ids(data).into_iter().enumerate() { let request_id = LEGACY_CLOSE_REQUEST_ID_START - .checked_add(u64::try_from(index)?) - .ok_or_else(|| anyhow::anyhow!("too many surfaces to close"))?; + .checked_add(u64::try_from(index).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_too_many_surfaces) + })?) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_too_many_surfaces) + })?; write_json_line( self.reader.get_mut(), &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), ) - .map_err(|error| anyhow::anyhow!("transport error: {error}"))?; + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; let response = read_response(&mut self.reader, request_id)?; if response.get("ok").and_then(Value::as_bool) != Some(true) { let error = response.get("error").and_then(Value::as_str).unwrap_or("close-surface failed"); if !error.starts_with("unknown surface ") { - anyhow::bail!("close-surface failed"); + anyhow::bail!(crate::localization::catalog().server.legacy_close_failed); } } } @@ -187,16 +241,27 @@ impl ServerLifecycle { } #[cfg(unix)] -fn terminate_legacy_server(pid: u32) -> anyhow::Result<()> { +fn verified_legacy_pid( + reported_pid: u32, + peer_process_id: Option, +) -> anyhow::Result { + let messages = &crate::localization::catalog().server; + let peer_process_id = + peer_process_id.ok_or_else(|| anyhow::anyhow!(messages.legacy_peer_unavailable))?; + if reported_pid != peer_process_id { + anyhow::bail!(messages.legacy_peer_mismatch); + } let current_pid = libc::pid_t::try_from(std::process::id()).ok(); - let pid = libc::pid_t::try_from(pid) + libc::pid_t::try_from(reported_pid) .ok() .filter(|pid| *pid > 1 && Some(*pid) != current_pid) - .ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) - })?; + .ok_or_else(|| anyhow::anyhow!(messages.shutdown_unsupported)) +} + +#[cfg(unix)] +fn terminate_legacy_server(pid: libc::pid_t) -> anyhow::Result<()> { if unsafe { libc::kill(pid, libc::SIGTERM) } != 0 { - return Err(std::io::Error::last_os_error().into()); + anyhow::bail!(crate::localization::catalog().server.legacy_signal_failed); } Ok(()) } @@ -209,11 +274,16 @@ pub(crate) fn validate_local_identity(data: &Value, path: &Path) -> anyhow::Resu pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path) -> String { let messages = &crate::localization::catalog().server; let client = ReleaseIdentity::current(PROTOCOL_VERSION); + let reasons = release_mismatches(&identity.release, &client) + .into_iter() + .map(|reason| reason.message(messages)) + .collect::>() + .join(messages.reason_separator); let command = format!("cmux-tui server stop --socket {}", shell_quote(path)); let restart = format!("{}{}{}", messages.restart_before_command, command, messages.restart_after_command); format!( - "{}\n\n{}: v{} {} {}\n{}: v{} {} {}\n\n{}\n{}\n{}", + "{}\n\n{}: v{} {} {}\n{}: v{} {} {}\n{}: {}\n\n{}\n{}\n{}", messages.incompatible_local_server, messages.server_label, identity.release.version, @@ -223,6 +293,8 @@ pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path client.version, messages.protocol_label, client.protocol, + messages.reason_label, + reasons, messages.stop_to_use, messages.stopping_exits_panes, restart, @@ -234,22 +306,16 @@ pub(crate) fn write_json_line(writer: &mut dyn Write, value: &Value) -> std::io: writer.write_all(b"\n") } -fn read_response( - reader: &mut BufReader>, - request_id: u64, -) -> anyhow::Result { +fn read_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { read_matching_response(reader, request_id, false) } -fn read_shutdown_response( - reader: &mut BufReader>, - request_id: u64, -) -> anyhow::Result { +fn read_shutdown_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { read_matching_response(reader, request_id, true) } fn read_matching_response( - reader: &mut BufReader>, + reader: &mut TransportReader, request_id: u64, accept_unidentified_error: bool, ) -> anyhow::Result { @@ -257,7 +323,7 @@ fn read_matching_response( let mut line = String::new(); loop { match reader.read_line(&mut line) { - Ok(0) => anyhow::bail!("transport closed before response"), + Ok(0) => anyhow::bail!(crate::localization::catalog().server.response_closed), Ok(_) => {} Err(error) if matches!( @@ -267,10 +333,10 @@ fn read_matching_response( { continue; } - Err(error) => return Err(error.into()), + Err(_) => anyhow::bail!(crate::localization::catalog().server.transport_failed), } let value: Value = serde_json::from_str(&line) - .map_err(|error| anyhow::anyhow!("bad response: {error}"))?; + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.response_invalid))?; line.clear(); if value.get("event").is_some() { continue; @@ -287,9 +353,9 @@ fn read_matching_response( } #[cfg(unix)] -fn response_data<'a>(response: &'a Value, command: &str) -> anyhow::Result<&'a Value> { +fn response_data(response: &Value) -> anyhow::Result<&Value> { if response.get("ok").and_then(Value::as_bool) != Some(true) { - anyhow::bail!("{command} failed"); + anyhow::bail!(crate::localization::catalog().server.legacy_list_failed); } Ok(response.get("data").unwrap_or(&Value::Null)) } @@ -311,7 +377,7 @@ fn legacy_surface_ids(data: &Value) -> Vec { surfaces } -fn wait_for_disconnect(reader: &mut BufReader>) -> anyhow::Result<()> { +fn wait_for_disconnect(reader: &mut TransportReader) -> anyhow::Result<()> { let deadline = Instant::now() + RESPONSE_TIMEOUT; let mut line = String::new(); loop { @@ -331,7 +397,7 @@ fn wait_for_disconnect(reader: &mut BufReader>) -> an { anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out) } - Err(error) => return Err(error.into()), + Err(_) => anyhow::bail!(crate::localization::catalog().server.transport_failed), } } } @@ -375,6 +441,10 @@ mod tests { assert!(message.contains("server: v0.1.0-old protocol 8")); assert!(message.contains("client: v")); + assert!(message.contains("reason: distribution version differs")); + assert!(message.contains("source build differs")); + assert!(message.contains("terminal engine build differs")); + assert!(message.contains("protocol differs")); assert!(message.contains("cmux-tui server stop --socket '/tmp/test socket'")); assert!(message.contains("Stopping exits pane processes.")); } @@ -385,11 +455,20 @@ mod tests { let mut child = Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); - terminate_legacy_server(child.id()).unwrap(); + let pid = verified_legacy_pid(child.id(), Some(child.id())).unwrap(); + terminate_legacy_server(pid).unwrap(); assert!(!child.wait().unwrap().success()); } + #[cfg(unix)] + #[test] + fn legacy_stop_rejects_a_reported_pid_that_is_not_the_socket_peer() { + let error = verified_legacy_pid(41, Some(42)).unwrap_err(); + + assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_peer_mismatch); + } + #[cfg(unix)] #[test] fn legacy_surface_ids_follow_the_workspace_tree() { diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index d272c67a5541..74cc4195c4ff 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -6,6 +6,8 @@ use std::io::{BufRead, BufReader, Read, Write}; use std::os::fd::FromRawFd; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; +#[cfg(unix)] +use std::os::unix::process::ExitStatusExt; use std::path::PathBuf; use std::process::{Child, Command, Output, Stdio}; use std::sync::mpsc; @@ -198,6 +200,174 @@ fn process_group_exists(_pid: u32) -> bool { false } +#[cfg(unix)] +struct LegacyServerProcess { + child: Child, + socket: PathBuf, + dir: PathBuf, +} + +#[cfg(unix)] +impl LegacyServerProcess { + fn start(name: &str, scenario: &str, reported_pid: Option) -> Self { + let dir = unique_temp_dir(name); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--ignored", "--exact", "legacy_server_process_helper"]) + .env("CMUX_TUI_TEST_LEGACY_SOCKET", &socket) + .env("CMUX_TUI_TEST_LEGACY_SCENARIO", scenario) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + if let Some(reported_pid) = reported_pid { + command.env("CMUX_TUI_TEST_LEGACY_REPORTED_PID", reported_pid.to_string()); + } + let child = command.spawn().unwrap(); + let mut server = Self { child, socket, dir }; + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline { + if server.socket.exists() { + return server; + } + if let Some(status) = server.child.try_wait().unwrap() { + let mut stderr = String::new(); + server.child.stderr.take().unwrap().read_to_string(&mut stderr).unwrap(); + panic!("legacy server helper exited with {status}: {stderr}"); + } + std::thread::sleep(Duration::from_millis(25)); + } + panic!("legacy server helper did not create socket at {}", server.socket.display()); + } +} + +#[cfg(unix)] +impl Drop for LegacyServerProcess { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + let _ = fs::remove_file(&self.socket); + let _ = fs::remove_dir_all(&self.dir); + } +} + +#[cfg(unix)] +#[test] +#[ignore] +fn legacy_server_process_helper() { + let socket = PathBuf::from(std::env::var_os("CMUX_TUI_TEST_LEGACY_SOCKET").unwrap()); + let scenario = std::env::var("CMUX_TUI_TEST_LEGACY_SCENARIO").unwrap(); + let reported_pid = std::env::var("CMUX_TUI_TEST_LEGACY_REPORTED_PID") + .ok() + .map(|value| value.parse::().unwrap()) + .unwrap_or_else(std::process::id); + let listener = transport::listen(&socket).unwrap(); + let mut stream = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + + reader.read_line(&mut request).unwrap(); + let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(identify_request["cmd"].as_str(), Some("identify")); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": identify_request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": reported_pid, + "version": "0.0.0-stale", + "protocol": 8, + }, + }) + ) + .unwrap(); + + request.clear(); + reader.read_line(&mut request).unwrap(); + assert_eq!( + serde_json::from_str::(&request).unwrap()["cmd"].as_str(), + Some("shutdown") + ); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": serde_json::Value::Null, + "ok": false, + "error": "bad request", + }) + ) + .unwrap(); + + request.clear(); + if reader.read_line(&mut request).unwrap() == 0 { + loop { + std::thread::park(); + } + } + let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); + if scenario == "cleanup-failure" { + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": false, + "error": "cleanup unavailable", + }) + ) + .unwrap(); + loop { + std::thread::park(); + } + } + assert_eq!(scenario, "success"); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": true, + "data": { + "workspaces": [{ + "screens": [{ + "panes": [{ + "tabs": [{"surface": 41}, {"surface": 42}], + }], + }], + }], + }, + }) + ) + .unwrap(); + + for expected_surface in [41, 42] { + request.clear(); + reader.read_line(&mut request).unwrap(); + let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); + assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": close_request["id"], + "ok": true, + "data": {}, + }) + ) + .unwrap(); + } + loop { + std::thread::park(); + } +} + fn wait_for_socket_path(path: &std::path::Path) { let deadline = Instant::now() + Duration::from_secs(15); while Instant::now() < deadline { @@ -517,7 +687,7 @@ fn incompatible_server_allows_identity_and_status_but_rejects_ping() { let listener = transport::listen(&socket).unwrap(); let server = std::thread::spawn(move || { let mut commands = Vec::new(); - for _ in 0..3 { + for _ in 0..4 { let mut stream = listener.accept().unwrap(); let mut request = String::new(); BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); @@ -546,7 +716,24 @@ fn incompatible_server_allows_identity_and_status_but_rejects_ping() { .output() .unwrap(); assert_success(&status); - assert!(String::from_utf8_lossy(&status.stdout).contains("status: incompatible")); + let status_text = String::from_utf8_lossy(&status.stdout); + assert!(status_text.contains("status: incompatible")); + assert!(status_text.contains("reason: distribution version differs")); + assert!(status_text.contains("source build differs")); + assert!(status_text.contains("terminal engine build differs")); + + let status_json = Command::new(bin()) + .args(["--json", "server", "status", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_success(&status_json); + let status_json: serde_json::Value = serde_json::from_slice(&status_json.stdout).unwrap(); + assert_eq!( + status_json["mismatch_reasons"], + serde_json::json!(["distribution-version", "source-build", "terminal-engine"]) + ); let identify = Command::new(bin()) .args(["identify", "--socket"]) @@ -566,7 +753,7 @@ fn incompatible_server_allows_identity_and_status_but_rejects_ping() { assert_eq!(ping.status.code(), Some(1)); assert!(String::from_utf8_lossy(&ping.stderr).contains("cmux-tui server stop")); - assert_eq!(server.join().unwrap(), ["identify", "identify", "identify"]); + assert_eq!(server.join().unwrap(), ["identify", "identify", "identify", "identify"]); let _ = fs::remove_file(&socket); let _ = fs::remove_dir_all(&dir); } @@ -579,14 +766,18 @@ fn server_status_and_stop_control_a_compatible_headless_session() { let status: serde_json::Value = serde_json::from_slice(&status.stdout).unwrap(); assert_eq!(status["running"].as_bool(), Some(true)); assert_eq!(status["compatible"].as_bool(), Some(true)); + assert_eq!(status["mismatch_reasons"], serde_json::json!([])); assert_eq!( status["server"]["version"].as_str(), Some(cmux_tui_core::release::distribution_version()) ); - let stop = cli(&server, &["server", "stop"]); + let stop = cli(&server, &["--json", "server", "stop"]); assert_success(&stop); - assert!(String::from_utf8_lossy(&stop.stdout).contains("Stopped the cmux-tui server.")); + assert_eq!( + serde_json::from_slice::(&stop.stdout).unwrap(), + serde_json::json!({"stopped": true}) + ); assert!(server.child.wait().unwrap().success()); assert!(!server.socket.exists()); } @@ -594,175 +785,52 @@ fn server_status_and_stop_control_a_compatible_headless_session() { #[cfg(unix)] #[test] fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { - let dir = unique_temp_dir("legacy-server-stop"); - fs::create_dir_all(&dir).unwrap(); - let socket = dir.join("mux.sock"); - let listener = transport::listen(&socket).unwrap(); - let mut pane_process = - Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); - let pane_pid = pane_process.id(); - let server = std::thread::spawn(move || { - let mut stream = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone_box().unwrap()); - let mut request = String::new(); - reader.read_line(&mut request).unwrap(); - let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(identify_request["cmd"].as_str(), Some("identify")); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": identify_request["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "session": "test", - "pid": pane_pid, - "version": "0.0.0-stale", - "protocol": 8, - }, - }) - ) - .unwrap(); - - request.clear(); - reader.read_line(&mut request).unwrap(); - assert_eq!( - serde_json::from_str::(&request).unwrap()["cmd"].as_str(), - Some("shutdown") - ); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": serde_json::Value::Null, - "ok": false, - "error": "bad request", - }) - ) - .unwrap(); - - request.clear(); - reader.read_line(&mut request).unwrap(); - let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": list_request["id"], - "ok": true, - "data": { - "workspaces": [{ - "screens": [{ - "panes": [{ - "tabs": [{"surface": 41}, {"surface": 42}], - }], - }], - }], - }, - }) - ) - .unwrap(); - - for expected_surface in [41, 42] { - request.clear(); - reader.read_line(&mut request).unwrap(); - let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); - assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": close_request["id"], - "ok": true, - "data": {}, - }) - ) - .unwrap(); - } - }); + let mut server = LegacyServerProcess::start("legacy-server-stop", "success", None); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) - .arg(&socket) + .arg(&server.socket) .env_remove("CMUX_TUI_SOCKET") .output() .unwrap(); assert_success(&output); - assert!(!pane_process.wait().unwrap().success()); - server.join().unwrap(); - let _ = fs::remove_file(&socket); - let _ = fs::remove_dir_all(&dir); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGTERM)); } #[cfg(unix)] #[test] fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { - let dir = unique_temp_dir("legacy-server-cleanup-failure"); - fs::create_dir_all(&dir).unwrap(); - let socket = dir.join("mux.sock"); - let listener = transport::listen(&socket).unwrap(); - let mut server_process = - Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); - let server_pid = server_process.id(); - let server = std::thread::spawn(move || { - let mut stream = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone_box().unwrap()); - let mut request = String::new(); - reader.read_line(&mut request).unwrap(); - let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": identify_request["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "session": "test", - "pid": server_pid, - "version": "0.0.0-stale", - "protocol": 8, - }, - }) - ) - .unwrap(); + let mut server = + LegacyServerProcess::start("legacy-server-cleanup-failure", "cleanup-failure", None); - request.clear(); - reader.read_line(&mut request).unwrap(); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": serde_json::Value::Null, - "ok": false, - "error": "bad request", - }) - ) + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() .unwrap(); - request.clear(); - reader.read_line(&mut request).unwrap(); - let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": list_request["id"], - "ok": false, - "error": "cleanup unavailable", - }) - ) - .unwrap(); - }); + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(server.child.try_wait().unwrap().is_none()); +} + +#[cfg(unix)] +#[test] +fn server_stop_refuses_a_legacy_server_that_spoofs_another_process_id() { + let mut victim = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let mut server = + LegacyServerProcess::start("legacy-server-spoofed-pid", "spoofed-pid", Some(victim.id())); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) - .arg(&socket) + .arg(&server.socket) .env_remove("CMUX_TUI_SOCKET") .output() .unwrap(); @@ -770,14 +838,12 @@ fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { assert_eq!(output.status.code(), Some(1)); assert!( String::from_utf8_lossy(&output.stderr) - .contains("could not close pane processes before stopping the older server") + .contains("reported process does not own its socket") ); - assert!(server_process.try_wait().unwrap().is_none()); - server_process.kill().unwrap(); - server_process.wait().unwrap(); - server.join().unwrap(); - let _ = fs::remove_file(&socket); - let _ = fs::remove_dir_all(&dir); + assert!(victim.try_wait().unwrap().is_none()); + assert!(server.child.try_wait().unwrap().is_none()); + victim.kill().unwrap(); + victim.wait().unwrap(); } #[cfg(unix)] diff --git a/cmux-tui/dist/RELEASING-TUI.md b/cmux-tui/dist/RELEASING-TUI.md index 42641be0f57d..5d84b6da13a9 100644 --- a/cmux-tui/dist/RELEASING-TUI.md +++ b/cmux-tui/dist/RELEASING-TUI.md @@ -14,6 +14,11 @@ clients can distinguish releases even though the Cargo crate version remains fixed. Cutting a stable TUI release is just creating a `cmux-tui-vX.Y.Z` tag on `main`. +The workflow also stamps the repository and terminal-engine commits. Ordinary +source builds derive those identities automatically and add a deterministic +content fingerprint for dirty trees. Explicit workflow stamps remain the +canonical distribution identity. + ## Packages - npm `cmux`: launcher package for `npx cmux`. diff --git a/cmux-tui/docs/protocol.md b/cmux-tui/docs/protocol.md index 6d57a77d0dd0..f3e02676ab2d 100644 --- a/cmux-tui/docs/protocol.md +++ b/cmux-tui/docs/protocol.md @@ -120,7 +120,9 @@ When the stream ends, it sends: ## Client Compatibility -Direct local clients require the server's distribution version, build commit, Ghostty commit, and protocol to match exactly. `identify`, `cmux-tui server status`, and `cmux-tui server stop` bypass that check so an upgraded client can inspect and stop an older server. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. +Direct local clients require the server's distribution version, source build, terminal-engine build, and protocol to match exactly. Ordinary source builds derive both build identities automatically, including a deterministic fingerprint when the source tree is dirty. `identify`, `cmux-tui server status`, and `cmux-tui server stop` bypass that check so an upgraded client can inspect and stop an older server. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. + +When an older local server lacks orderly shutdown, the replacement client verifies from the Unix socket that the reported server PID is the connected peer. It refuses to signal an unverified process. Remote transports require protocol v9 but do not require the same release build. They reject protocol-v8 servers before loading their workspace tree because v8 does not define stack layout nodes or `new-pane`. diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index f4b53a43eff9..0fdf510cb513 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -180,7 +180,7 @@ Result: object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:string|null,protocol:uint32,capabilities:array,session:string,pid:uint32,registry_id:string,generation:string,workspace_revision:uint64} ``` -`build_commit` and `ghostty_commit` are additive build-stamp fields. They are omitted or `null` when the binary was built without the corresponding stamp, so clients must preserve compatibility with older servers and unstamped local builds. +`build_commit` and `ghostty_commit` are additive build-stamp fields. Current source builds derive them from the repository and terminal-engine checkout. A dirty source build appends a deterministic content fingerprint. Distribution builds may override both stamps explicitly. Older binaries can omit the fields or return `null`, so clients must preserve compatibility with those identities. `capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. @@ -261,7 +261,21 @@ Errors: | `shutdown is only available over the local session socket` | Request arrived over WebSocket | | `bad request: ...` | Malformed request envelope | -CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client closes every surface reported by `list-workspaces` before signaling the PID reported by `identify`. If orderly pane cleanup fails, the client leaves the server running and reports the failure. +CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client first verifies that the PID reported by `identify` owns the connected Unix socket. It then closes every surface reported by `list-workspaces` before signaling that verified process. If socket ownership cannot be verified or orderly pane cleanup fails, the client leaves the server running and reports the failure. + +`cmux-tui server status` compares the local server's distribution version, source build, terminal-engine build, and protocol with the client. Plain output reports both versions and protocols plus localized mismatch reasons. JSON output is: + +```text +object{ + running:true, + compatible:boolean, + mismatch_reasons:array<"distribution-version"|"source-build"|"terminal-engine"|"protocol">, + server:object{version:string,protocol:uint32}, + client:object{version:string,protocol:uint32} +} +``` + +`cmux-tui server stop` prints a localized confirmation. With `--json`, it prints `object{stopped:true}`. Example: From 9d863b43c747fd5be699c987a1c7d240c1e9376f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 03:36:44 -0700 Subject: [PATCH 005/521] fix(tui): make server shutdown race-safe --- cmux-tui/crates/cmux-tui-core/build.rs | 28 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 405 +++++++++++++++++- cmux-tui/crates/cmux-tui-core/src/release.rs | 10 +- cmux-tui/crates/cmux-tui-core/src/server.rs | 6 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 40 +- .../src/terminal_host_runtime.rs | 22 + cmux-tui/crates/cmux-tui/src/cli.rs | 6 +- cmux-tui/crates/cmux-tui/src/localization.rs | 3 + cmux-tui/crates/cmux-tui/src/main.rs | 9 + .../crates/cmux-tui/src/server_lifecycle.rs | 88 +++- cmux-tui/crates/cmux-tui/tests/cli.rs | 196 +++++++-- cmux-tui/docs/protocol.md | 2 +- cmux-tui/spec/commands.md | 4 +- 13 files changed, 740 insertions(+), 79 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index de434ba7407d..180b651b8f4b 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -23,6 +23,7 @@ fn main() { let repository_root = workspace_root.parent().expect("the cmux-tui workspace is inside the repository"); let ghostty_root = env::var_os("CMUX_GHOSTTY_SRC") + .filter(|value| !value.is_empty()) .map(PathBuf::from) .unwrap_or_else(|| repository_root.join("ghostty")); @@ -35,19 +36,23 @@ fn main() { track_git_identity(repository_root); track_git_identity(&ghostty_root); - if env::var_os("CMUX_TUI_BUILD_COMMIT").is_none() - && env::var_os("CMUX_MUX_BUILD_COMMIT").is_none() + if !nonempty_env("CMUX_TUI_BUILD_COMMIT") + && !nonempty_env("CMUX_MUX_BUILD_COMMIT") && let Some(identity) = source_identity(workspace_root, workspace_root) { println!("cargo:rustc-env=CMUX_TUI_SOURCE_COMMIT={identity}"); } - if env::var_os("CMUX_TUI_GHOSTTY_COMMIT").is_none() + if !nonempty_env("CMUX_TUI_GHOSTTY_COMMIT") && let Some(identity) = source_identity(&ghostty_root, &ghostty_root) { println!("cargo:rustc-env=CMUX_TUI_SOURCE_GHOSTTY_COMMIT={identity}"); } } +fn nonempty_env(name: &str) -> bool { + env::var_os(name).is_some_and(|value| !value.is_empty()) +} + fn source_identity(git_root: &Path, fallback_root: &Path) -> Option { if let Some(commit) = git_text(git_root, &["rev-parse", "HEAD"]) { let status = @@ -81,7 +86,10 @@ fn directory_fingerprint(root: &Path) -> std::io::Result { entries.sort_by_key(|entry| entry.file_name()); for entry in entries { let name = entry.file_name(); - if matches!(name.to_str(), Some(".git" | "target" | "zig-cache" | "zig-out")) { + if matches!( + name.to_str(), + Some(".git" | "target" | ".zig-cache" | "zig-cache" | "zig-out") + ) { continue; } let path = entry.path(); @@ -112,15 +120,23 @@ fn hash_bytes(hash: &mut u64, bytes: &[u8]) { fn track_git_identity(root: &Path) { let Some(head) = git_text(root, &["rev-parse", "--git-path", "HEAD"]) else { return }; - println!("cargo:rerun-if-changed={head}"); + println!("cargo:rerun-if-changed={}", resolve_git_path(root, &head).display()); if let Some(reference) = git_text(root, &["symbolic-ref", "-q", "HEAD"]) && let Some(reference_path) = git_text(root, &["rev-parse", "--git-path", reference.as_str()]) { - println!("cargo:rerun-if-changed={reference_path}"); + println!("cargo:rerun-if-changed={}", resolve_git_path(root, &reference_path).display()); + } + if let Some(packed_refs) = git_text(root, &["rev-parse", "--git-path", "packed-refs"]) { + println!("cargo:rerun-if-changed={}", resolve_git_path(root, &packed_refs).display()); } } +fn resolve_git_path(root: &Path, value: &str) -> PathBuf { + let path = PathBuf::from(value); + if path.is_absolute() { path } else { root.join(path) } +} + fn git_text(root: &Path, args: &[&str]) -> Option { let output = git_bytes(root, args)?; let value = std::str::from_utf8(&output).ok()?.trim(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fbed126ffedc..949095865974 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -4,9 +4,10 @@ use std::collections::{HashMap, HashSet}; use std::fmt; use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; use std::sync::{Arc, Mutex, MutexGuard, Weak}; +use std::thread::ThreadId; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use anyhow::Context; @@ -41,6 +42,8 @@ const WORKSPACE_KEY_MAX_BYTES: usize = 256; const WORKSPACE_NAME_MAX_BYTES: usize = 1_024; const PROVIDER_WORKSPACE_AUTHORITY_MIN_BYTES: usize = 32; const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; +const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); +const SHUTDOWN_FANOUT_WORKERS: usize = 128; /// An opaque per-mux credential provisioned by the external machine /// provider. Debug output is deliberately redacted. @@ -588,6 +591,65 @@ impl Drop for PendingWorkspaceSurface<'_> { } } +#[derive(Default)] +struct SurfaceCreationState { + shutting_down: bool, + active_threads: HashMap, +} + +#[derive(Default)] +struct SurfaceCreationGate { + state: Mutex, + idle: std::sync::Condvar, +} + +impl SurfaceCreationGate { + fn begin(&self) -> anyhow::Result> { + let thread = std::thread::current().id(); + let mut state = self.state.lock().unwrap(); + if let Some(depth) = state.active_threads.get_mut(&thread) { + *depth = depth.saturating_add(1); + } else { + if state.shutting_down { + anyhow::bail!("server is shutting down"); + } + state.active_threads.insert(thread, 1); + } + Ok(SurfaceCreationGuard { gate: self, thread }) + } + + fn stop_and_wait(&self) { + let mut state = self.state.lock().unwrap(); + state.shutting_down = true; + while !state.active_threads.is_empty() { + state = self.idle.wait(state).unwrap(); + } + } +} + +struct SurfaceCreationGuard<'a> { + gate: &'a SurfaceCreationGate, + thread: ThreadId, +} + +impl Drop for SurfaceCreationGuard<'_> { + fn drop(&mut self) { + let mut state = self.gate.state.lock().unwrap(); + let remove = { + let depth = + state.active_threads.get_mut(&self.thread).expect("surface creation guard tracked"); + *depth -= 1; + *depth == 0 + }; + if remove { + state.active_threads.remove(&self.thread); + } + if state.active_threads.is_empty() { + self.gate.idle.notify_all(); + } + } +} + enum SurfaceResizeRestore { Complete(bool), Pending(Receiver), @@ -709,6 +771,7 @@ pub struct Mux { state: Mutex, subscribers: MuxEventBroadcaster, shutdown_requested: AtomicBool, + surface_creations: SurfaceCreationGate, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -935,6 +998,7 @@ impl Mux { }), subscribers: MuxEventBroadcaster::default(), shutdown_requested: AtomicBool::new(false), + surface_creations: SurfaceCreationGate::default(), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -1194,6 +1258,7 @@ impl Mux { terminal_id: &str, options: &SurfaceOptions, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; let terminal = self.workspace_registry.lock().unwrap().terminal_record(terminal_id)?; let Some(terminal) = terminal.filter(|terminal| { terminal.lifecycle == TerminalLifecycle::Exited && terminal.incarnation.is_some() @@ -1328,6 +1393,7 @@ impl Mux { incarnation: &str, surface: Arc, ) -> anyhow::Result<()> { + let _creation = self.begin_surface_creation()?; if surface.is_dead() { self.transition_terminal_lifecycle( "terminal-exited", @@ -1609,6 +1675,10 @@ impl Mux { self.next_id.fetch_add(1, Ordering::Relaxed) } + fn begin_surface_creation(&self) -> anyhow::Result> { + self.surface_creations.begin() + } + fn next_active_at(&self) -> u64 { self.next_active_at.fetch_add(1, Ordering::Relaxed) } @@ -3059,6 +3129,7 @@ impl Mux { incarnation: &str, workspace_key: &str, ) -> anyhow::Result { + let _creation = self.begin_surface_creation()?; let mut registry = self.workspace_registry.lock().unwrap(); commit_terminal_transition( &mut registry, @@ -3541,6 +3612,7 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); + self.surface_creations.stop_and_wait(); let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); for surface in surfaces { surface.disconnect_for_daemon_shutdown(); @@ -3558,6 +3630,117 @@ impl Mux { self.shutdown_requested.load(Ordering::Acquire) } + /// Permanently fence new surface creation, tombstone every durable + /// terminal in one transaction, detach the complete topology in one + /// state mutation, and terminate each runtime before acknowledging the + /// shutdown request. A failed durable commit leaves topology untouched; + /// a failed host termination leaves the fence closed so a retry cannot + /// race a new PTY into the session. + pub fn close_all_surfaces_for_shutdown(&self) -> anyhow::Result { + self.shutting_down.store(true, Ordering::Release); + self.surface_creations.stop_and_wait(); + + let (surfaces, tree_changed) = { + let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); + let mut registry = self.workspace_registry.lock().unwrap(); + let terminals = registry + .terminal_snapshot()? + .terminals + .into_iter() + .map(|terminal| (terminal.terminal_id, terminal.incarnation)) + .collect::>(); + let batch = registry.close_terminals_atomically(&mutation, &terminals)?; + + let (surfaces, tree_changed) = { + let mut state = self.state.lock().unwrap(); + let tree_changed = !state.surfaces.is_empty() + || !state.panes.is_empty() + || state.workspaces.iter().any(|workspace| !workspace.screens.is_empty()); + let surfaces = + state.surfaces.drain().map(|(_, surface)| surface).collect::>(); + for workspace in &mut state.workspaces { + workspace.screens.clear(); + workspace.active_screen = 0; + } + if !state.panes.is_empty() { + state.pane_revision = state.pane_revision.saturating_add(1); + } + state.panes.clear(); + state.split_screens.clear(); + (surfaces, tree_changed) + }; + + if batch.closed != 0 { + self.emit_terminal_registry_changed(®istry, batch.revision); + } + (surfaces, tree_changed) + }; + + self.pending_workspace_surfaces.lock().unwrap().clear(); + self.agent_records.lock().unwrap().clear(); + self.surface_notifications.lock().unwrap().clear(); + self.sidebar_plugin.lock().unwrap().surface = None; + { + let mut sizing = self.client_sizing.lock().unwrap(); + sizing.surfaces.clear(); + sizing.report_order.clear(); + let attached_clients = self.control_clients.attached_client_ids(); + self.reconcile_latest_client_size(&sizing, &attached_clients); + } + if tree_changed { + self.emit(MuxEvent::TreeChanged); + } + + #[cfg(unix)] + let terminated_hosts = Mutex::new(HashSet::new()); + bounded_shutdown_fanout(&surfaces, |surface| { + let accepted = surface.terminate_for_server_shutdown(SHUTDOWN_TERMINATION_TIMEOUT); + #[cfg(unix)] + if accepted && let Some(identity) = surface.terminal_host_identity() { + terminated_hosts + .lock() + .unwrap() + .insert((identity.terminal_id, identity.incarnation)); + } + }); + + #[cfg(unix)] + { + let terminated_hosts = terminated_hosts.into_inner().unwrap(); + let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + if let Some(root) = root { + let records = crate::terminal_host_runtime::load_terminal_host_records(&root) + .context("load terminal hosts for server shutdown")?; + let failed = Mutex::new(Vec::new()); + bounded_shutdown_fanout(&records, |(path, record)| { + if terminated_hosts + .contains(&(record.terminal_id.clone(), record.incarnation.clone())) + { + return; + } + if !cleanup_terminal_host_record_with_timeout( + record, + path, + SHUTDOWN_TERMINATION_TIMEOUT, + ) { + failed.lock().unwrap().push(record.terminal_id.clone()); + } + }); + let mut failed = failed.into_inner().unwrap(); + if !failed.is_empty() { + failed.sort(); + anyhow::bail!( + "could not terminate {} terminal host(s): {}", + failed.len(), + failed.join(", ") + ); + } + } + } + + Ok(surfaces.len()) + } + /// Atomically reserve a daemon handoff while proving no other native /// browser owns this mux. New owner announcements are rejected until the /// response is queued or the reservation is cancelled. @@ -3574,6 +3757,7 @@ impl Mux { /// and remain available for the replacement daemon to adopt. pub fn request_daemon_shutdown(&self) { self.shutting_down.store(true, Ordering::Release); + self.surface_creations.stop_and_wait(); } pub fn daemon_shutdown_requested(&self) -> bool { @@ -3613,6 +3797,16 @@ impl Mux { rows: u16, relaunch: bool, ) -> SidebarPluginStatus { + let _creation = match self.begin_surface_creation() { + Ok(creation) => creation, + Err(error) => { + return SidebarPluginStatus { + surface: None, + error: Some(error.to_string()), + retry_after: None, + }; + } + }; let now = Instant::now(); let size = (cols.max(1), rows.max(1)); let spawn_options = { @@ -3791,6 +3985,7 @@ impl Mux { name: Option, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; let workspace = self.create_empty_workspace(name, None, None)?.workspace; let (_, surface) = self.create_terminal_in_workspace_impl(workspace, None, None, None, size, None)?; @@ -3970,6 +4165,7 @@ impl Mux { argv: Vec, options: RunCommandOptions, ) -> anyhow::Result { + let _creation = self.begin_surface_creation()?; let RunCommandOptions { pane, new_workspace, workspace_key, cwd, name, size } = options; if workspace_key.is_some() && !new_workspace { anyhow::bail!("workspace key requires a new workspace"); @@ -4087,6 +4283,7 @@ impl Mux { cwd: Option, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; // Validate the target before spawning a child. let workspace_key = { let state = self.state.lock().unwrap(); @@ -4175,6 +4372,7 @@ impl Mux { cwd: Option, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; // Resolve and validate the target before spawning a child. let (target, empty_workspace) = { let state = self.state.lock().unwrap(); @@ -4396,6 +4594,7 @@ impl Mux { size: Option<(u16, u16)>, reservation: Option, ) -> anyhow::Result<(RunPlacement, Arc)> { + let _creation = self.begin_surface_creation()?; { let state = self.state.lock().unwrap(); if state.workspace_by_id(workspace).is_none() { @@ -4611,6 +4810,7 @@ impl Mux { pane: Option, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; let (target, empty_workspace) = { let state = self.state.lock().unwrap(); let target = match pane { @@ -4819,6 +5019,7 @@ impl Mux { url: String, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; let lifecycle = self.workspace_lifecycle(workspace); let workspace_lifecycle = lifecycle.lock().unwrap(); if self.state.lock().unwrap().workspace_by_id(workspace).is_none() { @@ -4922,6 +5123,9 @@ impl Mux { url: String, runtime: Arc, ) -> bool { + let Ok(_creation) = self.begin_surface_creation() else { + return false; + }; let (pane_id, size) = { let state = self.state.lock().unwrap(); let Some(pane_id) = state.pane_of(opener_surface) else { @@ -5022,6 +5226,7 @@ impl Mux { dir: SplitDir, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; let cwd = self.pane_cwd(target); let workspace_key = self .workspace_key_for_pane(target) @@ -5110,6 +5315,7 @@ impl Mux { target: PaneId, size: Option<(u16, u16)>, ) -> anyhow::Result> { + let _creation = self.begin_surface_creation()?; if self.state.lock().unwrap().screen_of(target).is_none() { anyhow::bail!("unknown pane {target}"); } @@ -6259,6 +6465,7 @@ impl Mux { layout: &LayoutSpec, size: Option<(u16, u16)>, ) -> anyhow::Result { + let _creation = self.begin_surface_creation()?; let target_workspace = { let state = self.state.lock().unwrap(); if let Some(id) = workspace @@ -7238,6 +7445,23 @@ fn terminate_host_record( } } +#[cfg(unix)] +fn terminate_host_record_with_timeout( + record: crate::terminal_host_runtime::TerminalHostRecord, + record_path: std::path::PathBuf, + timeout: Duration, +) -> bool { + if let Ok(host) = + crate::terminal_host_runtime::adopt_terminal_host_with_timeout(record, record_path, timeout) + { + let terminated = host.terminate_with_timeout(timeout).is_ok(); + host.disconnect(); + terminated + } else { + false + } +} + #[cfg(unix)] fn terminal_host_record_liveness( record_path: &Path, @@ -7276,6 +7500,50 @@ fn cleanup_terminal_host_record( } } +#[cfg(unix)] +fn cleanup_terminal_host_record_with_timeout( + record: &crate::terminal_host_runtime::TerminalHostRecord, + record_path: &Path, + timeout: Duration, +) -> bool { + match terminal_host_record_liveness(record_path, record) { + TerminalHostLiveness::Dead => { + match crate::terminal_host_runtime::remove_stale_terminal_host_record( + record_path, + record, + ) { + Ok(removed) => removed, + Err(_) if !record_path.exists() => true, + Err(_) => false, + } + } + TerminalHostLiveness::Live | TerminalHostLiveness::Indeterminate => { + terminate_host_record_with_timeout(record.clone(), record_path.to_path_buf(), timeout) + } + } +} + +fn bounded_shutdown_fanout(items: &[T], operation: impl Fn(&T) + Sync) { + if items.is_empty() { + return; + } + let next = AtomicUsize::new(0); + let workers = items.len().min(SHUTDOWN_FANOUT_WORKERS); + std::thread::scope(|scope| { + for _ in 0..workers { + let next = &next; + let operation = &operation; + scope.spawn(move || { + loop { + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(index) else { break }; + operation(item); + } + }); + } + }); +} + fn insert_surface_checked(state: &mut State, surface: Arc) -> anyhow::Result<()> { if state.surfaces.contains_key(&surface.id) { anyhow::bail!("duplicate_surface_id"); @@ -9486,6 +9754,137 @@ mod tests { }); } + #[test] + fn server_shutdown_clears_many_surfaces_with_one_tree_event() { + let mux = test_mux(); + let first = mux.new_workspace(None, Some((120, 40))).unwrap(); + let pane = mux.with_state(|state| state.pane_of(first.id).unwrap()); + for _ in 0..450 { + mux.new_tab(Some(pane), None, None).unwrap(); + } + let events = mux.subscribe(); + + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 451); + + mux.with_state(|state| { + assert!(state.surfaces.is_empty()); + assert!(state.panes.is_empty()); + assert!(state.split_screens.is_empty()); + assert!(state.workspaces.iter().all(|workspace| workspace.screens.is_empty())); + }); + let events = events.try_iter().collect::>(); + assert_eq!(events.iter().filter(|event| matches!(event, MuxEvent::TreeChanged)).count(), 1); + assert_eq!(events.len(), 1); + let error = mux.new_workspace(None, None).unwrap_err(); + assert_eq!(error.to_string(), "server is shutting down"); + } + + #[test] + fn shutdown_fanout_runs_terminations_concurrently() { + let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(8); + let worker = std::thread::spawn({ + let gate = gate.clone(); + move || { + bounded_shutdown_fanout(&[(); 8], |_| { + started_tx.send(()).unwrap(); + let (released, wake) = &*gate; + let released = + wake.wait_while(released.lock().unwrap(), |released| !*released).unwrap(); + drop(released); + }); + } + }); + + let deadline = Instant::now() + Duration::from_secs(1); + let mut started = 0; + while started < 8 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { break }; + if started_rx.recv_timeout(remaining).is_err() { + break; + } + started += 1; + } + let (released, wake) = &*gate; + *released.lock().unwrap() = true; + wake.notify_all(); + worker.join().unwrap(); + assert_eq!(started, 8); + } + + #[test] + fn server_shutdown_waits_for_in_flight_surface_creation() { + let mux = test_mux(); + let workspace = mux.create_empty_workspace(Some("race".into()), None, None).unwrap(); + let (creation_reached_tx, creation_reached_rx) = std::sync::mpsc::sync_channel(1); + let (release_creation_tx, release_creation_rx) = std::sync::mpsc::sync_channel(1); + let release_creation_rx = Arc::new(Mutex::new(release_creation_rx)); + *mux.terminal_create_after_materialization_lock.lock().unwrap() = Some(Arc::new({ + move || { + creation_reached_tx.send(()).unwrap(); + release_creation_rx.lock().unwrap().recv().unwrap(); + } + })); + + let create = std::thread::spawn({ + let mux = mux.clone(); + move || { + mux.create_terminal_surface_in_workspace( + workspace.workspace, + None, + None, + None, + Some((80, 24)), + ) + .unwrap() + .0 + .id + } + }); + creation_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); + let shutdown = std::thread::spawn({ + let mux = mux.clone(); + move || { + shutdown_done_tx.send(mux.close_all_surfaces_for_shutdown().unwrap()).unwrap(); + } + }); + assert!(shutdown_done_rx.recv_timeout(Duration::from_millis(100)).is_err()); + + release_creation_tx.send(()).unwrap(); + let created = create.join().unwrap(); + assert_eq!(shutdown_done_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 1); + shutdown.join().unwrap(); + assert!(mux.surface(created).is_none()); + assert_eq!( + mux.new_tab(None, None, None).unwrap_err().to_string(), + "server is shutting down" + ); + *mux.terminal_create_after_materialization_lock.lock().unwrap() = None; + } + + #[cfg(unix)] + #[test] + fn server_shutdown_keeps_topology_when_terminal_batch_commit_fails() { + let mux = test_mux(); + let workspace = mux.create_empty_workspace(Some("durable".into()), None, None).unwrap(); + let terminal_id = "00000000000040008000000000000021"; + let incarnation = "10000000000040008000000000000021"; + let surface = + mux.seed_running_terminal_for_test(terminal_id, incarnation, &workspace.key).unwrap(); + mux.set_terminal_close_failure_for_test(true).unwrap(); + + let error = mux.close_all_surfaces_for_shutdown().unwrap_err(); + + assert!(format!("{error:#}").contains("forced terminal close failure")); + assert!(mux.surface(surface).is_some()); + assert!(mux.with_state(|state| state.pane_of(surface).is_some())); + mux.set_terminal_close_failure_for_test(false).unwrap(); + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + assert!(mux.surface(surface).is_none()); + } + #[test] fn closing_active_pane_focuses_most_recent_remaining_pane() { let mux = test_mux(); @@ -10636,7 +11035,7 @@ mod tests { .unwrap(); let surface = insert_terminal_identity_surface(&mux, TERMINAL, INCARNATION, &first.key); - let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let calls = Arc::new(AtomicUsize::new(0)); let (entered_tx, entered_rx) = std::sync::mpsc::channel(); let (release_tx, release_rx) = std::sync::mpsc::channel(); let release_rx = Arc::new(Mutex::new(release_rx)); @@ -10929,7 +11328,7 @@ mod tests { fn concurrent_empty_workspace_terminal_inherits_the_first_terminals_cwd() { let mux = test_mux(); let workspace = mux.create_empty_workspace(Some("shared".into()), None, None).unwrap(); - let empty_checks = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let empty_checks = Arc::new(AtomicUsize::new(0)); let (second_checked_tx, second_checked_rx) = std::sync::mpsc::sync_channel(1); *mux.terminal_create_after_empty_check.lock().unwrap() = Some(Arc::new({ move || { diff --git a/cmux-tui/crates/cmux-tui-core/src/release.rs b/cmux-tui/crates/cmux-tui-core/src/release.rs index 2b201b77af1e..3ffbd5881c53 100644 --- a/cmux-tui/crates/cmux-tui-core/src/release.rs +++ b/cmux-tui/crates/cmux-tui-core/src/release.rs @@ -60,15 +60,15 @@ pub fn distribution_version() -> &'static str { pub fn stamped_build_commit() -> Option<&'static str> { option_env!("CMUX_TUI_BUILD_COMMIT") - .or(option_env!("CMUX_MUX_BUILD_COMMIT")) - .or(option_env!("CMUX_TUI_SOURCE_COMMIT")) .filter(|commit| !commit.is_empty()) + .or_else(|| option_env!("CMUX_MUX_BUILD_COMMIT").filter(|commit| !commit.is_empty())) + .or_else(|| option_env!("CMUX_TUI_SOURCE_COMMIT").filter(|commit| !commit.is_empty())) } pub fn stamped_ghostty_commit() -> Option<&'static str> { - option_env!("CMUX_TUI_GHOSTTY_COMMIT") - .or(option_env!("CMUX_TUI_SOURCE_GHOSTTY_COMMIT")) - .filter(|commit| !commit.is_empty()) + option_env!("CMUX_TUI_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()).or_else(|| { + option_env!("CMUX_TUI_SOURCE_GHOSTTY_COMMIT").filter(|commit| !commit.is_empty()) + }) } fn optional_string(data: &Value, key: &str) -> Option { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 3de2ad70e247..51a73d1a120b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2969,11 +2969,7 @@ fn handle_command( if !mux.control_clients.is_unix(client) { anyhow::bail!("shutdown is only available over the local session socket"); } - let surfaces = - mux.with_state(|state| state.surfaces.keys().copied().collect::>()); - for surface in surfaces { - mux.close_surface(surface)?; - } + mux.close_all_surfaces_for_shutdown()?; Ok(json!({})) } Command::ShutdownDaemon { pid, generation } => { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 0c6d5c263b32..7a4d24f2e9f6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2061,12 +2061,17 @@ impl Surface { } pub fn kill(&self) { + let _ = self.terminate_runtime(); + } + + fn terminate_runtime(&self) -> bool { match self { Surface::Pty(pty) => { let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { PtyRuntime::Local { killer, .. } => { let _ = killer.kill(); + true } #[cfg(unix)] PtyRuntime::Hosted(host) => { @@ -2074,13 +2079,42 @@ impl Surface { // after the PTY process has actually exited. Unlinking // here would make a failed Terminate write turn a live // shell into an undiscoverable orphan. - let _ = host.terminate(); + host.terminate().is_ok() } #[cfg(unix)] - PtyRuntime::ExitedHosted => {} + PtyRuntime::ExitedHosted => true, } } - Surface::Browser(browser) => browser.kill(), + Surface::Browser(browser) => { + browser.kill(); + true + } + } + } + + /// Terminate the runtime while reporting whether an attached terminal + /// host accepted the request. Callers can retry a failed hosted request + /// through its durable discovery record after removing topology. + pub(crate) fn terminate_for_server_shutdown(&self, timeout: Duration) -> bool { + match self { + #[cfg(unix)] + Surface::Pty(pty) => { + let mut runtime = pty.runtime.lock().unwrap(); + match &mut *runtime { + PtyRuntime::Hosted(host) => host.terminate_with_timeout(timeout).is_ok(), + PtyRuntime::Local { killer, .. } => { + let _ = killer.kill(); + true + } + PtyRuntime::ExitedHosted => true, + } + } + #[cfg(not(unix))] + Surface::Pty(_) => self.terminate_runtime(), + Surface::Browser(browser) => { + browser.kill(); + true + } } } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 9443b82320cc..c8def7412b09 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -636,6 +636,17 @@ mod unix { self.send(MessageKind::Terminate, &[]) } + pub(crate) fn terminate_with_timeout(&self, timeout: Duration) -> std::io::Result<()> { + let mut writer = self.writer.lock().unwrap(); + writer.set_write_timeout(Some(timeout))?; + let frame = Frame::new(MessageKind::Terminate, Vec::new()); + let result = write_frame(&mut *writer, &frame).map_err(protocol_io_error); + if result.is_err() { + let _ = writer.shutdown(std::net::Shutdown::Both); + } + result + } + pub fn disconnect(&self) { let _ = self.writer.lock().unwrap().shutdown(std::net::Shutdown::Both); } @@ -931,6 +942,15 @@ mod unix { connect_record(record, record_path) } + pub(crate) fn adopt_terminal_host_with_timeout( + record: TerminalHostRecord, + record_path: PathBuf, + timeout: Duration, + ) -> anyhow::Result { + validate_terminal_host_record(&record_path, &record)?; + connect_record_with_timeout(record, record_path, timeout) + } + /// Validate a discovery record without trusting paths or alternate /// identity spellings supplied by its JSON payload. pub fn validate_terminal_host_record( @@ -3378,6 +3398,8 @@ mod unix { } } +#[cfg(unix)] +pub(crate) use unix::adopt_terminal_host_with_timeout; #[cfg(unix)] pub use unix::{ HostAttachment, adopt_terminal_host, isolate_terminal_host_process_fds, launch_terminal_host, diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 65d952853795..3650b7aa10ca 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1430,9 +1430,11 @@ fn print_empty(_: &Value, _: &mut dyn Write) -> io::Result<()> { fn print_identify(data: &Value, out: &mut dyn Write) -> io::Result<()> { writeln!( out, - "cmux-tui version={} protocol={}", - data.get("version").and_then(Value::as_str).unwrap_or(""), + "cmux-tui session={} protocol={} pid={} version={}", + data.get("session").and_then(Value::as_str).unwrap_or(""), data.get("protocol").and_then(Value::as_u64).unwrap_or(0), + data.get("pid").and_then(Value::as_u64).unwrap_or(0), + data.get("version").and_then(Value::as_str).unwrap_or(""), ) } diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 28858b8859de..6a0ddb877b54 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -62,6 +62,7 @@ pub(crate) struct ServerMessages { pub legacy_peer_mismatch: &'static str, #[cfg(unix)] pub legacy_signal_failed: &'static str, + pub shutdown_failed: &'static str, pub shutdown_unsupported: &'static str, pub shutdown_timed_out: &'static str, } @@ -234,6 +235,7 @@ static ENGLISH: Catalog = Catalog { legacy_peer_mismatch: "The older server's reported process does not own its socket.", #[cfg(unix)] legacy_signal_failed: "The verified older server process could not be stopped.", + shutdown_failed: "the server could not stop cleanly", shutdown_unsupported: "this server cannot be stopped by this client", shutdown_timed_out: "the server did not stop within 10 seconds", }, @@ -359,6 +361,7 @@ static JAPANESE: Catalog = Catalog { legacy_peer_mismatch: "古いサーバーが報告したプロセスはソケットを所有していません。", #[cfg(unix)] legacy_signal_failed: "検証済みの古いサーバープロセスを停止できませんでした。", + shutdown_failed: "サーバーを正常に停止できませんでした", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", }, diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index ba5bfed4b1c6..cca987bef1ab 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -635,6 +635,15 @@ fn main() { } return; } + #[cfg(unix)] + if raw_args.first().map(String::as_str) == Some("__legacy-stop-helper") { + discard_provider_secret_environment(); + if let Err(error) = server_lifecycle::run_legacy_stop_helper(&raw_args[1..]) { + eprintln!("{error}"); + std::process::exit(1); + } + return; + } if let Err(error) = harden_provider_secret_process() { eprintln!("cmux-tui: cannot protect machine-provider credentials: {error}"); std::process::exit(1); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 82fd525f8730..718e3a7d275b 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1,6 +1,8 @@ use std::collections::HashSet; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; +#[cfg(unix)] +use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; @@ -184,23 +186,25 @@ impl ServerLifecycle { Err(_error) if connection_is_gone(&self.path) => return Ok(()), Err(error) => return Err(error), }; - if !accepted { - self.stop_legacy_server()?; + if accepted { + return wait_for_disconnect(&mut self.reader); + } + if self.probe.is_compatible() { + anyhow::bail!(crate::localization::catalog().server.shutdown_failed); } - wait_for_disconnect(&mut self.reader) + self.stop_legacy_server() } #[cfg(unix)] - fn stop_legacy_server(&mut self) -> anyhow::Result<()> { + fn stop_legacy_server(self) -> anyhow::Result<()> { let pid = verified_legacy_pid(self.probe.identity.pid, self.peer_process_id)?; - self.close_legacy_surfaces().map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - terminate_legacy_server(pid) + let result = run_detached_legacy_stop(&self.path, pid); + drop(self.reader); + result } #[cfg(not(unix))] - fn stop_legacy_server(&mut self) -> anyhow::Result<()> { + fn stop_legacy_server(self) -> anyhow::Result<()> { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported) } @@ -240,6 +244,72 @@ impl ServerLifecycle { } } +#[cfg(unix)] +fn run_detached_legacy_stop(path: &Path, expected_pid: libc::pid_t) -> anyhow::Result<()> { + use std::os::unix::process::CommandExt; + + let executable = std::env::current_exe().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let mut command = Command::new(executable); + command + .arg("__legacy-stop-helper") + .arg(path) + .arg(expected_pid.to_string()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let output = command.output().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + if output.status.success() { + return Ok(()); + } + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) +} + +#[cfg(unix)] +pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { + let [path, expected_pid] = args else { + anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); + }; + let expected_pid = + expected_pid.parse::().ok().filter(|pid| *pid > 1).ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) + })?; + let mut lifecycle = ServerLifecycle::connect(PathBuf::from(path))?; + let actual_pid = verified_legacy_pid(lifecycle.probe.identity.pid, lifecycle.peer_process_id)?; + if actual_pid != expected_pid { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + } + + write_json_line( + lifecycle.reader.get_mut(), + &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), + ) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; + let response = read_shutdown_response(&mut lifecycle.reader, SHUTDOWN_REQUEST_ID)?; + if response.get("ok").and_then(Value::as_bool) == Some(true) { + return wait_for_disconnect(&mut lifecycle.reader); + } + if lifecycle.probe.is_compatible() { + anyhow::bail!(crate::localization::catalog().server.shutdown_failed); + } + lifecycle.close_legacy_surfaces().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + terminate_legacy_server(actual_pid)?; + wait_for_disconnect(&mut lifecycle.reader) +} + #[cfg(unix)] fn verified_legacy_pid( reported_pid: u32, diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 74cc4195c4ff..f5f5f3c146c4 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -262,53 +262,73 @@ fn legacy_server_process_helper() { .map(|value| value.parse::().unwrap()) .unwrap_or_else(std::process::id); let listener = transport::listen(&socket).unwrap(); - let mut stream = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone_box().unwrap()); - let mut request = String::new(); - - reader.read_line(&mut request).unwrap(); - let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(identify_request["cmd"].as_str(), Some("identify")); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": identify_request["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "session": "test", - "pid": reported_pid, - "version": "0.0.0-stale", - "protocol": 8, - }, - }) - ) - .unwrap(); + let serve_probe_and_reject = |stream: &mut Box| { + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); + let identify_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(identify_request["cmd"].as_str(), Some("identify")); + let release = cmux_tui_core::release::ReleaseIdentity::current( + cmux_tui_core::server::PROTOCOL_VERSION, + ); + let (version, build_commit, ghostty_commit, protocol) = if scenario == "compatible-failure" + { + (release.version, release.build_commit, release.ghostty_commit, release.protocol) + } else { + ("0.0.0-stale".to_string(), None, None, 8) + }; + writeln!( + stream, + "{}", + serde_json::json!({ + "id": identify_request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": reported_pid, + "version": version, + "build_commit": build_commit, + "ghostty_commit": ghostty_commit, + "protocol": protocol, + }, + }) + ) + .unwrap(); - request.clear(); - reader.read_line(&mut request).unwrap(); - assert_eq!( - serde_json::from_str::(&request).unwrap()["cmd"].as_str(), - Some("shutdown") - ); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": serde_json::Value::Null, - "ok": false, - "error": "bad request", - }) - ) - .unwrap(); + request.clear(); + reader.read_line(&mut request).unwrap(); + assert_eq!( + serde_json::from_str::(&request).unwrap()["cmd"].as_str(), + Some("shutdown") + ); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": serde_json::Value::Null, + "ok": false, + "error": "bad request", + }) + ) + .unwrap(); + }; - request.clear(); - if reader.read_line(&mut request).unwrap() == 0 { + let mut initial_stream = listener.accept().unwrap(); + let caller_pid = initial_stream.peer_process_id().unwrap().unwrap(); + serve_probe_and_reject(&mut initial_stream); + drop(initial_stream); + if matches!(scenario.as_str(), "spoofed-pid" | "compatible-failure") { loop { std::thread::park(); } } + + let mut stream = listener.accept().unwrap(); + serve_probe_and_reject(&mut stream); + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + reader.read_line(&mut request).unwrap(); let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); if scenario == "cleanup-failure" { @@ -326,7 +346,7 @@ fn legacy_server_process_helper() { std::thread::park(); } } - assert_eq!(scenario, "success"); + assert!(matches!(scenario.as_str(), "success" | "kill-caller")); writeln!( stream, "{}", @@ -362,6 +382,10 @@ fn legacy_server_process_helper() { }) ) .unwrap(); + if scenario == "kill-caller" && expected_surface == 41 { + let caller_pid = libc::pid_t::try_from(caller_pid).unwrap(); + assert_eq!(unsafe { libc::kill(caller_pid, libc::SIGKILL) }, 0); + } } loop { std::thread::park(); @@ -782,6 +806,54 @@ fn server_status_and_stop_control_a_compatible_headless_session() { assert!(!server.socket.exists()); } +#[cfg(unix)] +#[test] +fn server_stop_drains_many_hosted_panes_before_acknowledging() { + let mut server = HeadlessServer::start("server-stop-many-panes"); + let applied = cli( + &server, + &[ + "--json", + "apply-layout", + "--layout", + r#"{"type":"stack","panes":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"expanded":1}"#, + ], + ); + assert_success(&applied); + let applied: serde_json::Value = serde_json::from_slice(&applied.stdout).unwrap(); + let surfaces = applied["panes"].as_array().unwrap(); + assert_eq!(surfaces.len(), 32); + let first_surface = surfaces[0]["surface"].as_u64().unwrap(); + let info = cli(&server, &["--json", "process-info", "--surface", &first_surface.to_string()]); + assert_success(&info); + let first_terminal_pid = u32::try_from( + serde_json::from_slice::(&info.stdout).unwrap()["pid"].as_u64().unwrap(), + ) + .unwrap(); + let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); + let host_pids = terminal_host_pids(&host_root); + assert_eq!(host_pids.len(), 32); + + let stop_started = Instant::now(); + let stop = cli(&server, &["--json", "server", "stop"]); + + assert_success(&stop); + assert!(stop_started.elapsed() < Duration::from_secs(5)); + assert!(server.child.wait().unwrap().success()); + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline + && (host_pids.iter().copied().any(process_exists) + || process_exists(first_terminal_pid) + || process_group_exists(first_terminal_pid)) + { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!host_pids.iter().copied().any(process_exists)); + assert!(!process_exists(first_terminal_pid)); + assert!(!process_group_exists(first_terminal_pid)); + assert!(!server.socket.exists()); +} + #[cfg(unix)] #[test] fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { @@ -799,6 +871,41 @@ fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { assert_eq!(status.signal(), Some(libc::SIGTERM)); } +#[cfg(unix)] +#[test] +fn detached_legacy_stop_survives_the_calling_pane_process_exit() { + let mut server = LegacyServerProcess::start("legacy-server-detached", "kill-caller", None); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.signal(), Some(libc::SIGKILL)); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGTERM)); +} + +#[cfg(unix)] +#[test] +fn server_stop_does_not_force_kill_a_compatible_server_after_shutdown_failure() { + let mut server = + LegacyServerProcess::start("server-shutdown-failure", "compatible-failure", None); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&output.stderr).contains("could not stop cleanly")); + assert!(server.child.try_wait().unwrap().is_none()); +} + #[cfg(unix)] #[test] fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { @@ -875,7 +982,10 @@ fn cli_verbs_cover_command_output_errors_and_streams() { let identify = cli(&server, &["identify"]); assert_success(&identify); - assert!(String::from_utf8_lossy(&identify.stdout).starts_with("cmux-tui version=")); + let identify = String::from_utf8(identify.stdout).unwrap(); + assert!(identify.starts_with("cmux-tui session=main protocol=")); + assert!(identify.contains(" pid=")); + assert!(identify.contains(" version=")); let identify_json = cli(&server, &["--json", "identify"]); assert_success(&identify_json); diff --git a/cmux-tui/docs/protocol.md b/cmux-tui/docs/protocol.md index f3e02676ab2d..bb299d87c267 100644 --- a/cmux-tui/docs/protocol.md +++ b/cmux-tui/docs/protocol.md @@ -122,7 +122,7 @@ When the stream ends, it sends: Direct local clients require the server's distribution version, source build, terminal-engine build, and protocol to match exactly. Ordinary source builds derive both build identities automatically, including a deterministic fingerprint when the source tree is dirty. `identify`, `cmux-tui server status`, and `cmux-tui server stop` bypass that check so an upgraded client can inspect and stop an older server. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. -When an older local server lacks orderly shutdown, the replacement client verifies from the Unix socket that the reported server PID is the connected peer. It refuses to signal an unverified process. +Current servers fence new surface creation, wait for in-flight creation, tombstone hosted terminals in one registry transaction, clear topology in one state mutation, and terminate pane runtimes before acknowledging shutdown. When an older local server lacks orderly shutdown, the replacement client starts a detached cleanup helper. The helper reconnects, verifies from the Unix socket that the same reported server PID is still the connected peer, closes every surface, and then signals only that verified process. Detaching lets cleanup finish when closing the caller's pane exits the original client. Remote transports require protocol v9 but do not require the same release build. They reject protocol-v8 servers before loading their workspace tree because v8 does not define stack layout nodes or `new-pane`. diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 0fdf510cb513..13bd0e9bed06 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -248,7 +248,7 @@ Example: | status | implemented | | since | protocol 9 additive extension | -Requests an orderly server exit after the success response is queued. The command is accepted only on the local Unix-domain socket. Stopping the mux exits every pane process. +Requests an orderly server exit after the success response is queued. The command is accepted only on the local Unix-domain socket. Stopping the mux exits every pane process. Shutdown first fences new surface creation and waits for in-flight creation. It then tombstones all hosted terminals in one registry transaction, clears the complete topology in one state mutation, and terminates pane runtimes before returning success. Params: none. @@ -261,7 +261,7 @@ Errors: | `shutdown is only available over the local session socket` | Request arrived over WebSocket | | `bad request: ...` | Malformed request envelope | -CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client first verifies that the PID reported by `identify` owns the connected Unix socket. It then closes every surface reported by `list-workspaces` before signaling that verified process. If socket ownership cannot be verified or orderly pane cleanup fails, the client leaves the server running and reports the failure. +CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client verifies that the PID reported by `identify` owns the connected Unix socket, then starts a detached helper. The helper reconnects, verifies the same PID again, closes every surface reported by `list-workspaces`, and signals that verified process. Cleanup survives the caller's pane exit. If either socket-owner check or orderly pane cleanup fails, the helper leaves the server running and reports the failure. `cmux-tui server status` compares the local server's distribution version, source build, terminal-engine build, and protocol with the client. Plain output reports both versions and protocols plus localized mismatch reasons. JSON output is: From 02d0bb0fbcb1f8d4310eba80e6d2eea9034b31e6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 03:47:50 -0700 Subject: [PATCH 006/521] fix(tui): force-stop verified legacy peers --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 6 +++++- cmux-tui/crates/cmux-tui/tests/cli.rs | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 718e3a7d275b..987404287df0 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -330,7 +330,11 @@ fn verified_legacy_pid( #[cfg(unix)] fn terminate_legacy_server(pid: libc::pid_t) -> anyhow::Result<()> { - if unsafe { libc::kill(pid, libc::SIGTERM) } != 0 { + // Older interactive clients can install a SIGTERM handler while blocking + // indefinitely in terminal input, so a graceful signal does not reliably + // stop the verified socket peer. Every surface has already acknowledged + // close above; kill the still-connected, kernel-verified peer directly. + if unsafe { libc::kill(pid, libc::SIGKILL) } != 0 { anyhow::bail!(crate::localization::catalog().server.legacy_signal_failed); } Ok(()) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index f5f5f3c146c4..606de0aee078 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -868,7 +868,7 @@ fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { assert_success(&output); let status = server.child.wait().unwrap(); - assert_eq!(status.signal(), Some(libc::SIGTERM)); + assert_eq!(status.signal(), Some(libc::SIGKILL)); } #[cfg(unix)] @@ -885,7 +885,7 @@ fn detached_legacy_stop_survives_the_calling_pane_process_exit() { assert_eq!(output.status.signal(), Some(libc::SIGKILL)); let status = server.child.wait().unwrap(); - assert_eq!(status.signal(), Some(libc::SIGTERM)); + assert_eq!(status.signal(), Some(libc::SIGKILL)); } #[cfg(unix)] From 94f7d9e480660a15171af657c4587dd248804ed2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 05:06:28 -0700 Subject: [PATCH 007/521] test(tui): cover server upgrade shutdown races --- cmux-tui/crates/cmux-tui/tests/cli.rs | 207 ++++++++++++++++++-------- 1 file changed, 144 insertions(+), 63 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 606de0aee078..470793d1f32e 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -205,6 +205,7 @@ struct LegacyServerProcess { child: Child, socket: PathBuf, dir: PathBuf, + descendant_pid_file: PathBuf, } #[cfg(unix)] @@ -213,18 +214,20 @@ impl LegacyServerProcess { let dir = unique_temp_dir(name); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); + let descendant_pid_file = dir.join("descendant.pid"); let mut command = Command::new(std::env::current_exe().unwrap()); command .args(["--ignored", "--exact", "legacy_server_process_helper"]) .env("CMUX_TUI_TEST_LEGACY_SOCKET", &socket) .env("CMUX_TUI_TEST_LEGACY_SCENARIO", scenario) + .env("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE", &descendant_pid_file) .stdout(Stdio::null()) .stderr(Stdio::piped()); if let Some(reported_pid) = reported_pid { command.env("CMUX_TUI_TEST_LEGACY_REPORTED_PID", reported_pid.to_string()); } let child = command.spawn().unwrap(); - let mut server = Self { child, socket, dir }; + let mut server = Self { child, socket, dir, descendant_pid_file }; let deadline = Instant::now() + Duration::from_secs(15); while Instant::now() < deadline { if server.socket.exists() { @@ -239,6 +242,10 @@ impl LegacyServerProcess { } panic!("legacy server helper did not create socket at {}", server.socket.display()); } + + fn descendant_pid(&self) -> Option { + fs::read_to_string(&self.descendant_pid_file).ok()?.trim().parse().ok() + } } #[cfg(unix)] @@ -246,6 +253,10 @@ impl Drop for LegacyServerProcess { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); + if let Some(pid) = self.descendant_pid().and_then(|pid| libc::pid_t::try_from(pid).ok()) { + // SAFETY: test cleanup targets the PID written by this fixture. + let _ = unsafe { libc::kill(pid, libc::SIGKILL) }; + } let _ = fs::remove_file(&self.socket); let _ = fs::remove_dir_all(&self.dir); } @@ -261,8 +272,18 @@ fn legacy_server_process_helper() { .ok() .map(|value| value.parse::().unwrap()) .unwrap_or_else(std::process::id); + if matches!(scenario.as_str(), "success" | "kill-caller" | "cleanup-failure") { + let descendant = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + fs::write( + std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), + descendant.id().to_string(), + ) + .unwrap(); + drop(descendant); + } let listener = transport::listen(&socket).unwrap(); - let serve_probe_and_reject = |stream: &mut Box| { + let serve_identify = |stream: &mut Box| { let mut reader = BufReader::new(stream.try_clone_box().unwrap()); let mut request = String::new(); reader.read_line(&mut request).unwrap(); @@ -271,12 +292,19 @@ fn legacy_server_process_helper() { let release = cmux_tui_core::release::ReleaseIdentity::current( cmux_tui_core::server::PROTOCOL_VERSION, ); + let modern = + matches!(scenario.as_str(), "compatible-failure" | "mismatched-modern-failure"); let (version, build_commit, ghostty_commit, protocol) = if scenario == "compatible-failure" { (release.version, release.build_commit, release.ghostty_commit, release.protocol) } else { ("0.0.0-stale".to_string(), None, None, 8) }; + let capabilities = if modern { + vec!["server-shutdown-v1"] + } else { + Vec::new() + }; writeln!( stream, "{}", @@ -291,24 +319,26 @@ fn legacy_server_process_helper() { "build_commit": build_commit, "ghostty_commit": ghostty_commit, "protocol": protocol, + "capabilities": capabilities, }, }) ) .unwrap(); + }; - request.clear(); + let reject_shutdown = |stream: &mut Box| { + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); reader.read_line(&mut request).unwrap(); - assert_eq!( - serde_json::from_str::(&request).unwrap()["cmd"].as_str(), - Some("shutdown") - ); + let request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(request["cmd"].as_str(), Some("shutdown")); writeln!( stream, "{}", serde_json::json!({ - "id": serde_json::Value::Null, + "id": request["id"], "ok": false, - "error": "bad request", + "error": "shutdown failed", }) ) .unwrap(); @@ -316,79 +346,95 @@ fn legacy_server_process_helper() { let mut initial_stream = listener.accept().unwrap(); let caller_pid = initial_stream.peer_process_id().unwrap().unwrap(); - serve_probe_and_reject(&mut initial_stream); - drop(initial_stream); - if matches!(scenario.as_str(), "spoofed-pid" | "compatible-failure") { + serve_identify(&mut initial_stream); + if matches!(scenario.as_str(), "compatible-failure" | "mismatched-modern-failure") { + reject_shutdown(&mut initial_stream); loop { std::thread::park(); } } - - let mut stream = listener.accept().unwrap(); - serve_probe_and_reject(&mut stream); - let mut reader = BufReader::new(stream.try_clone_box().unwrap()); - let mut request = String::new(); - reader.read_line(&mut request).unwrap(); - let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); - if scenario == "cleanup-failure" { - writeln!( - stream, - "{}", - serde_json::json!({ - "id": list_request["id"], - "ok": false, - "error": "cleanup unavailable", - }) - ) - .unwrap(); + drop(initial_stream); + if scenario == "spoofed-pid" { loop { std::thread::park(); } } - assert!(matches!(scenario.as_str(), "success" | "kill-caller")); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": list_request["id"], - "ok": true, - "data": { - "workspaces": [{ - "screens": [{ - "panes": [{ - "tabs": [{"surface": 41}, {"surface": 42}], - }], - }], - }], - }, - }) - ) - .unwrap(); - for expected_surface in [41, 42] { + let mut stream = listener.accept().unwrap(); + serve_identify(&mut stream); + let mut reader = BufReader::new(stream.try_clone_box().unwrap()); + let mut request = String::new(); + let mut snapshot_index = 0; + loop { request.clear(); reader.read_line(&mut request).unwrap(); - let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); - assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); - assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); + if scenario == "cleanup-failure" { + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": false, + "error": "cleanup unavailable", + }) + ) + .unwrap(); + loop { + std::thread::park(); + } + } + assert!(matches!(scenario.as_str(), "success" | "kill-caller")); + let surfaces: &[u64] = match snapshot_index { + 0 => &[41], + 1 => &[42], + _ => &[], + }; + let tabs = surfaces + .iter() + .map(|surface| serde_json::json!({"surface": surface})) + .collect::>(); writeln!( stream, "{}", serde_json::json!({ - "id": close_request["id"], + "id": list_request["id"], "ok": true, - "data": {}, + "data": { + "workspaces": [{ + "screens": [{ + "panes": [{ + "tabs": tabs, + }], + }], + }], + }, }) ) .unwrap(); - if scenario == "kill-caller" && expected_surface == 41 { - let caller_pid = libc::pid_t::try_from(caller_pid).unwrap(); - assert_eq!(unsafe { libc::kill(caller_pid, libc::SIGKILL) }, 0); + for &expected_surface in surfaces { + request.clear(); + reader.read_line(&mut request).unwrap(); + let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); + assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": close_request["id"], + "ok": true, + "data": {}, + }) + ) + .unwrap(); + if scenario == "kill-caller" && expected_surface == 41 { + let caller_pid = libc::pid_t::try_from(caller_pid).unwrap(); + assert_eq!(unsafe { libc::kill(caller_pid, libc::SIGKILL) }, 0); + } } - } - loop { - std::thread::park(); + snapshot_index += 1; } } @@ -856,8 +902,9 @@ fn server_stop_drains_many_hosted_panes_before_acknowledging() { #[cfg(unix)] #[test] -fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { +fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { let mut server = LegacyServerProcess::start("legacy-server-stop", "success", None); + let descendant_pid = server.descendant_pid().unwrap(); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) @@ -869,12 +916,18 @@ fn server_stop_falls_back_when_an_older_server_rejects_shutdown() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); } #[cfg(unix)] #[test] fn detached_legacy_stop_survives_the_calling_pane_process_exit() { let mut server = LegacyServerProcess::start("legacy-server-detached", "kill-caller", None); + let descendant_pid = server.descendant_pid().unwrap(); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) @@ -886,6 +939,11 @@ fn detached_legacy_stop_survives_the_calling_pane_process_exit() { assert_eq!(output.status.signal(), Some(libc::SIGKILL)); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); } #[cfg(unix)] @@ -906,11 +964,33 @@ fn server_stop_does_not_force_kill_a_compatible_server_after_shutdown_failure() assert!(server.child.try_wait().unwrap().is_none()); } +#[cfg(unix)] +#[test] +fn server_stop_does_not_force_kill_a_mismatched_modern_server_after_shutdown_failure() { + let mut server = LegacyServerProcess::start( + "mismatched-server-shutdown-failure", + "mismatched-modern-failure", + None, + ); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!(String::from_utf8_lossy(&output.stderr).contains("could not stop cleanly")); + assert!(server.child.try_wait().unwrap().is_none()); +} + #[cfg(unix)] #[test] fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { let mut server = LegacyServerProcess::start("legacy-server-cleanup-failure", "cleanup-failure", None); + let descendant_pid = server.descendant_pid().unwrap(); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) @@ -925,6 +1005,7 @@ fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { .contains("could not close pane processes before stopping the older server") ); assert!(server.child.try_wait().unwrap().is_none()); + assert!(process_exists(descendant_pid)); } #[cfg(unix)] From fea556ee080170ccf1d180262ce1875de8664295 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 05:06:48 -0700 Subject: [PATCH 008/521] fix(tui): make upgrade shutdown verifiable --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 122 ++++--- cmux-tui/crates/cmux-tui-core/src/server.rs | 5 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 19 +- .../crates/cmux-tui/src/server_lifecycle.rs | 139 +++++--- .../src/server_lifecycle/legacy_process.rs | 314 ++++++++++++++++++ cmux-tui/docs/protocol.md | 2 +- cmux-tui/spec/README.md | 2 +- cmux-tui/spec/commands.md | 6 +- 8 files changed, 488 insertions(+), 121 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 949095865974..354d11c457c6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -43,6 +43,7 @@ const WORKSPACE_NAME_MAX_BYTES: usize = 1_024; const PROVIDER_WORKSPACE_AUTHORITY_MIN_BYTES: usize = 32; const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); +const SHUTDOWN_HOST_EXIT_TIMEOUT: Duration = Duration::from_secs(4); const SHUTDOWN_FANOUT_WORKERS: usize = 128; /// An opaque per-mux credential provisioned by the external machine @@ -812,6 +813,7 @@ pub struct Mux { /// its own lock so a new native-browser owner cannot race the shutdown. pub(crate) daemon_handoff_pending: AtomicBool, shutting_down: AtomicBool, + daemon_shutdown_requested: AtomicBool, pub(crate) control_clients: crate::server::ClientRegistry, pairing: PairingBroker, #[cfg(test)] @@ -1036,6 +1038,7 @@ impl Mux { terminal_adoptions: Mutex::new(HashSet::new()), daemon_handoff_pending: AtomicBool::new(false), shutting_down: AtomicBool::new(false), + daemon_shutdown_requested: AtomicBool::new(false), control_clients: crate::server::ClientRegistry::new(), pairing: PairingBroker::new(), #[cfg(test)] @@ -3640,6 +3643,16 @@ impl Mux { self.shutting_down.store(true, Ordering::Release); self.surface_creations.stop_and_wait(); + #[cfg(unix)] + let terminal_host_records = { + let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + match root { + Some(root) => crate::terminal_host_runtime::load_terminal_host_records(&root) + .context("load terminal hosts for server shutdown")?, + None => Vec::new(), + } + }; + let (surfaces, tree_changed) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); let mut registry = self.workspace_registry.lock().unwrap(); @@ -3691,50 +3704,30 @@ impl Mux { self.emit(MuxEvent::TreeChanged); } - #[cfg(unix)] - let terminated_hosts = Mutex::new(HashSet::new()); bounded_shutdown_fanout(&surfaces, |surface| { - let accepted = surface.terminate_for_server_shutdown(SHUTDOWN_TERMINATION_TIMEOUT); - #[cfg(unix)] - if accepted && let Some(identity) = surface.terminal_host_identity() { - terminated_hosts - .lock() - .unwrap() - .insert((identity.terminal_id, identity.incarnation)); - } + surface.terminate_for_server_shutdown(SHUTDOWN_TERMINATION_TIMEOUT); }); #[cfg(unix)] { - let terminated_hosts = terminated_hosts.into_inner().unwrap(); - let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - if let Some(root) = root { - let records = crate::terminal_host_runtime::load_terminal_host_records(&root) - .context("load terminal hosts for server shutdown")?; - let failed = Mutex::new(Vec::new()); - bounded_shutdown_fanout(&records, |(path, record)| { - if terminated_hosts - .contains(&(record.terminal_id.clone(), record.incarnation.clone())) - { - return; - } - if !cleanup_terminal_host_record_with_timeout( - record, - path, - SHUTDOWN_TERMINATION_TIMEOUT, - ) { - failed.lock().unwrap().push(record.terminal_id.clone()); - } - }); - let mut failed = failed.into_inner().unwrap(); - if !failed.is_empty() { - failed.sort(); - anyhow::bail!( - "could not terminate {} terminal host(s): {}", - failed.len(), - failed.join(", ") - ); + let failed = Mutex::new(Vec::new()); + bounded_shutdown_fanout(&terminal_host_records, |(path, record)| { + if !terminate_and_confirm_terminal_host_record( + record, + path, + SHUTDOWN_HOST_EXIT_TIMEOUT, + ) { + failed.lock().unwrap().push(record.terminal_id.clone()); } + }); + let mut failed = failed.into_inner().unwrap(); + if !failed.is_empty() { + failed.sort(); + anyhow::bail!( + "could not terminate {} terminal host(s): {}", + failed.len(), + failed.join(", ") + ); } } @@ -3758,10 +3751,11 @@ impl Mux { pub fn request_daemon_shutdown(&self) { self.shutting_down.store(true, Ordering::Release); self.surface_creations.stop_and_wait(); + self.daemon_shutdown_requested.store(true, Ordering::Release); } pub fn daemon_shutdown_requested(&self) -> bool { - self.shutting_down.load(Ordering::Acquire) + self.daemon_shutdown_requested.load(Ordering::Acquire) } /// Update options used for future surface/browser launches. @@ -7501,25 +7495,47 @@ fn cleanup_terminal_host_record( } #[cfg(unix)] -fn cleanup_terminal_host_record_with_timeout( +fn terminate_and_confirm_terminal_host_record( record: &crate::terminal_host_runtime::TerminalHostRecord, record_path: &Path, timeout: Duration, ) -> bool { - match terminal_host_record_liveness(record_path, record) { - TerminalHostLiveness::Dead => { - match crate::terminal_host_runtime::remove_stale_terminal_host_record( - record_path, - record, - ) { - Ok(removed) => removed, - Err(_) if !record_path.exists() => true, - Err(_) => false, + let deadline = Instant::now() + timeout; + let mut termination_sent = false; + let mut last_terminate_attempt = None; + loop { + match terminal_host_record_liveness(record_path, record) { + TerminalHostLiveness::Dead => { + return match crate::terminal_host_runtime::remove_stale_terminal_host_record( + record_path, + record, + ) { + Ok(removed) => removed, + Err(_) if !record_path.exists() => true, + Err(_) => false, + }; } + TerminalHostLiveness::Live | TerminalHostLiveness::Indeterminate => {} } - TerminalHostLiveness::Live | TerminalHostLiveness::Indeterminate => { - terminate_host_record_with_timeout(record.clone(), record_path.to_path_buf(), timeout) + + let now = Instant::now(); + if !termination_sent + && last_terminate_attempt.is_none_or(|attempt: Instant| { + now.duration_since(attempt) >= Duration::from_millis(100) + }) + { + termination_sent = terminate_host_record_with_timeout( + record.clone(), + record_path.to_path_buf(), + SHUTDOWN_TERMINATION_TIMEOUT, + ); + last_terminate_attempt = Some(Instant::now()); + } + let now = Instant::now(); + if now >= deadline { + return false; } + std::thread::sleep(deadline.saturating_duration_since(now).min(Duration::from_millis(10))); } } @@ -9878,10 +9894,14 @@ mod tests { let error = mux.close_all_surfaces_for_shutdown().unwrap_err(); assert!(format!("{error:#}").contains("forced terminal close failure")); + assert!(!mux.shutdown_requested()); + assert!(!mux.daemon_shutdown_requested()); assert!(mux.surface(surface).is_some()); assert!(mux.with_state(|state| state.pane_of(surface).is_some())); mux.set_terminal_close_failure_for_test(false).unwrap(); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + assert!(!mux.shutdown_requested()); + assert!(!mux.daemon_shutdown_requested()); assert!(mux.surface(surface).is_none()); } diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 51a73d1a120b..1353f6306bee 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -57,6 +57,7 @@ use crate::{ const ATTACH_INITIAL_SIZE_CAPABILITY: &str = "attach-initial-size"; const WORKSPACE_REGISTRY_CAPABILITY: &str = "workspace-registry-v1"; +pub const SERVER_SHUTDOWN_CAPABILITY: &str = "server-shutdown-v1"; pub const PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY: &str = "provider-managed-workspace-authority-v2"; const INITIAL_BROWSER_RESIZE_TIMEOUT: Duration = Duration::from_secs(10); @@ -2944,7 +2945,8 @@ fn handle_command( "capabilities": [ ATTACH_INITIAL_SIZE_CAPABILITY, WORKSPACE_REGISTRY_CAPABILITY, - PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY + PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, + SERVER_SHUTDOWN_CAPABILITY ], "session": mux.session, "pid": std::process::id(), @@ -6477,6 +6479,7 @@ mod tests { "attach-initial-size", "workspace-registry-v1", PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, + SERVER_SHUTDOWN_CAPABILITY, ] { assert!(capabilities.iter().any(|value| value.as_str() == Some(expected))); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 7a4d24f2e9f6..9b475b31fec5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2092,28 +2092,29 @@ impl Surface { } } - /// Terminate the runtime while reporting whether an attached terminal - /// host accepted the request. Callers can retry a failed hosted request - /// through its durable discovery record after removing topology. - pub(crate) fn terminate_for_server_shutdown(&self, timeout: Duration) -> bool { + /// Start runtime termination. Durable terminal-host death is confirmed + /// separately through the process-bound discovery record. + pub(crate) fn terminate_for_server_shutdown(&self, timeout: Duration) { match self { #[cfg(unix)] Surface::Pty(pty) => { let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { - PtyRuntime::Hosted(host) => host.terminate_with_timeout(timeout).is_ok(), + PtyRuntime::Hosted(host) => { + let _ = host.terminate_with_timeout(timeout); + } PtyRuntime::Local { killer, .. } => { let _ = killer.kill(); - true } - PtyRuntime::ExitedHosted => true, + PtyRuntime::ExitedHosted => {} } } #[cfg(not(unix))] - Surface::Pty(_) => self.terminate_runtime(), + Surface::Pty(_) => { + let _ = self.terminate_runtime(); + } Surface::Browser(browser) => { browser.kill(); - true } } } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 987404287df0..1c1ba4d25383 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -7,15 +7,22 @@ use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; use cmux_tui_core::release::ReleaseIdentity; -use cmux_tui_core::server::PROTOCOL_VERSION; +use cmux_tui_core::server::{PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY}; use serde_json::{Value, json}; +#[cfg(unix)] +mod legacy_process; +#[cfg(unix)] +use legacy_process::{ProcessIdentity, terminate_process_tree}; + const PROBE_REQUEST_ID: u64 = 0; const SHUTDOWN_REQUEST_ID: u64 = 1; #[cfg(unix)] const LEGACY_LIST_REQUEST_ID: u64 = 2; #[cfg(unix)] -const LEGACY_CLOSE_REQUEST_ID_START: u64 = 3; +const LEGACY_STABLE_EMPTY_SCANS: usize = 2; +#[cfg(unix)] +const LEGACY_MAX_SCAN_ROUNDS: usize = 64; const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); type TransportReader = BufReader>; @@ -175,6 +182,10 @@ impl ServerLifecycle { } pub(crate) fn stop(mut self) -> anyhow::Result<()> { + if !self.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { + return self.stop_legacy_server(); + } + write_json_line( self.reader.get_mut(), &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), @@ -189,16 +200,13 @@ impl ServerLifecycle { if accepted { return wait_for_disconnect(&mut self.reader); } - if self.probe.is_compatible() { - anyhow::bail!(crate::localization::catalog().server.shutdown_failed); - } - self.stop_legacy_server() + anyhow::bail!(crate::localization::catalog().server.shutdown_failed) } #[cfg(unix)] fn stop_legacy_server(self) -> anyhow::Result<()> { - let pid = verified_legacy_pid(self.probe.identity.pid, self.peer_process_id)?; - let result = run_detached_legacy_stop(&self.path, pid); + let process = verified_legacy_process(self.probe.identity.pid, self.peer_process_id)?; + let result = run_detached_legacy_stop(&self.path, process); drop(self.reader); result } @@ -209,23 +217,42 @@ impl ServerLifecycle { } #[cfg(unix)] - fn close_legacy_surfaces(&mut self) -> anyhow::Result<()> { + fn close_legacy_surfaces_until_stable(&mut self) -> anyhow::Result<()> { + let mut next_request_id = LEGACY_LIST_REQUEST_ID; + let mut consecutive_empty_scans = 0; + for _ in 0..LEGACY_MAX_SCAN_ROUNDS { + let closed = self.close_legacy_surface_snapshot(&mut next_request_id)?; + if closed == 0 { + consecutive_empty_scans += 1; + if consecutive_empty_scans == LEGACY_STABLE_EMPTY_SCANS { + return Ok(()); + } + } else { + consecutive_empty_scans = 0; + } + } + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) + } + + #[cfg(unix)] + fn close_legacy_surface_snapshot( + &mut self, + next_request_id: &mut u64, + ) -> anyhow::Result { + let list_request_id = take_legacy_request_id(next_request_id)?; write_json_line( self.reader.get_mut(), - &json!({"id": LEGACY_LIST_REQUEST_ID, "cmd": "list-workspaces"}), + &json!({"id": list_request_id, "cmd": "list-workspaces"}), ) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_response(&mut self.reader, LEGACY_LIST_REQUEST_ID)?; + let response = read_response(&mut self.reader, list_request_id)?; let data = response_data(&response)?; + let mut surfaces = legacy_surface_ids(data); + surfaces.sort_unstable(); + surfaces.dedup(); - for (index, surface) in legacy_surface_ids(data).into_iter().enumerate() { - let request_id = LEGACY_CLOSE_REQUEST_ID_START - .checked_add(u64::try_from(index).map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_too_many_surfaces) - })?) - .ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_too_many_surfaces) - })?; + for &surface in &surfaces { + let request_id = take_legacy_request_id(next_request_id)?; write_json_line( self.reader.get_mut(), &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), @@ -240,12 +267,21 @@ impl ServerLifecycle { } } } - Ok(()) + Ok(surfaces.len()) } } #[cfg(unix)] -fn run_detached_legacy_stop(path: &Path, expected_pid: libc::pid_t) -> anyhow::Result<()> { +fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { + let request_id = *next_request_id; + *next_request_id = request_id.checked_add(1).ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_too_many_surfaces) + })?; + Ok(request_id) +} + +#[cfg(unix)] +fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::Result<()> { use std::os::unix::process::CommandExt; let executable = std::env::current_exe().map_err(|_| { @@ -255,7 +291,8 @@ fn run_detached_legacy_stop(path: &Path, expected_pid: libc::pid_t) -> anyhow::R command .arg("__legacy-stop-helper") .arg(path) - .arg(expected_pid.to_string()) + .arg(expected.pid().to_string()) + .arg(expected.started_at().to_string()) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::piped()); @@ -278,43 +315,38 @@ fn run_detached_legacy_stop(path: &Path, expected_pid: libc::pid_t) -> anyhow::R #[cfg(unix)] pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { - let [path, expected_pid] = args else { + let [path, expected_pid, expected_started_at] = args else { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); }; let expected_pid = expected_pid.parse::().ok().filter(|pid| *pid > 1).ok_or_else(|| { anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) })?; + let expected_started_at = expected_started_at + .parse::() + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; + let expected = ProcessIdentity::from_parts(expected_pid, expected_started_at); let mut lifecycle = ServerLifecycle::connect(PathBuf::from(path))?; - let actual_pid = verified_legacy_pid(lifecycle.probe.identity.pid, lifecycle.peer_process_id)?; - if actual_pid != expected_pid { + if lifecycle.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - - write_json_line( - lifecycle.reader.get_mut(), - &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), - ) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_shutdown_response(&mut lifecycle.reader, SHUTDOWN_REQUEST_ID)?; - if response.get("ok").and_then(Value::as_bool) == Some(true) { - return wait_for_disconnect(&mut lifecycle.reader); - } - if lifecycle.probe.is_compatible() { - anyhow::bail!(crate::localization::catalog().server.shutdown_failed); + let actual = verified_legacy_process(lifecycle.probe.identity.pid, lifecycle.peer_process_id)?; + if actual != expected { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - lifecycle.close_legacy_surfaces().map_err(|_| { + + lifecycle.close_legacy_surfaces_until_stable().map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - terminate_legacy_server(actual_pid)?; + terminate_legacy_process_tree(actual)?; wait_for_disconnect(&mut lifecycle.reader) } #[cfg(unix)] -fn verified_legacy_pid( +fn verified_legacy_process( reported_pid: u32, peer_process_id: Option, -) -> anyhow::Result { +) -> anyhow::Result { let messages = &crate::localization::catalog().server; let peer_process_id = peer_process_id.ok_or_else(|| anyhow::anyhow!(messages.legacy_peer_unavailable))?; @@ -322,22 +354,19 @@ fn verified_legacy_pid( anyhow::bail!(messages.legacy_peer_mismatch); } let current_pid = libc::pid_t::try_from(std::process::id()).ok(); - libc::pid_t::try_from(reported_pid) + let pid = libc::pid_t::try_from(reported_pid) .ok() .filter(|pid| *pid > 1 && Some(*pid) != current_pid) - .ok_or_else(|| anyhow::anyhow!(messages.shutdown_unsupported)) + .ok_or_else(|| anyhow::anyhow!(messages.shutdown_unsupported))?; + ProcessIdentity::capture(pid) + .map_err(|_| anyhow::anyhow!(messages.legacy_peer_unavailable))? + .ok_or_else(|| anyhow::anyhow!(messages.legacy_peer_mismatch)) } #[cfg(unix)] -fn terminate_legacy_server(pid: libc::pid_t) -> anyhow::Result<()> { - // Older interactive clients can install a SIGTERM handler while blocking - // indefinitely in terminal input, so a graceful signal does not reliably - // stop the verified socket peer. Every surface has already acknowledged - // close above; kill the still-connected, kernel-verified peer directly. - if unsafe { libc::kill(pid, libc::SIGKILL) } != 0 { - anyhow::bail!(crate::localization::catalog().server.legacy_signal_failed); - } - Ok(()) +fn terminate_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result<()> { + terminate_process_tree(process) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) } pub(crate) fn validate_local_identity(data: &Value, path: &Path) -> anyhow::Result<()> { @@ -529,8 +558,8 @@ mod tests { let mut child = Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); - let pid = verified_legacy_pid(child.id(), Some(child.id())).unwrap(); - terminate_legacy_server(pid).unwrap(); + let process = verified_legacy_process(child.id(), Some(child.id())).unwrap(); + terminate_legacy_process_tree(process).unwrap(); assert!(!child.wait().unwrap().success()); } @@ -538,7 +567,7 @@ mod tests { #[cfg(unix)] #[test] fn legacy_stop_rejects_a_reported_pid_that_is_not_the_socket_peer() { - let error = verified_legacy_pid(41, Some(42)).unwrap_err(); + let error = verified_legacy_process(41, Some(42)).unwrap_err(); assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_peer_mismatch); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs new file mode 100644 index 000000000000..2a8d44310cc3 --- /dev/null +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -0,0 +1,314 @@ +use std::collections::HashSet; +use std::io; + +const PROCESS_TREE_MAX_ROUNDS: usize = 64; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ProcessIdentity { + pid: libc::pid_t, + started_at: u128, +} + +impl ProcessIdentity { + pub(super) fn from_parts(pid: libc::pid_t, started_at: u128) -> Self { + Self { pid, started_at } + } + + pub(super) fn pid(self) -> libc::pid_t { + self.pid + } + + pub(super) fn started_at(self) -> u128 { + self.started_at + } + + pub(super) fn capture(pid: libc::pid_t) -> io::Result> { + process_snapshot(pid).map(|snapshot| snapshot.map(|snapshot| snapshot.identity)) + } + + fn signal(self, signal: libc::c_int) -> io::Result { + let Some(current) = Self::capture(self.pid)? else { + return Ok(ExactSignalResult::Gone); + }; + if current != self { + return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); + } + // SAFETY: the PID was range-checked and its birth identity was + // revalidated immediately before this signal. + if unsafe { libc::kill(self.pid, signal) } == 0 { + return Ok(ExactSignalResult::Signaled); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(ExactSignalResult::Gone) + } else { + Err(error) + } + } +} + +#[derive(Clone, Copy, Debug)] +struct ProcessSnapshot { + identity: ProcessIdentity, + parent: libc::pid_t, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ExactSignalResult { + Signaled, + Gone, +} + +pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { + FrozenProcessTree::freeze(process).and_then(FrozenProcessTree::kill) +} + +struct FrozenProcessTree { + root: ProcessIdentity, + descendants: Vec, + armed: bool, +} + +impl FrozenProcessTree { + fn freeze(root: ProcessIdentity) -> io::Result { + let mut tree = Self { root, descendants: Vec::new(), armed: true }; + if root.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "server exited before its process tree was fenced", + )); + } + if ProcessIdentity::capture(root.pid)? != Some(root) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "server process identity changed after fencing", + )); + } + + let helper_pid = libc::pid_t::try_from(std::process::id()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid helper process id"))?; + let mut known = HashSet::from([root.pid, helper_pid]); + for _ in 0..PROCESS_TREE_MAX_ROUNDS { + let parents = + std::iter::once(root).chain(tree.descendants.iter().copied()).collect::>(); + let mut added = false; + for parent in parents { + for pid in direct_child_pids(parent.pid)? { + if known.contains(&pid) { + continue; + } + let Some(snapshot) = process_snapshot(pid)? else { + continue; + }; + if snapshot.parent != parent.pid { + continue; + } + match snapshot.identity.signal(libc::SIGSTOP)? { + ExactSignalResult::Gone => continue, + ExactSignalResult::Signaled => {} + } + if ProcessIdentity::capture(pid)? != Some(snapshot.identity) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "descendant process identity changed after fencing", + )); + } + known.insert(pid); + tree.descendants.push(snapshot.identity); + added = true; + } + } + if !added { + return Ok(tree); + } + } + Err(io::Error::other("legacy process tree did not stabilize")) + } + + fn kill(mut self) -> io::Result<()> { + if !self.armed { + return Ok(()); + } + for process in std::iter::once(self.root).chain(self.descendants.iter().copied()) { + if let Some(current) = ProcessIdentity::capture(process.pid)? + && current != process + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "frozen process identity changed", + )); + } + } + for process in self.descendants.iter().rev().copied() { + let _ = process.signal(libc::SIGKILL)?; + } + let _ = self.root.signal(libc::SIGKILL)?; + self.armed = false; + Ok(()) + } +} + +impl Drop for FrozenProcessTree { + fn drop(&mut self) { + if !self.armed { + return; + } + for process in self.descendants.iter().rev().copied() { + let _ = process.signal(libc::SIGCONT); + } + let _ = self.root.signal(libc::SIGCONT); + } +} + +#[cfg(target_os = "macos")] +fn process_snapshot(pid: libc::pid_t) -> io::Result> { + use std::mem::{size_of, zeroed}; + + let mut info = unsafe { zeroed::() }; + let expected = i32::try_from(size_of::()) + .map_err(|_| io::Error::other("process metadata size overflow"))?; + // SAFETY: `info` is a writable buffer of exactly `expected` bytes. + let result = unsafe { + libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), expected) + }; + if result != expected { + // SAFETY: signal zero does not deliver a signal. + if unsafe { libc::kill(pid, 0) } != 0 + && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + return Ok(None); + } + return Err(io::Error::other("could not read process birth identity")); + } + let info_pid = + libc::pid_t::try_from(info.pbi_pid).map_err(|_| io::Error::other("invalid process id"))?; + if info_pid != pid { + return Err(io::Error::other("process metadata id mismatch")); + } + let parent = libc::pid_t::try_from(info.pbi_ppid) + .map_err(|_| io::Error::other("invalid parent process id"))?; + let started_at = (u128::from(info.pbi_start_tvsec) << 64) | u128::from(info.pbi_start_tvusec); + Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent })) +} + +#[cfg(target_os = "macos")] +fn direct_child_pids(parent: libc::pid_t) -> io::Result> { + use std::ffi::c_void; + use std::mem::size_of; + + // SAFETY: a null buffer with length zero asks libproc for the count. + let count = unsafe { libc::proc_listchildpids(parent, std::ptr::null_mut(), 0) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + if count == 0 { + return Ok(Vec::new()); + } + let mut capacity = usize::try_from(count) + .map_err(|_| io::Error::other("invalid child process count"))? + .saturating_add(16); + for _ in 0..4 { + let mut pids = vec![0; capacity]; + let bytes = capacity + .checked_mul(size_of::()) + .and_then(|bytes| i32::try_from(bytes).ok()) + .ok_or_else(|| io::Error::other("child process buffer overflow"))?; + // SAFETY: `pids` owns a writable buffer of `bytes` bytes. + let listed = + unsafe { libc::proc_listchildpids(parent, pids.as_mut_ptr().cast::(), bytes) }; + if listed < 0 { + return Err(io::Error::last_os_error()); + } + let listed = + usize::try_from(listed).map_err(|_| io::Error::other("invalid child process count"))?; + if listed < capacity { + pids.truncate(listed); + pids.retain(|pid| *pid > 1); + return Ok(pids); + } + capacity = capacity + .checked_mul(2) + .ok_or_else(|| io::Error::other("child process buffer overflow"))?; + } + Err(io::Error::other("child process list did not stabilize")) +} + +#[cfg(target_os = "linux")] +fn process_snapshot(pid: libc::pid_t) -> io::Result> { + let path = format!("/proc/{pid}/stat"); + let stat = match std::fs::read_to_string(path) { + Ok(stat) => stat, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let (pid_text, remainder) = stat + .split_once(" (") + .and_then(|(pid_text, remainder)| { + remainder.rsplit_once(") ").map(|(_, fields)| (pid_text, fields)) + }) + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + if pid_text.parse::().ok() != Some(pid) { + return Err(io::Error::other("process metadata id mismatch")); + } + let fields = remainder.split_whitespace().collect::>(); + let parent = fields + .get(1) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| io::Error::other("invalid parent process id"))?; + let started_at = fields + .get(19) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| io::Error::other("invalid process birth identity"))?; + Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent })) +} + +#[cfg(target_os = "linux")] +fn direct_child_pids(parent: libc::pid_t) -> io::Result> { + let path = format!("/proc/{parent}/task/{parent}/children"); + let children = match std::fs::read_to_string(path) { + Ok(children) => children, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut pids = children + .split_whitespace() + .map(|pid| { + pid.parse::().map_err(|_| io::Error::other("invalid child process id")) + }) + .collect::>>()?; + pids.retain(|pid| *pid > 1); + Ok(pids) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn process_snapshot(_pid: libc::pid_t) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, "stable process identity is unavailable")) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn direct_child_pids(_parent: libc::pid_t) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, "process tree enumeration is unavailable")) +} + +#[cfg(test)] +mod tests { + use std::process::{Command, Stdio}; + + use super::*; + + #[test] + fn stable_identity_refuses_to_signal_the_same_pid_with_a_different_birth() { + let mut child = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let process = + ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); + let stale = + ProcessIdentity { pid: process.pid, started_at: process.started_at.wrapping_add(1) }; + + assert!(stale.signal(libc::SIGKILL).is_err()); + assert!(child.try_wait().unwrap().is_none()); + + child.kill().unwrap(); + child.wait().unwrap(); + } +} diff --git a/cmux-tui/docs/protocol.md b/cmux-tui/docs/protocol.md index bb299d87c267..888e74c871ff 100644 --- a/cmux-tui/docs/protocol.md +++ b/cmux-tui/docs/protocol.md @@ -16,7 +16,7 @@ $TMPDIR/cmux-tui-/.sock ```json {"id":1,"cmd":"identify"} -{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"...","protocol":9,"capabilities":["attach-initial-size","workspace-registry-v1","provider-managed-workspace-authority-v2"],"session":"main","pid":12345}} +{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"...","protocol":9,"capabilities":["attach-initial-size","workspace-registry-v1","provider-managed-workspace-authority-v2","server-shutdown-v1"],"session":"main","pid":12345}} ``` Responses have this shape: diff --git a/cmux-tui/spec/README.md b/cmux-tui/spec/README.md index 1abcdb953171..30e96cb4ad9e 100644 --- a/cmux-tui/spec/README.md +++ b/cmux-tui/spec/README.md @@ -22,7 +22,7 @@ Protocol v7 is additive for v6 clients: `attach-surface.mode` defaults to `"byte Generated clients must inspect `identify.protocol` before using features newer than the connected server. Bindings may expose proposed APIs behind version checks, but they must not send proposed commands to an older server unless the caller explicitly opts into probing. -`identify.capabilities` negotiates additive build-level features within one protocol version. Clients must treat a missing capability list as empty. They must require `attach-initial-size` before sending initial `cols` or `rows` on `attach-surface`, `workspace-registry-v1` before using registry creation, placement, stable-key, or revision-CAS APIs, and `provider-managed-workspace-authority-v2` before committing provider-owned workspace mirrors with a pre-provisioned authority. +`identify.capabilities` negotiates additive build-level features within one protocol version. Clients must treat a missing capability list as empty. They must require `attach-initial-size` before sending initial `cols` or `rows` on `attach-surface`, `workspace-registry-v1` before using registry creation, placement, stable-key, or revision-CAS APIs, `provider-managed-workspace-authority-v2` before committing provider-owned workspace mirrors with a pre-provisioned authority, and `server-shutdown-v1` before assuming `shutdown` is implemented. ## Generation Model diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 13bd0e9bed06..acd6f47e4c41 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -182,7 +182,7 @@ object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:s `build_commit` and `ghostty_commit` are additive build-stamp fields. Current source builds derive them from the repository and terminal-engine checkout. A dirty source build appends a deterministic content fingerprint. Distribution builds may override both stamps explicitly. Older binaries can omit the fields or return `null`, so clients must preserve compatibility with those identities. -`capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. +`capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. `server-shutdown-v1` advertises atomic destructive shutdown; clients may use the compatibility cleanup path only when this capability is absent. Errors: @@ -196,7 +196,7 @@ CLI mapping: | --- | --- | | Verb | `identify` | | Flags | none | -| Plain stdout | `cmux-tui version= protocol=` | +| Plain stdout | `cmux-tui session= protocol= pid= version=` | | JSON stdout | exact result object | | Exit codes | common | @@ -204,7 +204,7 @@ Example: ```json {"id":1,"cmd":"identify"} -{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"0.1.0","build_commit":"abc123","ghostty_commit":"def456","protocol":9,"capabilities":["attach-initial-size","workspace-registry-v1","provider-managed-workspace-authority-v2"],"session":"main","pid":12345}} +{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"0.1.0","build_commit":"abc123","ghostty_commit":"def456","protocol":9,"capabilities":["attach-initial-size","workspace-registry-v1","provider-managed-workspace-authority-v2","server-shutdown-v1"],"session":"main","pid":12345}} ``` The current server reports protocol `9` in this field and in `ping`. Clients must negotiate protocol 8 before requiring stable split ids or sending `set-split-ratio`, and protocol 9 before decoding stack layouts or sending `new-pane`. From f34263d34fbcda5e94b7d550e9fc4bfb70461753 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 05:41:27 -0700 Subject: [PATCH 009/521] test(tui): cover bounded server replacement --- .../src/server_lifecycle/legacy_process.rs | 27 +++ cmux-tui/crates/cmux-tui/tests/cli.rs | 216 ++++++++++++++++-- 2 files changed, 228 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 2a8d44310cc3..3a0643a01f79 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -293,6 +293,8 @@ fn direct_child_pids(_parent: libc::pid_t) -> io::Result> { #[cfg(test)] mod tests { use std::process::{Command, Stdio}; + #[cfg(target_os = "linux")] + use std::sync::mpsc; use super::*; @@ -311,4 +313,29 @@ mod tests { child.kill().unwrap(); child.wait().unwrap(); } + + #[cfg(target_os = "linux")] + #[test] + fn linux_child_enumeration_includes_children_spawned_by_worker_threads() { + let (child_tx, child_rx) = mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + let child = + Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + child_tx.send(child.id()).unwrap(); + child + }); + let child_pid = child_rx.recv().unwrap(); + let parent = libc::pid_t::try_from(std::process::id()).unwrap(); + let child_pid = libc::pid_t::try_from(child_pid).unwrap(); + + let children = direct_child_pids(parent).unwrap(); + + let mut child = worker.join().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + assert!( + children.contains(&child_pid), + "worker child {child_pid} missing from {children:?}" + ); + } } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 470793d1f32e..926ab554bcbc 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -7,7 +7,7 @@ use std::os::fd::FromRawFd; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; #[cfg(unix)] -use std::os::unix::process::ExitStatusExt; +use std::os::unix::process::{CommandExt, ExitStatusExt}; use std::path::PathBuf; use std::process::{Child, Command, Output, Stdio}; use std::sync::mpsc; @@ -24,19 +24,25 @@ struct HeadlessServer { impl HeadlessServer { fn start(name: &str) -> Self { + Self::start_with(name, false, &[]) + } + + fn start_with(name: &str, ephemeral: bool, environment: &[(&str, &str)]) -> Self { let dir = unique_temp_dir(name); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let state = dir.join("state"); - let child = Command::new(bin()) - .args(["--headless", "--socket"]) - .arg(&socket) - .arg("--state") - .arg(&state) - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .spawn() - .unwrap(); + let mut command = Command::new(bin()); + command.args(["--headless", "--socket"]).arg(&socket); + if ephemeral { + command.arg("--ephemeral"); + } else { + command.arg("--state").arg(&state); + } + for (name, value) in environment { + command.env(name, value); + } + let child = command.stdout(Stdio::null()).stderr(Stdio::piped()).spawn().unwrap(); let server = Self { child, socket, state, dir }; server.wait_for_socket(); server @@ -200,6 +206,20 @@ fn process_group_exists(_pid: u32) -> bool { false } +#[cfg(unix)] +fn wait_for_pid_file(path: &std::path::Path, timeout: Duration) -> u32 { + let deadline = Instant::now() + timeout; + loop { + if let Some(pid) = + fs::read_to_string(path).ok().and_then(|value| value.trim().parse::().ok()) + { + return pid; + } + assert!(Instant::now() < deadline, "PID file was not ready: {}", path.display()); + std::thread::sleep(Duration::from_millis(10)); + } +} + #[cfg(unix)] struct LegacyServerProcess { child: Child, @@ -300,11 +320,7 @@ fn legacy_server_process_helper() { } else { ("0.0.0-stale".to_string(), None, None, 8) }; - let capabilities = if modern { - vec!["server-shutdown-v1"] - } else { - Vec::new() - }; + let capabilities = if modern { vec!["server-shutdown-v1"] } else { Vec::new() }; writeln!( stream, "{}", @@ -828,6 +844,49 @@ fn incompatible_server_allows_identity_and_status_but_rejects_ping() { let _ = fs::remove_dir_all(&dir); } +#[cfg(unix)] +#[test] +fn packaged_launcher_name_is_used_in_upgrade_instructions() { + let dir = unique_temp_dir("packaged-launcher-message"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = transport::listen(&socket).unwrap(); + let server = std::thread::spawn(move || { + let mut stream = listener.accept().unwrap(); + let mut request = String::new(); + BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); + let request: serde_json::Value = serde_json::from_str(&request).unwrap(); + let response = serde_json::json!({ + "id": request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": 4242, + "version": "0.0.0-stale", + "protocol": cmux_tui_core::server::PROTOCOL_VERSION, + }, + }); + writeln!(stream, "{response}").unwrap(); + }); + + let output = Command::new(bin()) + .arg0("cmux") + .args(["ping", "--socket"]) + .arg(&socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("cmux server stop"), "{stderr}"); + assert!(stderr.contains("then run `cmux` again"), "{stderr}"); + server.join().unwrap(); + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); +} + #[test] fn server_status_and_stop_control_a_compatible_headless_session() { let mut server = HeadlessServer::start("server-lifecycle"); @@ -852,6 +911,133 @@ fn server_status_and_stop_control_a_compatible_headless_session() { assert!(!server.socket.exists()); } +#[cfg(unix)] +#[test] +fn server_stop_cancels_a_blocked_terminal_host_launch() { + let mut server = HeadlessServer::start_with( + "server-stop-blocked-launch", + false, + &[("CMUX_TUI_TEST_BOOTSTRAP_READY_DELAY_MS", "3000")], + ); + let mut create = Command::new(bin()) + .args(["--socket"]) + .arg(&server.socket) + .arg("new-workspace") + .env_remove("CMUX_TUI_SOCKET") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); + std::thread::sleep(Duration::from_millis(250)); + assert!(create.try_wait().unwrap().is_none(), "terminal creation did not remain in flight"); + + let stop_started = Instant::now(); + let stop = cli(&server, &["--json", "server", "stop"]); + + assert_success(&stop); + assert!( + stop_started.elapsed() < Duration::from_secs(2), + "shutdown waited for the blocked creator: {:?}", + stop_started.elapsed() + ); + assert!(server.child.wait().unwrap().success()); + assert!(!create.wait().unwrap().success()); + assert!(terminal_host_pids(&host_root).is_empty()); +} + +#[cfg(unix)] +#[test] +fn cancelled_published_host_is_terminated_through_its_record() { + let mut server = HeadlessServer::start_with( + "server-stop-published-launch", + false, + &[("CMUX_TUI_TEST_HOST_READY_DELAY_MS", "1000")], + ); + let direct_pid_file = server.dir.join("published-direct.pid"); + let descendant_pid_file = server.dir.join("published-descendant.pid"); + let command = format!( + "echo $$ > {}; trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", + direct_pid_file.display(), + descendant_pid_file.display() + ); + let mut create = Command::new(bin()) + .args(["--socket"]) + .arg(&server.socket) + .args(["run", "--new-workspace", "--command", &command]) + .env_remove("CMUX_TUI_SOCKET") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); + let deadline = Instant::now() + Duration::from_secs(5); + while (!direct_pid_file.exists() + || !descendant_pid_file.exists() + || terminal_host_pids(&host_root).is_empty()) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let direct_pid = wait_for_pid_file(&direct_pid_file, Duration::from_secs(5)); + let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + + let stop = cli(&server, &["--json", "server", "stop"]); + + assert_success(&stop); + assert!(server.child.wait().unwrap().success()); + assert!(!create.wait().unwrap().success()); + let deadline = Instant::now() + Duration::from_secs(5); + while (process_exists(direct_pid) + || process_group_exists(direct_pid) + || process_exists(descendant_pid)) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(direct_pid)); + assert!(!process_group_exists(direct_pid)); + assert!(!process_exists(descendant_pid)); + assert!(terminal_host_pids(&host_root).is_empty()); +} + +#[cfg(unix)] +#[test] +fn server_stop_kills_ephemeral_pty_process_groups() { + let mut server = HeadlessServer::start_with("server-stop-ephemeral-process-group", true, &[]); + let descendant_pid_file = server.dir.join("ephemeral-descendant.pid"); + let command = format!( + "trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", + descendant_pid_file.display() + ); + let run = cli(&server, &["run", "--command", &command]); + assert_success(&run); + let surface = String::from_utf8(run.stdout).unwrap().trim().parse::().unwrap(); + let info = cli(&server, &["--json", "process-info", "--surface", &surface.to_string()]); + assert_success(&info); + let direct_pid = u32::try_from( + serde_json::from_slice::(&info.stdout).unwrap()["pid"].as_u64().unwrap(), + ) + .unwrap(); + let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + + let stop = cli(&server, &["--json", "server", "stop"]); + + assert_success(&stop); + assert!(server.child.wait().unwrap().success()); + let deadline = Instant::now() + Duration::from_secs(5); + while (process_exists(direct_pid) + || process_group_exists(direct_pid) + || process_exists(descendant_pid)) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(direct_pid)); + assert!(!process_group_exists(direct_pid)); + assert!(!process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn server_stop_drains_many_hosted_panes_before_acknowledging() { From c9ea3c92c80e6c6ff09865e5460d4ca2951926af Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:16:10 -0700 Subject: [PATCH 010/521] fix(tui): bound and prove server shutdown --- cmux-tui/crates/cmux-tui-core/build.rs | 207 +++++++++-- cmux-tui/crates/cmux-tui-core/src/mux.rs | 218 ++++++++--- cmux-tui/crates/cmux-tui-core/src/server.rs | 3 + cmux-tui/crates/cmux-tui-core/src/surface.rs | 339 ++++++++++++++++-- .../src/terminal_host_runtime.rs | 169 ++++++++- .../tests/build_source_identity.rs | 3 + cmux-tui/crates/cmux-tui/src/localization.rs | 7 +- .../crates/cmux-tui/src/server_lifecycle.rs | 67 +++- .../src/server_lifecycle/legacy_process.rs | 36 +- cmux-tui/dist/npm/cmux/bin/cmux.js | 5 +- cmux-tui/docs/protocol.md | 4 +- cmux-tui/spec/commands.md | 4 +- 12 files changed, 912 insertions(+), 150 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/tests/build_source_identity.rs diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index 180b651b8f4b..aca025340d83 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -1,7 +1,10 @@ use std::env; use std::fs; +use std::io; use std::path::{Path, PathBuf}; use std::process::Command; +#[cfg(unix)] +use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; const FNV_OFFSET: u64 = 0xcbf29ce484222325; const FNV_PRIME: u64 = 0x100000001b3; @@ -36,15 +39,14 @@ fn main() { track_git_identity(repository_root); track_git_identity(&ghostty_root); - if !nonempty_env("CMUX_TUI_BUILD_COMMIT") - && !nonempty_env("CMUX_MUX_BUILD_COMMIT") - && let Some(identity) = source_identity(workspace_root, workspace_root) - { + if !nonempty_env("CMUX_TUI_BUILD_COMMIT") && !nonempty_env("CMUX_MUX_BUILD_COMMIT") { + let identity = source_identity(workspace_root, workspace_root) + .expect("derive an exact cmux-tui source identity"); println!("cargo:rustc-env=CMUX_TUI_SOURCE_COMMIT={identity}"); } - if !nonempty_env("CMUX_TUI_GHOSTTY_COMMIT") - && let Some(identity) = source_identity(&ghostty_root, &ghostty_root) - { + if !nonempty_env("CMUX_TUI_GHOSTTY_COMMIT") { + let identity = source_identity(&ghostty_root, &ghostty_root) + .expect("derive an exact Ghostty source identity"); println!("cargo:rustc-env=CMUX_TUI_SOURCE_GHOSTTY_COMMIT={identity}"); } } @@ -53,35 +55,43 @@ fn nonempty_env(name: &str) -> bool { env::var_os(name).is_some_and(|value| !value.is_empty()) } -fn source_identity(git_root: &Path, fallback_root: &Path) -> Option { - if let Some(commit) = git_text(git_root, &["rev-parse", "HEAD"]) { - let status = - git_bytes(git_root, &["status", "--porcelain=v1", "--untracked-files=all", "--", "."])?; +fn source_identity(git_root: &Path, fallback_root: &Path) -> io::Result { + if let Some(commit) = optional_git_bytes(git_root, &["rev-parse", "HEAD"])? { + let commit = std::str::from_utf8(&commit) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "git commit is not UTF-8"))? + .trim(); + if commit.is_empty() { + return Err(io::Error::new(io::ErrorKind::InvalidData, "git commit is empty")); + } + let status = required_git_bytes( + git_root, + &["status", "--porcelain=v1", "-z", "--untracked-files=all", "--", "."], + )?; if status.is_empty() { - return Some(commit); + return Ok(commit.to_string()); } - return Some(format!("{commit}-dirty-{:016x}", dirty_fingerprint(git_root)?)); + return Ok(format!("{commit}-dirty-{:016x}", dirty_fingerprint(git_root)?)); } - directory_fingerprint(fallback_root) - .ok() - .map(|fingerprint| format!("source-{fingerprint:016x}")) + directory_fingerprint(fallback_root).map(|fingerprint| format!("source-{fingerprint:016x}")) } -fn dirty_fingerprint(root: &Path) -> Option { +fn dirty_fingerprint(root: &Path) -> io::Result { let mut hash = FNV_OFFSET; - hash_bytes(&mut hash, &git_bytes(root, &["diff", "--binary", "HEAD", "--", "."])?); + hash_component( + &mut hash, + &required_git_bytes(root, &["diff", "--binary", "--full-index", "HEAD", "--", "."])?, + ); let untracked = - git_bytes(root, &["ls-files", "-z", "--others", "--exclude-standard", "--", "."])?; + required_git_bytes(root, &["ls-files", "-z", "--others", "--exclude-standard", "--", "."])?; for path in untracked.split(|byte| *byte == 0).filter(|path| !path.is_empty()) { - hash_bytes(&mut hash, path); - let path = std::str::from_utf8(path).ok()?; - hash_bytes(&mut hash, &fs::read(root.join(path)).ok()?); + hash_component(&mut hash, path); + hash_filesystem_entry(&mut hash, &path_from_git_bytes(root, path)?)?; } - Some(hash) + Ok(hash) } -fn directory_fingerprint(root: &Path) -> std::io::Result { - fn visit(root: &Path, path: &Path, hash: &mut u64) -> std::io::Result<()> { +fn directory_fingerprint(root: &Path) -> io::Result { + fn visit(root: &Path, path: &Path, hash: &mut u64) -> io::Result<()> { let mut entries = fs::read_dir(path)?.collect::, _>>()?; entries.sort_by_key(|entry| entry.file_name()); for entry in entries { @@ -93,14 +103,15 @@ fn directory_fingerprint(root: &Path) -> std::io::Result { continue; } let path = entry.path(); - if path.is_dir() { + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_dir() { visit(root, &path, hash)?; - } else if path.is_file() { - hash_bytes( + } else { + hash_component( hash, path.strip_prefix(root).unwrap_or(&path).as_os_str().as_encoded_bytes(), ); - hash_bytes(hash, &fs::read(path)?); + hash_filesystem_entry(hash, &path)?; } } Ok(()) @@ -111,6 +122,28 @@ fn directory_fingerprint(root: &Path) -> std::io::Result { Ok(hash) } +fn hash_filesystem_entry(hash: &mut u64, path: &Path) -> io::Result<()> { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() { + hash_component(hash, b"symlink"); + hash_component(hash, fs::read_link(path)?.as_os_str().as_encoded_bytes()); + } else if metadata.file_type().is_file() { + hash_component(hash, b"file"); + hash_component(hash, &fs::read(path)?); + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unsupported source entry {}", path.display()), + )); + } + Ok(()) +} + +fn hash_component(hash: &mut u64, bytes: &[u8]) { + hash_bytes(hash, &(bytes.len() as u64).to_le_bytes()); + hash_bytes(hash, bytes); +} + fn hash_bytes(hash: &mut u64, bytes: &[u8]) { for byte in bytes { *hash ^= u64::from(*byte); @@ -138,12 +171,120 @@ fn resolve_git_path(root: &Path, value: &str) -> PathBuf { } fn git_text(root: &Path, args: &[&str]) -> Option { - let output = git_bytes(root, args)?; + let output = optional_git_bytes(root, args).ok()??; let value = std::str::from_utf8(&output).ok()?.trim(); (!value.is_empty()).then(|| value.to_string()) } -fn git_bytes(root: &Path, args: &[&str]) -> Option> { - let output = Command::new("git").arg("-C").arg(root).args(args).output().ok()?; - output.status.success().then_some(output.stdout) +fn optional_git_bytes(root: &Path, args: &[&str]) -> io::Result>> { + let output = match Command::new("git").arg("-C").arg(root).args(args).output() { + Ok(output) => output, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + Ok(output.status.success().then_some(output.stdout)) +} + +fn required_git_bytes(root: &Path, args: &[&str]) -> io::Result> { + optional_git_bytes(root, args)?.ok_or_else(|| { + io::Error::other(format!("git {} failed in {}", args.join(" "), root.display())) + }) +} + +#[cfg(unix)] +fn path_from_git_bytes(root: &Path, path: &[u8]) -> io::Result { + Ok(root.join(OsStr::from_bytes(path))) +} + +#[cfg(not(unix))] +fn path_from_git_bytes(root: &Path, path: &[u8]) -> io::Result { + let path = std::str::from_utf8(path) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "git path is not UTF-8"))?; + Ok(root.join(path)) +} + +#[cfg(test)] +mod tests { + use super::*; + #[cfg(unix)] + use std::os::unix::fs::symlink; + use std::time::{SystemTime, UNIX_EPOCH}; + #[cfg(target_os = "linux")] + use std::{ffi::OsString, os::unix::ffi::OsStringExt}; + + struct TestRepository { + root: PathBuf, + } + + impl TestRepository { + fn new(name: &str) -> Self { + let nonce = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let root = env::temp_dir() + .join(format!("cmux-source-identity-{name}-{}-{nonce}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + run_git(&root, &["init", "-q"]); + run_git(&root, &["config", "user.email", "tests@cmux.invalid"]); + run_git(&root, &["config", "user.name", "cmux tests"]); + fs::write(root.join("tracked"), b"tracked").unwrap(); + run_git(&root, &["add", "tracked"]); + run_git(&root, &["commit", "-qm", "initial"]); + Self { root } + } + } + + impl Drop for TestRepository { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } + } + + fn run_git(root: &Path, args: &[&str]) { + let output = Command::new("git").arg("-C").arg(root).args(args).output().unwrap(); + assert!( + output.status.success(), + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr) + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn dirty_identity_hashes_non_utf8_untracked_paths_and_contents() { + let repository = TestRepository::new("non-utf8"); + let name = OsString::from_vec(vec![b'n', b'o', b'n', b'-', 0xff]); + let path = repository.root.join(name); + fs::write(&path, b"one").unwrap(); + let first = source_identity(&repository.root, &repository.root).unwrap(); + + fs::write(path, b"two").unwrap(); + let second = source_identity(&repository.root, &repository.root).unwrap(); + + assert!(first.contains("-dirty-")); + assert!(second.contains("-dirty-")); + assert_ne!(first, second); + } + + #[cfg(unix)] + #[test] + fn dirty_identity_hashes_dangling_symlink_targets() { + let repository = TestRepository::new("dangling-symlink"); + let link = repository.root.join("untracked-link"); + symlink("missing-one", &link).unwrap(); + let first = source_identity(&repository.root, &repository.root).unwrap(); + + fs::remove_file(&link).unwrap(); + symlink("missing-two", &link).unwrap(); + let second = source_identity(&repository.root, &repository.root).unwrap(); + + assert_ne!(first, second); + } + + #[test] + fn dirty_identity_fails_closed_when_git_cannot_describe_changes() { + let repository = TestRepository::new("git-failure"); + fs::write(repository.root.join(".git/index"), b"invalid index").unwrap(); + + assert!(source_identity(&repository.root, &repository.root).is_err()); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 354d11c457c6..641e056b8999 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6,7 +6,7 @@ use std::fmt; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; -use std::sync::{Arc, Mutex, MutexGuard, Weak}; +use std::sync::{Arc, Mutex, MutexGuard, TryLockError, Weak}; use std::thread::ThreadId; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -43,7 +43,6 @@ const WORKSPACE_NAME_MAX_BYTES: usize = 1_024; const PROVIDER_WORKSPACE_AUTHORITY_MIN_BYTES: usize = 32; const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); -const SHUTDOWN_HOST_EXIT_TIMEOUT: Duration = Duration::from_secs(4); const SHUTDOWN_FANOUT_WORKERS: usize = 128; /// An opaque per-mux credential provisioned by the external machine @@ -619,12 +618,26 @@ impl SurfaceCreationGate { Ok(SurfaceCreationGuard { gate: self, thread }) } - fn stop_and_wait(&self) { + fn stop(&self) { + let mut state = self.state.lock().unwrap(); + state.shutting_down = true; + self.idle.notify_all(); + } + + fn stop_and_wait_until(&self, deadline: Instant) -> bool { let mut state = self.state.lock().unwrap(); state.shutting_down = true; while !state.active_threads.is_empty() { - state = self.idle.wait(state).unwrap(); + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.idle.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && !state.active_threads.is_empty() { + return false; + } } + true } } @@ -773,6 +786,8 @@ pub struct Mux { subscribers: MuxEventBroadcaster, shutdown_requested: AtomicBool, surface_creations: SurfaceCreationGate, + shutdown_coordinator: Mutex<()>, + shutdown_surfaces: Mutex>>, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -1001,6 +1016,8 @@ impl Mux { subscribers: MuxEventBroadcaster::default(), shutdown_requested: AtomicBool::new(false), surface_creations: SurfaceCreationGate::default(), + shutdown_coordinator: Mutex::new(()), + shutdown_surfaces: Mutex::new(Vec::new()), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -3615,8 +3632,11 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); - self.surface_creations.stop_and_wait(); - let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); + let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + let _ = self.surface_creations.stop_and_wait_until(deadline); + let mut surfaces = + self.state.lock().unwrap().surfaces.values().cloned().collect::>(); + surfaces.extend(self.shutdown_surfaces.lock().unwrap().drain(..)); for surface in surfaces { surface.disconnect_for_daemon_shutdown(); } @@ -3637,11 +3657,15 @@ impl Mux { /// terminal in one transaction, detach the complete topology in one /// state mutation, and terminate each runtime before acknowledging the /// shutdown request. A failed durable commit leaves topology untouched; - /// a failed host termination leaves the fence closed so a retry cannot - /// race a new PTY into the session. + /// a failed runtime termination keeps process ownership and the fence + /// closed so a retry cannot race a new PTY into the session. pub fn close_all_surfaces_for_shutdown(&self) -> anyhow::Result { + let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; self.shutting_down.store(true, Ordering::Release); - self.surface_creations.stop_and_wait(); + if !self.surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("surface creation did not stop before the shutdown deadline"); + } #[cfg(unix)] let terminal_host_records = { @@ -3653,7 +3677,7 @@ impl Mux { } }; - let (surfaces, tree_changed) = { + let (mut surfaces, tree_changed) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); let mut registry = self.workspace_registry.lock().unwrap(); let terminals = registry @@ -3688,6 +3712,7 @@ impl Mux { } (surfaces, tree_changed) }; + surfaces.extend(self.shutdown_surfaces.lock().unwrap().drain(..)); self.pending_workspace_surfaces.lock().unwrap().clear(); self.agent_records.lock().unwrap().clear(); @@ -3704,36 +3729,81 @@ impl Mux { self.emit(MuxEvent::TreeChanged); } - bounded_shutdown_fanout(&surfaces, |surface| { - surface.terminate_for_server_shutdown(SHUTDOWN_TERMINATION_TIMEOUT); + let failed_surfaces = Mutex::new(Vec::>::new()); + bounded_shutdown_fanout(&surfaces, deadline, |surface, deadline| { + if !surface.terminate_for_server_shutdown(deadline) { + failed_surfaces.lock().unwrap().push(surface.clone()); + } }); + let mut failed_surfaces = failed_surfaces.into_inner().unwrap(); + failed_surfaces.sort_by_key(|surface| surface.id); + let failed_surface_ids = + failed_surfaces.iter().map(|surface| surface.id).collect::>(); + *self.shutdown_surfaces.lock().unwrap() = failed_surfaces; #[cfg(unix)] - { + let failed_hosts = { let failed = Mutex::new(Vec::new()); - bounded_shutdown_fanout(&terminal_host_records, |(path, record)| { - if !terminate_and_confirm_terminal_host_record( - record, - path, - SHUTDOWN_HOST_EXIT_TIMEOUT, - ) { - failed.lock().unwrap().push(record.terminal_id.clone()); - } - }); + bounded_shutdown_fanout( + &terminal_host_records, + deadline, + |(path, record), deadline| { + if !terminate_and_confirm_terminal_host_record(record, path, deadline) { + failed.lock().unwrap().push(record.terminal_id.clone()); + } + }, + ); let mut failed = failed.into_inner().unwrap(); - if !failed.is_empty() { - failed.sort(); - anyhow::bail!( - "could not terminate {} terminal host(s): {}", - failed.len(), - failed.join(", ") - ); - } + failed.sort(); + failed + }; + + let mut failures = Vec::new(); + if !failed_surface_ids.is_empty() { + failures.push(format!( + "could not terminate {} surface process(es): {}", + failed_surface_ids.len(), + failed_surface_ids + .into_iter() + .map(|surface| surface.to_string()) + .collect::>() + .join(", ") + )); + } + #[cfg(unix)] + if !failed_hosts.is_empty() { + failures.push(format!( + "could not terminate {} terminal host(s): {}", + failed_hosts.len(), + failed_hosts.join(", ") + )); + } + if !failures.is_empty() { + anyhow::bail!("{}", failures.join("; ")); } Ok(surfaces.len()) } + fn lock_shutdown_coordinator_until( + &self, + deadline: Instant, + ) -> anyhow::Result> { + loop { + match self.shutdown_coordinator.try_lock() { + Ok(coordinator) => return Ok(coordinator), + Err(TryLockError::Poisoned(_)) => { + anyhow::bail!("shutdown coordinator is unavailable") + } + Err(TryLockError::WouldBlock) => {} + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + anyhow::bail!("another shutdown request exceeded the shutdown deadline"); + }; + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } + } + /// Atomically reserve a daemon handoff while proving no other native /// browser owns this mux. New owner announcements are rejected until the /// response is queued or the reservation is cancelled. @@ -3750,7 +3820,7 @@ impl Mux { /// and remain available for the replacement daemon to adopt. pub fn request_daemon_shutdown(&self) { self.shutting_down.store(true, Ordering::Release); - self.surface_creations.stop_and_wait(); + self.surface_creations.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); } @@ -3758,6 +3828,10 @@ impl Mux { self.daemon_shutdown_requested.load(Ordering::Acquire) } + pub(crate) fn is_shutting_down(&self) -> bool { + self.shutting_down.load(Ordering::Acquire) + } + /// Update options used for future surface/browser launches. pub fn update_surface_options(&self, update: impl FnOnce(&mut SurfaceOptions)) { let mut options = self.surface_options.lock().unwrap(); @@ -7498,9 +7572,8 @@ fn cleanup_terminal_host_record( fn terminate_and_confirm_terminal_host_record( record: &crate::terminal_host_runtime::TerminalHostRecord, record_path: &Path, - timeout: Duration, + deadline: Instant, ) -> bool { - let deadline = Instant::now() + timeout; let mut termination_sent = false; let mut last_terminate_attempt = None; loop { @@ -7527,7 +7600,9 @@ fn terminate_and_confirm_terminal_host_record( termination_sent = terminate_host_record_with_timeout( record.clone(), record_path.to_path_buf(), - SHUTDOWN_TERMINATION_TIMEOUT, + deadline + .saturating_duration_since(Instant::now()) + .min(SHUTDOWN_TERMINATION_TIMEOUT), ); last_terminate_attempt = Some(Instant::now()); } @@ -7539,7 +7614,11 @@ fn terminate_and_confirm_terminal_host_record( } } -fn bounded_shutdown_fanout(items: &[T], operation: impl Fn(&T) + Sync) { +fn bounded_shutdown_fanout( + items: &[T], + deadline: Instant, + operation: impl Fn(&T, Instant) + Sync, +) { if items.is_empty() { return; } @@ -7553,7 +7632,7 @@ fn bounded_shutdown_fanout(items: &[T], operation: impl Fn(&T) + Sync) loop { let index = next.fetch_add(1, Ordering::Relaxed); let Some(item) = items.get(index) else { break }; - operation(item); + operation(item, deadline); } }); } @@ -9795,6 +9874,24 @@ mod tests { assert_eq!(error.to_string(), "server is shutting down"); } + #[test] + fn failed_surface_shutdown_retains_process_ownership_for_retry() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + owned.set_server_shutdown_failure_for_test(true); + + let error = mux.close_all_surfaces_for_shutdown().unwrap_err(); + + assert!(error.to_string().contains("could not terminate 1 surface process")); + assert!(mux.surface(surface.id).is_none()); + assert_eq!(mux.shutdown_surfaces.lock().unwrap().len(), 1); + + owned.set_server_shutdown_failure_for_test(false); + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); + } + #[test] fn shutdown_fanout_runs_terminations_concurrently() { let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); @@ -9802,13 +9899,18 @@ mod tests { let worker = std::thread::spawn({ let gate = gate.clone(); move || { - bounded_shutdown_fanout(&[(); 8], |_| { - started_tx.send(()).unwrap(); - let (released, wake) = &*gate; - let released = - wake.wait_while(released.lock().unwrap(), |released| !*released).unwrap(); - drop(released); - }); + bounded_shutdown_fanout( + &[(); 8], + Instant::now() + Duration::from_secs(1), + |_, _| { + started_tx.send(()).unwrap(); + let (released, wake) = &*gate; + let released = wake + .wait_while(released.lock().unwrap(), |released| !*released) + .unwrap(); + drop(released); + }, + ); } }); @@ -9828,6 +9930,36 @@ mod tests { assert_eq!(started, 8); } + #[test] + fn shutdown_fanout_reuses_one_deadline_across_worker_batches() { + let deadline = Instant::now() + Duration::from_secs(1); + let observed = Mutex::new(Vec::new()); + + bounded_shutdown_fanout( + &[(); SHUTDOWN_FANOUT_WORKERS * 2 + 1], + deadline, + |_, item_deadline| { + observed.lock().unwrap().push(item_deadline); + }, + ); + + let observed = observed.into_inner().unwrap(); + assert_eq!(observed.len(), SHUTDOWN_FANOUT_WORKERS * 2 + 1); + assert!(observed.into_iter().all(|item_deadline| item_deadline == deadline)); + } + + #[test] + fn surface_creation_fence_has_a_bounded_wait() { + let gate = SurfaceCreationGate::default(); + let creation = gate.begin().unwrap(); + let started = Instant::now(); + + assert!(!gate.stop_and_wait_until(started + Duration::from_millis(25))); + + drop(creation); + assert!(started.elapsed() < Duration::from_millis(250)); + } + #[test] fn server_shutdown_waits_for_in_flight_surface_creation() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 1353f6306bee..0a3063243e08 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -58,6 +58,9 @@ use crate::{ const ATTACH_INITIAL_SIZE_CAPABILITY: &str = "attach-initial-size"; const WORKSPACE_REGISTRY_CAPABILITY: &str = "workspace-registry-v1"; pub const SERVER_SHUTDOWN_CAPABILITY: &str = "server-shutdown-v1"; +/// Maximum wall-clock time the server spends draining a shutdown request. +/// Clients add their own transport margin to this server-owned bound. +pub const SERVER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); pub const PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY: &str = "provider-managed-workspace-authority-v2"; const INITIAL_BROWSER_RESIZE_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 9b475b31fec5..5d6a50b50dbf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -14,7 +14,7 @@ use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{ Receiver, RecvError, RecvTimeoutError, SyncSender, TryRecvError, TrySendError, sync_channel, }; -use std::sync::{Arc, Mutex, TryLockError, Weak}; +use std::sync::{Arc, Condvar, Mutex, TryLockError, Weak}; use std::time::{Duration, Instant}; use ghostty_vt::{ @@ -603,7 +603,7 @@ enum PtyRuntime { Local { writer: Box, master: Box, - killer: Box, + process: LocalProcess, }, #[cfg(unix)] Hosted(Box), @@ -611,6 +611,250 @@ enum PtyRuntime { ExitedHosted, } +enum LocalProcess { + Owned(Arc), + #[cfg(test)] + Untracked(Box), +} + +impl LocalProcess { + fn request_hangup(&mut self) -> bool { + match self { + Self::Owned(process) => process.request_hangup(), + #[cfg(test)] + Self::Untracked(killer) => killer.kill().is_ok(), + } + } + + fn terminate_and_wait(&mut self, master: &dyn MasterPty, deadline: Instant) -> bool { + match self { + Self::Owned(process) => process.terminate_and_wait(master, deadline), + #[cfg(test)] + Self::Untracked(killer) => killer.kill().is_ok(), + } + } +} + +struct LocalPtyProcess { + killer: Mutex>, + #[cfg(unix)] + pid: Option, + exited: (Mutex, Condvar), + #[cfg(unix)] + pty_drained: AtomicBool, + #[cfg(unix)] + child_waitable: AtomicBool, + #[cfg(unix)] + termination_started: AtomicBool, + #[cfg(unix)] + child_signal_lock: Mutex<()>, + #[cfg(unix)] + child_reaped: AtomicBool, + #[cfg(unix)] + group_escalation_complete: AtomicBool, +} + +impl LocalPtyProcess { + fn new(pid: Option, killer: Box) -> Arc { + #[cfg(not(unix))] + let _ = pid; + Arc::new(Self { + killer: Mutex::new(killer), + #[cfg(unix)] + pid, + exited: (Mutex::new(false), Condvar::new()), + #[cfg(unix)] + pty_drained: AtomicBool::new(false), + #[cfg(unix)] + child_waitable: AtomicBool::new(false), + #[cfg(unix)] + termination_started: AtomicBool::new(false), + #[cfg(unix)] + child_signal_lock: Mutex::new(()), + #[cfg(unix)] + child_reaped: AtomicBool::new(false), + #[cfg(unix)] + group_escalation_complete: AtomicBool::new(false), + }) + } + + fn spawn_reaper( + self: &Arc, + mut child: Box, + ) -> std::io::Result<()> { + let process = self.clone(); + std::thread::Builder::new().name("surface-child".into()).spawn(move || { + #[cfg(unix)] + { + let observed_without_reaping = process + .pid + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .is_some_and(|pid| wait_for_local_child_without_reaping(pid).is_ok()); + if observed_without_reaping { + process.child_waitable.store(true, Ordering::Release); + process.exited.1.notify_all(); + loop { + let signal = process.child_signal_lock.lock().unwrap(); + let escalation_complete = + process.group_escalation_complete.load(Ordering::Acquire); + let termination_started = + process.termination_started.load(Ordering::Acquire); + let pty_drained = process.pty_drained.load(Ordering::Acquire); + if escalation_complete || (!termination_started && pty_drained) { + let _ = child.wait(); + process.child_reaped.store(true, Ordering::Release); + drop(signal); + break; + } + drop(signal); + let state = process.exited.0.lock().unwrap(); + let _state = process + .exited + .1 + .wait_while(state, |_| { + !process.group_escalation_complete.load(Ordering::Acquire) + && (process.termination_started.load(Ordering::Acquire) + || !process.pty_drained.load(Ordering::Acquire)) + }) + .unwrap(); + } + } else { + let _ = child.wait(); + process.child_reaped.store(true, Ordering::Release); + process.child_waitable.store(true, Ordering::Release); + } + } + #[cfg(not(unix))] + { + let _ = child.wait(); + } + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + })?; + Ok(()) + } + + fn request_hangup(&self) -> bool { + self.killer.lock().unwrap().kill().is_ok() + } + + fn mark_pty_drained(&self) { + #[cfg(unix)] + { + self.pty_drained.store(true, Ordering::Release); + self.exited.1.notify_all(); + } + } + + fn wait_for_exit_until(&self, deadline: Instant) -> bool { + let exited = self.exited.0.lock().unwrap(); + if *exited { + return true; + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (exited, _) = + self.exited.1.wait_timeout_while(exited, remaining, |exited| !*exited).unwrap(); + *exited + } + + #[cfg(unix)] + fn wait_for_child_waitable_until(&self, deadline: Instant) { + if self.child_waitable.load(Ordering::Acquire) { + return; + } + let state = self.exited.0.lock().unwrap(); + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return }; + let _ = self + .exited + .1 + .wait_timeout_while(state, remaining, |_| !self.child_waitable.load(Ordering::Acquire)) + .unwrap(); + } + + #[cfg(unix)] + fn signal_terminal_process_groups(&self, master: &dyn MasterPty, signal: libc::c_int) { + let _signal = self.child_signal_lock.lock().unwrap(); + if self.child_reaped.load(Ordering::Acquire) { + return; + } + let mut groups = self + .pid + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .into_iter() + .chain(master.process_group_leader()) + .collect::>(); + groups.sort_unstable(); + groups.dedup(); + // portable-pty creates a new session for the shell. Signal its + // original group and the current foreground job group, while never + // addressing the mux's own process group. + let mux_group = unsafe { libc::getpgrp() }; + for group in groups.into_iter().filter(|group| *group > 0 && *group != mux_group) { + let _ = unsafe { libc::killpg(group, signal) }; + } + } + + fn terminate_and_wait(&self, master: &dyn MasterPty, deadline: Instant) -> bool { + #[cfg(unix)] + { + { + let _signal = self.child_signal_lock.lock().unwrap(); + if self.child_reaped.load(Ordering::Acquire) { + return true; + } + self.termination_started.store(true, Ordering::Release); + } + self.signal_terminal_process_groups(master, libc::SIGHUP); + let _ = self.request_hangup(); + self.wait_for_child_waitable_until( + deadline.min(Instant::now() + Duration::from_millis(250)), + ); + // SIGHUP is cooperative. Escalate every owned terminal group so + // an ignoring shell or foreground job cannot survive a successful + // shutdown acknowledgement. + self.signal_terminal_process_groups(master, libc::SIGKILL); + self.group_escalation_complete.store(true, Ordering::Release); + self.exited.1.notify_all(); + } + #[cfg(not(unix))] + { + let _ = master; + let _ = self.request_hangup(); + } + self.wait_for_exit_until(deadline) + } +} + +impl Drop for LocalPtyProcess { + fn drop(&mut self) { + let _ = self.killer.get_mut().map(|killer| killer.kill()); + } +} + +#[cfg(unix)] +fn wait_for_local_child_without_reaping(pid: libc::pid_t) -> std::io::Result<()> { + loop { + let mut status = std::mem::MaybeUninit::::uninit(); + let result = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + status.as_mut_ptr(), + libc::WEXITED | libc::WNOWAIT, + ) + }; + if result == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(error); + } + } +} + #[cfg(unix)] fn hosted_terminal_callbacks( id: SurfaceId, @@ -676,19 +920,22 @@ impl Surface { #[cfg(unix)] if let Some(root) = opts.terminal_host_root.clone() { let default_colors = mux.upgrade().map(|mux| mux.default_colors()).unwrap_or_default(); + let launch_cancelled = || mux.upgrade().is_none_or(|mux| mux.is_shutting_down()); let attachment = match terminal_id { Some(terminal_id) => { - crate::terminal_host_runtime::launch_terminal_host_with_identity( + crate::terminal_host_runtime::launch_terminal_host_with_identity_cancellable( &opts, &root, default_colors, terminal_id, + &launch_cancelled, )? } - None => crate::terminal_host_runtime::launch_terminal_host( + None => crate::terminal_host_runtime::launch_terminal_host_cancellable( &opts, &root, default_colors, + &launch_cancelled, )?, }; return Self::spawn_hosted(id, opts, mux, attachment, true); @@ -720,10 +967,12 @@ impl Surface { cmd.cwd(cwd); } - let mut child = pty.slave.spawn_command(cmd)?; + let child = pty.slave.spawn_command(cmd)?; let pid = child.process_id(); drop(pty.slave); let killer = child.clone_killer(); + let process = LocalPtyProcess::new(pid, killer); + let reader_process = process.clone(); let mut reader = pty.master.try_clone_reader()?; let writer = pty.master.take_writer()?; @@ -768,7 +1017,11 @@ impl Surface { meta: SurfaceMeta { id, name: Mutex::new(None), selection: Mutex::new(None) }, term: Mutex::new(term), mouse_encoders: Mutex::new(mouse_encoders), - runtime: Mutex::new(PtyRuntime::Local { writer, master: pty.master, killer }), + runtime: Mutex::new(PtyRuntime::Local { + writer, + master: pty.master, + process: LocalProcess::Owned(process.clone()), + }), host_identity: None, pid, command: argv, @@ -795,10 +1048,14 @@ impl Surface { frame_requests, })); - spawn_frame_producer(&surface, frame_rx)?; + process.spawn_reaper(child)?; + if let Err(error) = spawn_frame_producer(&surface, frame_rx) { + let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); + return Err(error); + } // PTY reader: pty bytes -> terminal state -> SurfaceOutput events. - std::thread::Builder::new().name(format!("surface-{id}-reader")).spawn({ + let reader = std::thread::Builder::new().name(format!("surface-{id}-reader")).spawn({ let surface = surface.clone(); move || { let mut buf = [0u8; 64 * 1024]; @@ -872,16 +1129,16 @@ impl Surface { if let Some(pty) = surface.as_pty() { pty.dead.store(true, Ordering::Release); } + reader_process.mark_pty_drained(); if let Some(mux) = mux.upgrade() { mux.surface_exited(surface.id); } } - })?; - - // Child reaper: avoid zombies; the reader thread handles EOF. - std::thread::Builder::new().name(format!("surface-{id}-wait")).spawn(move || { - let _ = child.wait(); - })?; + }); + if let Err(error) = reader { + let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); + return Err(error.into()); + } Ok(surface) } @@ -1498,7 +1755,7 @@ impl Surface { pixel_height: 0, }), }), - killer: Box::new(TestChildKiller), + process: LocalProcess::Untracked(Box::new(TestChildKiller)), }), host_identity: None, pid: Some(id as u32), @@ -1527,6 +1784,18 @@ impl Surface { }))) } + #[cfg(test)] + pub(crate) fn set_server_shutdown_failure_for_test(&self, fail: bool) { + let Self::Pty(pty) = self else { return }; + let mut runtime = pty.runtime.lock().unwrap(); + let PtyRuntime::Local { process, .. } = &mut *runtime else { return }; + *process = LocalProcess::Untracked(if fail { + Box::new(TestFailingChildKiller) + } else { + Box::new(TestChildKiller) + }); + } + fn as_pty(&self) -> Option<&PtySurface> { match self { Surface::Pty(surface) => Some(surface), @@ -2069,10 +2338,7 @@ impl Surface { Surface::Pty(pty) => { let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { - PtyRuntime::Local { killer, .. } => { - let _ = killer.kill(); - true - } + PtyRuntime::Local { process, .. } => process.request_hangup(), #[cfg(unix)] PtyRuntime::Hosted(host) => { // The host owns record cleanup and removes it only @@ -2094,27 +2360,29 @@ impl Surface { /// Start runtime termination. Durable terminal-host death is confirmed /// separately through the process-bound discovery record. - pub(crate) fn terminate_for_server_shutdown(&self, timeout: Duration) { + pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { match self { - #[cfg(unix)] Surface::Pty(pty) => { let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { + #[cfg(unix)] PtyRuntime::Hosted(host) => { + let timeout = deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(100)); let _ = host.terminate_with_timeout(timeout); + true } - PtyRuntime::Local { killer, .. } => { - let _ = killer.kill(); + PtyRuntime::Local { master, process, .. } => { + process.terminate_and_wait(master.as_ref(), deadline) } - PtyRuntime::ExitedHosted => {} + #[cfg(unix)] + PtyRuntime::ExitedHosted => true, } } - #[cfg(not(unix))] - Surface::Pty(_) => { - let _ = self.terminate_runtime(); - } Surface::Browser(browser) => { browser.kill(); + true } } } @@ -2310,6 +2578,21 @@ impl ChildKiller for TestChildKiller { } } +#[cfg(test)] +#[derive(Debug)] +struct TestFailingChildKiller; + +#[cfg(test)] +impl ChildKiller for TestFailingChildKiller { + fn kill(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("forced shutdown failure")) + } + + fn clone_killer(&self) -> Box { + Box::new(TestFailingChildKiller) + } +} + impl PtySurface { /// Snapshot sparse colors and the host-resolved cursor visual without /// touching the shared renderer or consuming its damage. diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index c8def7412b09..1103d9082eb7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -293,6 +293,8 @@ mod unix { const HOST_PTY_DRAIN_GRACE: Duration = Duration::from_millis(250); const HOST_FORCED_DRAIN_WINDOW: Duration = Duration::from_millis(100); const HOST_LAUNCH_ROLLBACK_WAIT: Duration = Duration::from_secs(4); + const HOST_LAUNCH_TIMEOUT: Duration = Duration::from_secs(10); + const HOST_LAUNCH_CANCEL_POLL: Duration = Duration::from_millis(25); struct SpawnedHostProcess { child: Option, @@ -323,6 +325,15 @@ mod unix { } } } + + fn detach_reaper(mut self) { + let Some(mut child) = self.child.take() else { return }; + let _ = thread::Builder::new().name("terminal-host-detached-reaper".into()).spawn( + move || { + let _ = child.wait(); + }, + ); + } } impl Drop for SpawnedHostProcess { @@ -807,9 +818,24 @@ mod unix { options: &SurfaceOptions, root: &Path, default_colors: DefaultColors, + ) -> anyhow::Result { + launch_terminal_host_cancellable(options, root, default_colors, &|| false) + } + + pub(crate) fn launch_terminal_host_cancellable( + options: &SurfaceOptions, + root: &Path, + default_colors: DefaultColors, + cancelled: &dyn Fn() -> bool, ) -> anyhow::Result { let terminal_id = TerminalId::random()?; - launch_terminal_host_with_identity(options, root, default_colors, terminal_id) + launch_terminal_host_with_identity_cancellable( + options, + root, + default_colors, + terminal_id, + cancelled, + ) } /// Launch using a registry-reserved stable UUID. The workspace registry @@ -820,6 +846,22 @@ mod unix { root: &Path, default_colors: DefaultColors, terminal_id: TerminalId, + ) -> anyhow::Result { + launch_terminal_host_with_identity_cancellable( + options, + root, + default_colors, + terminal_id, + &|| false, + ) + } + + pub(crate) fn launch_terminal_host_with_identity_cancellable( + options: &SurfaceOptions, + root: &Path, + default_colors: DefaultColors, + terminal_id: TerminalId, + cancelled: &dyn Fn() -> bool, ) -> anyhow::Result { prepare_private_dir(root)?; let owner_token = CapabilityToken::random()?; @@ -881,8 +923,13 @@ mod unix { let host_pid = process.child_mut().id(); let mut stdin = process.child_mut().stdin.take().context("open terminal-host bootstrap stdin")?; - let mut stdout = + let stdout = process.child_mut().stdout.take().context("open terminal-host bootstrap stdout")?; + let mut stdout = CancellableHostReader { + reader: stdout, + deadline: Instant::now() + HOST_LAUNCH_TIMEOUT, + cancelled, + }; let bootstrap = HostBootstrap { min_version: PROTOCOL_VERSION, @@ -891,7 +938,20 @@ mod unix { owner_token, }; write_frame(&mut stdin, &bootstrap.into_frame(1))?; - let ready_frame = read_required_frame(&mut stdout, "bootstrap ready")?; + let ready_frame = match read_required_frame(&mut stdout, "bootstrap ready") { + Ok(frame) => frame, + Err(error) => { + if cancelled() && record_path.exists() { + // Publication transfers PTY ownership to the durable + // record. Keep that host discoverable so the shutdown + // coordinator can terminate it through the authenticated + // endpoint instead of killing the host and orphaning its + // shell. + process.detach_reaper(); + } + return Err(error); + } + }; if ready_frame.kind != MessageKind::Ready { anyhow::bail!("terminal host returned {:?} instead of Ready", ready_frame.kind); } @@ -903,7 +963,15 @@ mod unix { let mut launch_frame = Frame::new(MessageKind::Launch, launch.encode()?); launch_frame.request_id = 2; write_frame(&mut stdin, &launch_frame)?; - let launched_frame = read_required_frame(&mut stdout, "launch ready")?; + let launched_frame = match read_required_frame(&mut stdout, "launch ready") { + Ok(frame) => frame, + Err(error) => { + if cancelled() && record_path.exists() { + process.detach_reaper(); + } + return Err(error); + } + }; if launched_frame.kind != MessageKind::Ready || launched_frame.request_id != 2 { anyhow::bail!("terminal host did not acknowledge launch"); } @@ -931,9 +999,65 @@ mod unix { // row Exited. let mut attachment = connect_record(record, record_path)?; attachment.launch_process = Some(process); + if cancelled() { + drop(attachment); + anyhow::bail!("terminal host launch cancelled because the server is shutting down"); + } Ok(attachment) } + struct CancellableHostReader<'a, R> { + reader: R, + deadline: Instant, + cancelled: &'a dyn Fn() -> bool, + } + + impl Read for CancellableHostReader<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + loop { + if (self.cancelled)() { + return Err(std::io::Error::new( + // `Read::read_exact` transparently retries + // `Interrupted`, so cancellation needs a terminal + // error kind to release the surface-creation permit. + std::io::ErrorKind::ConnectionAborted, + "terminal host launch cancelled during server shutdown", + )); + } + let Some(remaining) = self.deadline.checked_duration_since(Instant::now()) else { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host launch timed out", + )); + }; + let wait = remaining.min(HOST_LAUNCH_CANCEL_POLL); + let timeout_ms = wait.as_millis().max(1).min(i32::MAX as u128) as i32; + let mut descriptor = libc::pollfd { + fd: self.reader.as_raw_fd(), + events: libc::POLLIN | libc::POLLHUP, + revents: 0, + }; + let result = unsafe { libc::poll(&mut descriptor, 1, timeout_ms) }; + if result > 0 { + if descriptor.revents & libc::POLLNVAL != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "terminal host bootstrap pipe became invalid", + )); + } + return self.reader.read(buffer); + } + if result == 0 { + continue; + } + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(error); + } + } + } + } + pub fn adopt_terminal_host( record: TerminalHostRecord, record_path: PathBuf, @@ -1184,7 +1308,7 @@ mod unix { let terminal_id = TerminalId::from_bytes(decode_hex_array(&record.terminal_id)?); let incarnation = HostIncarnation::from_bytes(decode_hex_array(&record.incarnation)?); let owner_token = CapabilityToken::from_bytes(decode_hex_array(&record.owner_token)?); - let mut stream = connect_with_retry(Path::new(&record.endpoint))?; + let mut stream = connect_with_timeout(Path::new(&record.endpoint), handshake_timeout)?; stream.set_read_timeout(Some(handshake_timeout))?; stream.set_write_timeout(Some(handshake_timeout))?; let hello = ClientHello { @@ -1251,20 +1375,18 @@ mod unix { Ok(attachment) } - fn connect_with_retry(path: &Path) -> anyhow::Result { - let mut last_error = None; - for _ in 0..100 { - match UnixStream::connect(path) { + fn connect_with_timeout(path: &Path, timeout: Duration) -> anyhow::Result { + let deadline = Instant::now() + timeout; + loop { + let error = match UnixStream::connect(path) { Ok(stream) => return Ok(stream), - Err(error) => { - last_error = Some(error); - thread::sleep(Duration::from_millis(10)); - } - } + Err(error) => error, + }; + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(error.into()); + }; + thread::sleep(remaining.min(Duration::from_millis(10))); } - Err(last_error - .unwrap_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "host missing")) - .into()) } fn read_required_frame(reader: &mut impl Read, context: &str) -> anyhow::Result { @@ -1991,6 +2113,12 @@ mod unix { if args.iter().map(String::as_str).ne(["--bootstrap-stdio"]) { anyhow::bail!("hidden mode requires --bootstrap-stdio"); } + if let Ok(delay) = std::env::var("CMUX_TUI_TEST_BOOTSTRAP_READY_DELAY_MS") + && let Ok(delay) = delay.parse::() + && delay > 0 + { + thread::sleep(Duration::from_millis(delay.min(5_000))); + } let bootstrapped = crate::terminal_host::bootstrap_stdio_once(reader, writer)?; let Some(launch_frame) = read_frame(reader, MAX_LAUNCH_PAYLOAD)? else { // Keep the one-frame bootstrap probe useful for compatibility and @@ -3398,8 +3526,6 @@ mod unix { } } -#[cfg(unix)] -pub(crate) use unix::adopt_terminal_host_with_timeout; #[cfg(unix)] pub use unix::{ HostAttachment, adopt_terminal_host, isolate_terminal_host_process_fds, launch_terminal_host, @@ -3407,6 +3533,11 @@ pub use unix::{ remove_stale_terminal_host_record, serve_terminal_host_stdio, terminal_host_record_liveness, terminal_host_root, validate_terminal_host_record, }; +#[cfg(unix)] +pub(crate) use unix::{ + adopt_terminal_host_with_timeout, launch_terminal_host_cancellable, + launch_terminal_host_with_identity_cancellable, +}; #[cfg(not(unix))] pub fn terminal_host_root(state_root: &Path, session: &str) -> PathBuf { diff --git a/cmux-tui/crates/cmux-tui-core/tests/build_source_identity.rs b/cmux-tui/crates/cmux-tui-core/tests/build_source_identity.rs new file mode 100644 index 000000000000..a771ab321597 --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/tests/build_source_identity.rs @@ -0,0 +1,3 @@ +#[allow(dead_code)] +#[path = "../build.rs"] +mod build_script; diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 6a0ddb877b54..ed00a984959f 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -38,6 +38,7 @@ pub(crate) struct ServerMessages { pub stop_to_use: &'static str, pub stopping_exits_panes: &'static str, pub restart_before_command: &'static str, + pub restart_between_commands: &'static str, pub restart_after_command: &'static str, pub stopped: &'static str, pub missing_action: &'static str, @@ -211,7 +212,8 @@ static ENGLISH: Catalog = Catalog { stop_to_use: "Stop the old server to use this cmux-tui version.", stopping_exits_panes: "Stopping exits pane processes.", restart_before_command: "Run `", - restart_after_command: "`, then run `cmux-tui` again.", + restart_between_commands: "`, then run `", + restart_after_command: "` again.", stopped: "Stopped the cmux-tui server.", missing_action: "server needs an action: status or stop", unknown_action: "unknown server action", @@ -337,7 +339,8 @@ static JAPANESE: Catalog = Catalog { stop_to_use: "この cmux-tui バージョンを使うには、古いサーバーを停止してください。", stopping_exits_panes: "停止するとペインのプロセスが終了します。", restart_before_command: "`", - restart_after_command: "` を実行してから、`cmux-tui` をもう一度実行してください。", + restart_between_commands: "` を実行してから、`", + restart_after_command: "` をもう一度実行してください。", stopped: "cmux-tui サーバーを停止しました。", missing_action: "server には status または stop の操作が必要です", unknown_action: "不明な server 操作", diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 1c1ba4d25383..025bfadbaed9 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -7,7 +7,9 @@ use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; use cmux_tui_core::release::ReleaseIdentity; -use cmux_tui_core::server::{PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY}; +use cmux_tui_core::server::{ + PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, SERVER_SHUTDOWN_TIMEOUT, +}; use serde_json::{Value, json}; #[cfg(unix)] @@ -24,6 +26,9 @@ const LEGACY_STABLE_EMPTY_SCANS: usize = 2; #[cfg(unix)] const LEGACY_MAX_SCAN_ROUNDS: usize = 64; const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); +const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); +const SHUTDOWN_RESPONSE_TIMEOUT: Duration = + SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); type TransportReader = BufReader>; @@ -186,20 +191,27 @@ impl ServerLifecycle { return self.stop_legacy_server(); } + self.reader + .get_mut() + .set_read_timeout(Some(SHUTDOWN_RESPONSE_TIMEOUT)) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; write_json_line( self.reader.get_mut(), &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), ) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let accepted = match read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID) { - Ok(response) => response.get("ok").and_then(Value::as_bool) == Some(true), + let response = match read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID) { + Ok(response) => response, Err(_error) if connection_is_gone(&self.path) => return Ok(()), Err(error) => return Err(error), }; - if accepted { + if response.get("ok").and_then(Value::as_bool) == Some(true) { return wait_for_disconnect(&mut self.reader); } + if let Some(error) = response.get("error").and_then(Value::as_str) { + anyhow::bail!("{}: {error}", crate::localization::catalog().server.shutdown_failed); + } anyhow::bail!(crate::localization::catalog().server.shutdown_failed) } @@ -382,9 +394,16 @@ pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path .map(|reason| reason.message(messages)) .collect::>() .join(messages.reason_separator); - let command = format!("cmux-tui server stop --socket {}", shell_quote(path)); - let restart = - format!("{}{}{}", messages.restart_before_command, command, messages.restart_after_command); + let launcher = current_launcher_command(); + let command = format!("{launcher} server stop --socket {}", shell_quote(path)); + let restart = format!( + "{}{}{}{}{}", + messages.restart_before_command, + command, + messages.restart_between_commands, + launcher, + messages.restart_after_command + ); format!( "{}\n\n{}: v{} {} {}\n{}: v{} {} {}\n{}: {}\n\n{}\n{}\n{}", messages.incompatible_local_server, @@ -404,6 +423,15 @@ pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path ) } +fn current_launcher_command() -> String { + std::env::args_os() + .next() + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .map(|path| shell_quote(&path)) + .unwrap_or_else(|| "cmux-tui".to_string()) +} + pub(crate) fn write_json_line(writer: &mut dyn Write, value: &Value) -> std::io::Result<()> { serde_json::to_writer(&mut *writer, value).map_err(std::io::Error::other)?; writer.write_all(b"\n") @@ -414,7 +442,7 @@ fn read_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Resul } fn read_shutdown_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { - read_matching_response(reader, request_id, true) + read_matching_response_with_timeout(reader, request_id, true, SHUTDOWN_RESPONSE_TIMEOUT) } fn read_matching_response( @@ -422,7 +450,21 @@ fn read_matching_response( request_id: u64, accept_unidentified_error: bool, ) -> anyhow::Result { - let deadline = Instant::now() + RESPONSE_TIMEOUT; + read_matching_response_with_timeout( + reader, + request_id, + accept_unidentified_error, + RESPONSE_TIMEOUT, + ) +} + +fn read_matching_response_with_timeout( + reader: &mut TransportReader, + request_id: u64, + accept_unidentified_error: bool, + timeout: Duration, +) -> anyhow::Result { + let deadline = Instant::now() + timeout; let mut line = String::new(); loop { match reader.read_line(&mut line) { @@ -548,7 +590,12 @@ mod tests { assert!(message.contains("source build differs")); assert!(message.contains("terminal engine build differs")); assert!(message.contains("protocol differs")); - assert!(message.contains("cmux-tui server stop --socket '/tmp/test socket'")); + let launcher = current_launcher_command(); + assert!( + message.contains(&format!("{launcher} server stop --socket '/tmp/test socket'")), + "{message}" + ); + assert!(message.contains(&format!("then run `{launcher}` again")), "{message}"); assert!(message.contains("Stopping exits pane processes.")); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 3a0643a01f79..ef4ff851fe73 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -264,19 +264,35 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { #[cfg(target_os = "linux")] fn direct_child_pids(parent: libc::pid_t) -> io::Result> { - let path = format!("/proc/{parent}/task/{parent}/children"); - let children = match std::fs::read_to_string(path) { - Ok(children) => children, + let task_root = format!("/proc/{parent}/task"); + let tasks = match std::fs::read_dir(task_root) { + Ok(tasks) => tasks, Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), Err(error) => return Err(error), }; - let mut pids = children - .split_whitespace() - .map(|pid| { - pid.parse::().map_err(|_| io::Error::other("invalid child process id")) - }) - .collect::>>()?; - pids.retain(|pid| *pid > 1); + let mut pids = HashSet::new(); + for task in tasks { + let task = match task { + Ok(task) => task, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + let children = match std::fs::read_to_string(task.path().join("children")) { + Ok(children) => children, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + for pid in children.split_whitespace() { + let pid = pid + .parse::() + .map_err(|_| io::Error::other("invalid child process id"))?; + if pid > 1 { + pids.insert(pid); + } + } + } + let mut pids = pids.into_iter().collect::>(); + pids.sort_unstable(); Ok(pids) } diff --git a/cmux-tui/dist/npm/cmux/bin/cmux.js b/cmux-tui/dist/npm/cmux/bin/cmux.js index b8683888f50d..b00c4432ad01 100644 --- a/cmux-tui/dist/npm/cmux/bin/cmux.js +++ b/cmux-tui/dist/npm/cmux/bin/cmux.js @@ -40,7 +40,10 @@ try { process.exit(1); } -const result = spawnSync(binPath, process.argv.slice(2), { stdio: "inherit" }); +const result = spawnSync(binPath, process.argv.slice(2), { + argv0: "cmux", + stdio: "inherit", +}); if (result.error) { console.error(`cmux: failed to launch ${binPath}: ${result.error.message}`); diff --git a/cmux-tui/docs/protocol.md b/cmux-tui/docs/protocol.md index 888e74c871ff..dc1a2e53b0a4 100644 --- a/cmux-tui/docs/protocol.md +++ b/cmux-tui/docs/protocol.md @@ -120,9 +120,9 @@ When the stream ends, it sends: ## Client Compatibility -Direct local clients require the server's distribution version, source build, terminal-engine build, and protocol to match exactly. Ordinary source builds derive both build identities automatically, including a deterministic fingerprint when the source tree is dirty. `identify`, `cmux-tui server status`, and `cmux-tui server stop` bypass that check so an upgraded client can inspect and stop an older server. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. +Direct local clients require the server's distribution version, source build, terminal-engine build, and protocol to match exactly. Ordinary source builds derive both build identities automatically, including a deterministic fingerprint when the source tree is dirty. `identify`, ` server status`, and ` server stop` bypass that check so an upgraded client can inspect and stop an older server. `` is the current invocation, including `cmux-tui` for native binaries and `cmux` for packaged npm and Python installs. All other local CLI commands and TUI attach paths reject a mismatch before loading or mutating session state. -Current servers fence new surface creation, wait for in-flight creation, tombstone hosted terminals in one registry transaction, clear topology in one state mutation, and terminate pane runtimes before acknowledging shutdown. When an older local server lacks orderly shutdown, the replacement client starts a detached cleanup helper. The helper reconnects, verifies from the Unix socket that the same reported server PID is still the connected peer, closes every surface, and then signals only that verified process. Detaching lets cleanup finish when closing the caller's pane exits the original client. +Current servers apply one five-second deadline to fencing new surface creation, draining in-flight creation, tombstoning hosted terminals in one registry transaction, clearing topology in one state mutation, and terminating every tracked pane runtime. A successful shutdown response confirms that cleanup completed within the shared deadline. When an older local server lacks orderly shutdown, the replacement client starts a detached cleanup helper. The helper reconnects, verifies from the Unix socket that the same reported server PID is still the connected peer, closes every surface, and then signals only that verified process. Detaching lets cleanup finish when closing the caller's pane exits the original client. Remote transports require protocol v9 but do not require the same release build. They reject protocol-v8 servers before loading their workspace tree because v8 does not define stack layout nodes or `new-pane`. diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index acd6f47e4c41..a63703708ec3 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -248,7 +248,7 @@ Example: | status | implemented | | since | protocol 9 additive extension | -Requests an orderly server exit after the success response is queued. The command is accepted only on the local Unix-domain socket. Stopping the mux exits every pane process. Shutdown first fences new surface creation and waits for in-flight creation. It then tombstones all hosted terminals in one registry transaction, clears the complete topology in one state mutation, and terminates pane runtimes before returning success. +Requests an orderly server exit after the success response is queued. The command is accepted only on the local Unix-domain socket. Stopping the mux exits every pane process. Shutdown uses one five-second deadline to fence new surface creation, drain in-flight creation, tombstone all hosted terminals in one registry transaction, clear the complete topology in one state mutation, and terminate pane runtimes. A success response means every tracked pane process was confirmed terminated before that deadline. Params: none. @@ -261,7 +261,7 @@ Errors: | `shutdown is only available over the local session socket` | Request arrived over WebSocket | | `bad request: ...` | Malformed request envelope | -CLI mapping: `cmux-tui server stop`; `cmux-tui server status` uses `identify` and does not issue this command. When an older Unix server does not implement `shutdown`, a newer client verifies that the PID reported by `identify` owns the connected Unix socket, then starts a detached helper. The helper reconnects, verifies the same PID again, closes every surface reported by `list-workspaces`, and signals that verified process. Cleanup survives the caller's pane exit. If either socket-owner check or orderly pane cleanup fails, the helper leaves the server running and reports the failure. +CLI mapping: ` server stop`; ` server status` uses `identify` and does not issue this command. `` is the current executable invocation, such as `cmux-tui` for a native binary or `cmux` for the packaged npm and Python launchers. Upgrade recovery instructions use that same launcher for both stop and restart commands. When an older Unix server does not implement `shutdown`, a newer client verifies that the PID reported by `identify` owns the connected Unix socket, then starts a detached helper. The helper reconnects, verifies the same PID again, closes every surface reported by `list-workspaces`, and signals that verified process. Cleanup survives the caller's pane exit. If either socket-owner check or orderly pane cleanup fails, the helper leaves the server running and reports the failure. `cmux-tui server status` compares the local server's distribution version, source build, terminal-engine build, and protocol with the client. Plain output reports both versions and protocols plus localized mismatch reasons. JSON output is: From bc5aa1bdbd64bbf00dc826f49b6eb8852698b2a3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:43:21 -0700 Subject: [PATCH 011/521] test(tui): cover shutdown closeout regressions --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 38 ++++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 29 +++++++++++++++ tests/test_tui_npm_package_artifact.py | 12 +++++++ 3 files changed, 79 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 641e056b8999..a6dcb1c2bac1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -816,6 +816,8 @@ pub struct Mux { terminal_create_after_materialization_lock: Mutex>>, #[cfg(test)] terminal_create_after_workspace_reservation: Mutex>>, + #[cfg(test)] + browser_bootstrap_before_runtime: Mutex>>, browser_runtime: Mutex>>, cell_pixels: Mutex<(u16, u16)>, default_colors: Mutex, @@ -1046,6 +1048,8 @@ impl Mux { terminal_create_after_materialization_lock: Mutex::new(None), #[cfg(test)] terminal_create_after_workspace_reservation: Mutex::new(None), + #[cfg(test)] + browser_bootstrap_before_runtime: Mutex::new(None), browser_runtime: Mutex::new(None), cell_pixels: Mutex::new((8, 16)), default_colors: Mutex::new(DefaultColors::default()), @@ -3220,6 +3224,10 @@ impl Mux { let id = surface.id; let _ = std::thread::Builder::new().name(format!("browser-surface-{id}-bootstrap")).spawn( move || { + #[cfg(test)] + if let Some(hook) = mux.browser_bootstrap_before_runtime.lock().unwrap().clone() { + hook(); + } let result = (|| -> anyhow::Result<()> { let runtime = match runtime { Some(runtime) => runtime, @@ -9892,6 +9900,36 @@ mod tests { assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); } + #[test] + fn server_shutdown_waits_for_async_browser_bootstrap() { + let mux = test_mux(); + let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); + let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); + let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); + *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ + move || { + bootstrap_reached_tx.send(()).unwrap(); + release_bootstrap_rx.lock().unwrap().recv().unwrap(); + } + })); + mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); + bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); + let shutdown = std::thread::spawn({ + let mux = mux.clone(); + move || { + shutdown_done_tx.send(mux.close_all_surfaces_for_shutdown()).unwrap(); + } + }); + assert!(shutdown_done_rx.recv_timeout(Duration::from_millis(100)).is_err()); + + release_bootstrap_tx.send(()).unwrap(); + assert_eq!(shutdown_done_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(), 1); + shutdown.join().unwrap(); + assert!(mux.browser_runtime.lock().unwrap().is_none()); + } + #[test] fn shutdown_fanout_runs_terminations_concurrently() { let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 5d6a50b50dbf..c3eb0b7dea0f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2880,6 +2880,35 @@ fn terminal_scroll_position(term: &Terminal) -> (u64, bool) { mod tests { use super::*; + #[cfg(unix)] + #[derive(Debug)] + struct CountingChildKiller(Arc); + + #[cfg(unix)] + impl ChildKiller for CountingChildKiller { + fn kill(&mut self) -> std::io::Result<()> { + self.0.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + fn clone_killer(&self) -> Box { + Box::new(Self(self.0.clone())) + } + } + + #[cfg(unix)] + #[test] + fn reaped_local_child_is_never_signaled_again() { + let signals = Arc::new(AtomicUsize::new(0)); + let process = LocalPtyProcess::new(None, Box::new(CountingChildKiller(signals.clone()))); + process.child_reaped.store(true, Ordering::Release); + + assert!(process.request_hangup()); + assert_eq!(signals.load(Ordering::Relaxed), 0); + drop(process); + assert_eq!(signals.load(Ordering::Relaxed), 0); + } + #[cfg(unix)] #[test] fn hosted_mirror_never_answers_terminal_queries() { diff --git a/tests/test_tui_npm_package_artifact.py b/tests/test_tui_npm_package_artifact.py index aa4ae6ac3a1c..a2b021aa0951 100644 --- a/tests/test_tui_npm_package_artifact.py +++ b/tests/test_tui_npm_package_artifact.py @@ -84,12 +84,24 @@ def test_publish_workflows_restore_the_mode_preserving_archive() -> None: assert "--archive dist/npm-packages.tar.gz" in workflow +def test_raw_artifacts_do_not_inherit_the_packaging_version() -> None: + build = (ROOT / ".github/workflows/cmux-tui-build-package.yml").read_text() + conditional_stamp = ( + "CMUX_TUI_DISTRIBUTION_VERSION: " + "${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }}" + ) + + assert "CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }}" not in build + assert build.count(conditional_stamp) == 3 + + def main() -> None: with tempfile.TemporaryDirectory() as directory: test_archive_round_trip_preserves_package_executables(Path(directory)) with tempfile.TemporaryDirectory() as directory: test_extract_rejects_paths_outside_package_root(Path(directory)) test_publish_workflows_restore_the_mode_preserving_archive() + test_raw_artifacts_do_not_inherit_the_packaging_version() if __name__ == "__main__": From c7e25889c652fbbc2d65e54a9c256f26a146c628 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:46:45 -0700 Subject: [PATCH 012/521] fix(tui): preserve upgrade identity and ownership --- .github/workflows/cmux-tui-build-package.yml | 6 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 126 +++++++++++++++++-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 14 +++ 3 files changed, 132 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index a3437ab1f40e..d6259d9b4ab7 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -104,7 +104,7 @@ jobs: working-directory: cmux-tui shell: bash env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} + CMUX_TUI_DISTRIBUTION_VERSION: ${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }} run: | # Package artifacts must build and stamp the checked-out submodule, # independent of any self-hosted runner source override. @@ -119,7 +119,7 @@ jobs: working-directory: cmux-tui shell: bash env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} + CMUX_TUI_DISTRIBUTION_VERSION: ${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }} run: | # Package artifacts must build and stamp the checked-out submodule, # independent of any self-hosted runner source override. @@ -182,7 +182,7 @@ jobs: - name: Build libghostty-vt + cmux-tui (Windows GNU) shell: bash env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} + CMUX_TUI_DISTRIBUTION_VERSION: ${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }} run: | # Keep the manual Zig build and Cargo's build script on the same # checked-out Ghostty source whose revision is stamped below. diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a6dcb1c2bac1..d08b2a3e54f9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -664,6 +664,70 @@ impl Drop for SurfaceCreationGuard<'_> { } } +#[derive(Default)] +struct AsyncSurfaceCreationState { + shutting_down: bool, + active: usize, +} + +#[derive(Default)] +struct AsyncSurfaceCreationGate { + inner: Arc, +} + +#[derive(Default)] +struct AsyncSurfaceCreationGateInner { + state: Mutex, + idle: std::sync::Condvar, +} + +impl AsyncSurfaceCreationGate { + fn begin(&self) -> anyhow::Result { + let mut state = self.inner.state.lock().unwrap(); + if state.shutting_down { + anyhow::bail!("server is shutting down"); + } + state.active = state.active.saturating_add(1); + Ok(AsyncSurfaceCreationGuard { gate: self.inner.clone() }) + } + + fn stop(&self) { + let mut state = self.inner.state.lock().unwrap(); + state.shutting_down = true; + self.inner.idle.notify_all(); + } + + fn stop_and_wait_until(&self, deadline: Instant) -> bool { + let mut state = self.inner.state.lock().unwrap(); + state.shutting_down = true; + while state.active != 0 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.inner.idle.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.active != 0 { + return false; + } + } + true + } +} + +struct AsyncSurfaceCreationGuard { + gate: Arc, +} + +impl Drop for AsyncSurfaceCreationGuard { + fn drop(&mut self) { + let mut state = self.gate.state.lock().unwrap(); + state.active = state.active.checked_sub(1).expect("async surface creation guard tracked"); + if state.active == 0 { + self.gate.idle.notify_all(); + } + } +} + enum SurfaceResizeRestore { Complete(bool), Pending(Receiver), @@ -786,6 +850,7 @@ pub struct Mux { subscribers: MuxEventBroadcaster, shutdown_requested: AtomicBool, surface_creations: SurfaceCreationGate, + async_surface_creations: AsyncSurfaceCreationGate, shutdown_coordinator: Mutex<()>, shutdown_surfaces: Mutex>>, next_id: AtomicU64, @@ -1018,6 +1083,7 @@ impl Mux { subscribers: MuxEventBroadcaster::default(), shutdown_requested: AtomicBool::new(false), surface_creations: SurfaceCreationGate::default(), + async_surface_creations: AsyncSurfaceCreationGate::default(), shutdown_coordinator: Mutex::new(()), shutdown_surfaces: Mutex::new(Vec::new()), next_id: AtomicU64::new(next_id), @@ -3205,11 +3271,18 @@ impl Mux { fn browser_runtime(&self) -> anyhow::Result> { let mut runtime = self.browser_runtime.lock().unwrap(); + if self.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } if let Some(existing) = runtime.as_ref().filter(|existing| !existing.is_closed()) { return Ok(existing.clone()); } let opts = self.surface_options.lock().unwrap().clone(); let created = BrowserRuntime::connect(&opts)?; + if self.is_shutting_down() { + created.shutdown(); + anyhow::bail!("server is shutting down"); + } *runtime = Some(created.clone()); Ok(created) } @@ -3220,31 +3293,57 @@ impl Mux { bootstrap: BrowserBootstrap, runtime: Option>, ) { + let bootstrap_guard = match self.async_surface_creations.begin() { + Ok(guard) => guard, + Err(error) => { + self.report_browser_bootstrap_failure(&surface, error.to_string()); + return; + } + }; let mux = self.clone(); let id = surface.id; - let _ = std::thread::Builder::new().name(format!("browser-surface-{id}-bootstrap")).spawn( - move || { + let worker_surface = surface.clone(); + let worker = std::thread::Builder::new() + .name(format!("browser-surface-{id}-bootstrap")) + .spawn(move || { + let _bootstrap_guard = bootstrap_guard; #[cfg(test)] if let Some(hook) = mux.browser_bootstrap_before_runtime.lock().unwrap().clone() { hook(); } let result = (|| -> anyhow::Result<()> { + if mux.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } let runtime = match runtime { Some(runtime) => runtime, None => mux.browser_runtime()?, }; - runtime.bootstrap_surface_sync(surface.clone(), bootstrap, Arc::downgrade(&mux)) + if mux.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } + runtime.bootstrap_surface_sync( + worker_surface.clone(), + bootstrap, + Arc::downgrade(&mux), + ) })(); if let Err(err) = result { - if let Surface::Browser(browser) = surface.as_ref() { - browser.mark_failed(err.to_string()); - } - mux.emit(MuxEvent::Status(format!("browser failed: {err}"))); - mux.emit(MuxEvent::TitleChanged { surface: id, title: surface.title().into() }); - mux.emit(MuxEvent::SurfaceOutput(id)); + mux.report_browser_bootstrap_failure(&worker_surface, err.to_string()); } - }, - ); + }); + if let Err(error) = worker { + self.report_browser_bootstrap_failure(&surface, error.to_string()); + } + } + + fn report_browser_bootstrap_failure(&self, surface: &Arc, error: String) { + if let Surface::Browser(browser) = surface.as_ref() { + browser.mark_failed(error.clone()); + } + self.emit(MuxEvent::Status(format!("browser failed: {error}"))); + self.emit(MuxEvent::TitleChanged { surface: surface.id, title: surface.title().into() }); + self.emit(MuxEvent::SurfaceOutput(surface.id)); } /// A fresh single-tab pane wrapping `surface`. @@ -3642,6 +3741,7 @@ impl Mux { self.shutting_down.store(true, Ordering::Release); let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; let _ = self.surface_creations.stop_and_wait_until(deadline); + let _ = self.async_surface_creations.stop_and_wait_until(deadline); let mut surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); surfaces.extend(self.shutdown_surfaces.lock().unwrap().drain(..)); @@ -3674,6 +3774,9 @@ impl Mux { if !self.surface_creations.stop_and_wait_until(deadline) { anyhow::bail!("surface creation did not stop before the shutdown deadline"); } + if !self.async_surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("browser bootstrap did not stop before the shutdown deadline"); + } #[cfg(unix)] let terminal_host_records = { @@ -3829,6 +3932,7 @@ impl Mux { pub fn request_daemon_shutdown(&self) { self.shutting_down.store(true, Ordering::Release); self.surface_creations.stop(); + self.async_surface_creations.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index c3eb0b7dea0f..5ef4d8b071d4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -735,6 +735,15 @@ impl LocalPtyProcess { } fn request_hangup(&self) -> bool { + #[cfg(unix)] + { + let _signal = self.child_signal_lock.lock().unwrap(); + if self.child_reaped.load(Ordering::Acquire) { + return true; + } + return self.killer.lock().unwrap().kill().is_ok(); + } + #[cfg(not(unix))] self.killer.lock().unwrap().kill().is_ok() } @@ -829,6 +838,11 @@ impl LocalPtyProcess { impl Drop for LocalPtyProcess { fn drop(&mut self) { + #[cfg(unix)] + if !*self.child_reaped.get_mut() { + let _ = self.killer.get_mut().map(|killer| killer.kill()); + } + #[cfg(not(unix))] let _ = self.killer.get_mut().map(|killer| killer.kill()); } } From ca5f5d5a0b046c7f014b70edf8aa64f4d863a34f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:58:06 -0700 Subject: [PATCH 013/521] test(tui): cover complete PTY session shutdown --- cmux-tui/crates/cmux-tui/tests/cli.rs | 69 ++++++++++++++++ .../cmux-tui/tests/terminal_host_recovery.rs | 80 +++++++++++++++++++ 2 files changed, 149 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 926ab554bcbc..0b3e335776f5 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1038,6 +1038,75 @@ fn server_stop_kills_ephemeral_pty_process_groups() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { + let mut server = + HeadlessServer::start_with("server-stop-ephemeral-job-control-group", true, &[]); + let descendant_pid_file = server.dir.join("ephemeral-background-job.pid"); + let command = format!( + concat!( + "exec /bin/bash --noprofile --norc -i -c '", + "trap \"\" HUP; ", + "(trap \"\" HUP; while :; do sleep 60; done) & ", + "printf \"%s\" \"$!\" > {}; wait'", + ), + descendant_pid_file.display() + ); + let run = cli(&server, &["run", "--command", &command]); + assert_success(&run); + let surface = String::from_utf8(run.stdout).unwrap().trim().parse::().unwrap(); + let info = cli(&server, &["--json", "process-info", "--surface", &surface.to_string()]); + assert_success(&info); + let direct_pid = u32::try_from( + serde_json::from_slice::(&info.stdout).unwrap()["pid"].as_u64().unwrap(), + ) + .unwrap(); + let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + let direct_pid_raw = libc::pid_t::try_from(direct_pid).unwrap(); + let descendant_pid_raw = libc::pid_t::try_from(descendant_pid).unwrap(); + // SAFETY: both fixture processes are live and owned by this test. + let direct_group = unsafe { libc::getpgid(direct_pid_raw) }; + // SAFETY: both fixture processes are live and owned by this test. + let descendant_group = unsafe { libc::getpgid(descendant_pid_raw) }; + // SAFETY: both fixture processes are live and owned by this test. + let direct_session = unsafe { libc::getsid(direct_pid_raw) }; + // SAFETY: both fixture processes are live and owned by this test. + let descendant_session = unsafe { libc::getsid(descendant_pid_raw) }; + + let stop = cli(&server, &["--json", "server", "stop"]); + + assert_success(&stop); + assert!(server.child.wait().unwrap().success()); + let deadline = Instant::now() + Duration::from_secs(1); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let descendant_remained = process_exists(descendant_pid); + if descendant_remained { + // SAFETY: the live process and group belong to this isolated fixture. + let _ = unsafe { libc::killpg(descendant_group, libc::SIGKILL) }; + // SAFETY: the live process belongs to this isolated fixture. + let _ = unsafe { libc::kill(descendant_pid_raw, libc::SIGKILL) }; + } + + assert!(direct_group > 0); + assert!(descendant_group > 0); + assert_ne!( + descendant_group, direct_group, + "fixture background job did not enter a separate process group" + ); + assert_eq!( + descendant_session, direct_session, + "fixture background job left the owned PTY session" + ); + assert!(!process_exists(direct_pid)); + assert!( + !descendant_remained, + "server acknowledged shutdown while a background job in its PTY session remained alive" + ); +} + #[cfg(unix)] #[test] fn server_stop_drains_many_hosted_panes_before_acknowledging() { diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 0541e07bcd3d..be62b9aee6e7 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -506,6 +506,86 @@ fn explicit_terminate_reaps_descendants_in_the_pty_group() { wait_for_process_and_group_absent(descendant_pid); } +#[test] +fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { + let harness = RecoveryHarness::start("terminate-job-control-group"); + let marker = format!("job-control-background-ready-{}", std::process::id()); + let descendant_pid_path = harness.dir.join("job-control-background.pid"); + let created = request( + &harness.socket, + serde_json::json!({ + "id": 1, + "cmd": "run", + "argv": [ + "/bin/bash", + "--noprofile", + "--norc", + "-i", + "-c", + concat!( + "trap '' HUP; ", + "(trap '' HUP; while :; do sleep 60; done) & ", + "printf '%s' \"$!\" > \"$1\"; printf '%s\\n' \"$2\"; wait", + ), + "cmux-job-control-background", + descendant_pid_path, + marker, + ], + "new_workspace": true, + "name": "job-control-background", + }), + ); + let surface = created["surface"].as_u64().unwrap(); + assert!(wait_for_screen(&harness.socket, surface, &marker).contains(&marker)); + let descendant_pid = wait_for_pid_file(&harness.dir.join("job-control-background.pid")); + let (record_path, record) = wait_for_host_records(&harness.host_root(), 1).remove(0); + let observer = adopt_terminal_host(record.clone(), record_path.clone()).unwrap(); + let direct_pid = observer.snapshot.pid.unwrap() as libc::pid_t; + observer.disconnect(); + assert!(process_exists(direct_pid), "direct PTY child exited before Terminate"); + assert!(process_exists(descendant_pid), "background job exited before Terminate"); + // SAFETY: both fixture processes are live and owned by this test. + let direct_group = unsafe { libc::getpgid(direct_pid) }; + // SAFETY: both fixture processes are live and owned by this test. + let descendant_group = unsafe { libc::getpgid(descendant_pid) }; + // SAFETY: both fixture processes are live and owned by this test. + let direct_session = unsafe { libc::getsid(direct_pid) }; + // SAFETY: both fixture processes are live and owned by this test. + let descendant_session = unsafe { libc::getsid(descendant_pid) }; + + let host = adopt_terminal_host(record.clone(), record_path.clone()).unwrap(); + host.terminate().unwrap(); + host.disconnect(); + wait_for_no_host_records(&harness.host_root()); + let deadline = Instant::now() + Duration::from_secs(1); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let descendant_remained = process_exists(descendant_pid); + if descendant_remained { + // SAFETY: the live process and group belong to this isolated fixture. + let _ = unsafe { libc::killpg(descendant_group, libc::SIGKILL) }; + // SAFETY: the live process belongs to this isolated fixture. + let _ = unsafe { libc::kill(descendant_pid, libc::SIGKILL) }; + } + + assert!(direct_group > 0); + assert!(descendant_group > 0); + assert_ne!( + descendant_group, direct_group, + "fixture background job did not enter a separate process group" + ); + assert_eq!( + descendant_session, direct_session, + "fixture background job left the owned PTY session" + ); + wait_for_process_and_group_absent(direct_pid); + assert!( + !descendant_remained, + "terminal host exited after killing only the shell and foreground process groups" + ); +} + #[test] fn exit_follows_all_final_pty_bytes_on_the_live_stream() { let harness = RecoveryHarness::start("exit-after-final-bytes"); From 0c367dd01e17c5da72d603c030997b74e514b94c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:01:17 -0700 Subject: [PATCH 014/521] fix(tui): own complete PTY sessions and build inputs --- cmux-tui/crates/cmux-tui-core/build.rs | 100 +++++++--- cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 + .../cmux-tui-core/src/process_session.rs | 173 ++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 61 +++--- .../src/terminal_host_runtime.rs | 121 ++++++------ .../cmux-tui/tests/terminal_host_recovery.rs | 2 +- cmux-tui/crates/ghostty-vt-sys/build.rs | 1 + 7 files changed, 353 insertions(+), 107 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/process_session.rs diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index aca025340d83..8d51e93015a8 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -8,6 +8,8 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; const FNV_OFFSET: u64 = 0xcbf29ce484222325; const FNV_PRIME: u64 = 0x100000001b3; +const CMUX_TUI_SOURCE_PATHS: &[&str] = &["Cargo.toml", "Cargo.lock", "bindings/rust", "crates"]; +const GHOSTTY_SOURCE_PATHS: &[&str] = &["build.zig", "build.zig.zon", "include", "src"]; fn main() { for name in [ @@ -30,22 +32,18 @@ fn main() { .map(PathBuf::from) .unwrap_or_else(|| repository_root.join("ghostty")); - println!("cargo:rerun-if-changed={}", workspace_root.join("Cargo.toml").display()); - println!("cargo:rerun-if-changed={}", workspace_root.join("Cargo.lock").display()); - println!("cargo:rerun-if-changed={}", workspace_root.join("crates").display()); - println!("cargo:rerun-if-changed={}", ghostty_root.join("build.zig").display()); - println!("cargo:rerun-if-changed={}", ghostty_root.join("include").display()); - println!("cargo:rerun-if-changed={}", ghostty_root.join("src").display()); + track_source_paths(workspace_root, CMUX_TUI_SOURCE_PATHS); + track_source_paths(&ghostty_root, GHOSTTY_SOURCE_PATHS); track_git_identity(repository_root); track_git_identity(&ghostty_root); if !nonempty_env("CMUX_TUI_BUILD_COMMIT") && !nonempty_env("CMUX_MUX_BUILD_COMMIT") { - let identity = source_identity(workspace_root, workspace_root) + let identity = source_identity(workspace_root, workspace_root, CMUX_TUI_SOURCE_PATHS) .expect("derive an exact cmux-tui source identity"); println!("cargo:rustc-env=CMUX_TUI_SOURCE_COMMIT={identity}"); } if !nonempty_env("CMUX_TUI_GHOSTTY_COMMIT") { - let identity = source_identity(&ghostty_root, &ghostty_root) + let identity = source_identity(&ghostty_root, &ghostty_root, GHOSTTY_SOURCE_PATHS) .expect("derive an exact Ghostty source identity"); println!("cargo:rustc-env=CMUX_TUI_SOURCE_GHOSTTY_COMMIT={identity}"); } @@ -55,7 +53,13 @@ fn nonempty_env(name: &str) -> bool { env::var_os(name).is_some_and(|value| !value.is_empty()) } -fn source_identity(git_root: &Path, fallback_root: &Path) -> io::Result { +fn track_source_paths(root: &Path, paths: &[&str]) { + for path in paths { + println!("cargo:rerun-if-changed={}", root.join(path).display()); + } +} + +fn source_identity(git_root: &Path, fallback_root: &Path, paths: &[&str]) -> io::Result { if let Some(commit) = optional_git_bytes(git_root, &["rev-parse", "HEAD"])? { let commit = std::str::from_utf8(&commit) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "git commit is not UTF-8"))? @@ -63,26 +67,31 @@ fn source_identity(git_root: &Path, fallback_root: &Path) -> io::Result if commit.is_empty() { return Err(io::Error::new(io::ErrorKind::InvalidData, "git commit is empty")); } - let status = required_git_bytes( + let status = required_git_bytes_for_paths( git_root, - &["status", "--porcelain=v1", "-z", "--untracked-files=all", "--", "."], + &["status", "--porcelain=v1", "-z", "--untracked-files=all"], + paths, )?; if status.is_empty() { return Ok(commit.to_string()); } - return Ok(format!("{commit}-dirty-{:016x}", dirty_fingerprint(git_root)?)); + return Ok(format!("{commit}-dirty-{:016x}", dirty_fingerprint(git_root, paths)?)); } - directory_fingerprint(fallback_root).map(|fingerprint| format!("source-{fingerprint:016x}")) + directory_fingerprint(fallback_root, paths) + .map(|fingerprint| format!("source-{fingerprint:016x}")) } -fn dirty_fingerprint(root: &Path) -> io::Result { +fn dirty_fingerprint(root: &Path, paths: &[&str]) -> io::Result { let mut hash = FNV_OFFSET; hash_component( &mut hash, - &required_git_bytes(root, &["diff", "--binary", "--full-index", "HEAD", "--", "."])?, + &required_git_bytes_for_paths(root, &["diff", "--binary", "--full-index", "HEAD"], paths)?, ); - let untracked = - required_git_bytes(root, &["ls-files", "-z", "--others", "--exclude-standard", "--", "."])?; + let untracked = required_git_bytes_for_paths( + root, + &["ls-files", "-z", "--others", "--exclude-standard"], + paths, + )?; for path in untracked.split(|byte| *byte == 0).filter(|path| !path.is_empty()) { hash_component(&mut hash, path); hash_filesystem_entry(&mut hash, &path_from_git_bytes(root, path)?)?; @@ -90,7 +99,7 @@ fn dirty_fingerprint(root: &Path) -> io::Result { Ok(hash) } -fn directory_fingerprint(root: &Path) -> io::Result { +fn directory_fingerprint(root: &Path, paths: &[&str]) -> io::Result { fn visit(root: &Path, path: &Path, hash: &mut u64) -> io::Result<()> { let mut entries = fs::read_dir(path)?.collect::, _>>()?; entries.sort_by_key(|entry| entry.file_name()); @@ -118,7 +127,19 @@ fn directory_fingerprint(root: &Path) -> io::Result { } let mut hash = FNV_OFFSET; - visit(root, root, &mut hash)?; + for path in paths { + let path = root.join(path); + let metadata = fs::symlink_metadata(&path)?; + if metadata.file_type().is_dir() { + visit(root, &path, &mut hash)?; + } else { + hash_component( + &mut hash, + path.strip_prefix(root).unwrap_or(&path).as_os_str().as_encoded_bytes(), + ); + hash_filesystem_entry(&mut hash, &path)?; + } + } Ok(hash) } @@ -191,6 +212,14 @@ fn required_git_bytes(root: &Path, args: &[&str]) -> io::Result> { }) } +fn required_git_bytes_for_paths(root: &Path, args: &[&str], paths: &[&str]) -> io::Result> { + let mut scoped = Vec::with_capacity(args.len() + paths.len() + 1); + scoped.extend_from_slice(args); + scoped.push("--"); + scoped.extend_from_slice(paths); + required_git_bytes(root, &scoped) +} + #[cfg(unix)] fn path_from_git_bytes(root: &Path, path: &[u8]) -> io::Result { Ok(root.join(OsStr::from_bytes(path))) @@ -255,10 +284,10 @@ mod tests { let name = OsString::from_vec(vec![b'n', b'o', b'n', b'-', 0xff]); let path = repository.root.join(name); fs::write(&path, b"one").unwrap(); - let first = source_identity(&repository.root, &repository.root).unwrap(); + let first = source_identity(&repository.root, &repository.root, &["."]).unwrap(); fs::write(path, b"two").unwrap(); - let second = source_identity(&repository.root, &repository.root).unwrap(); + let second = source_identity(&repository.root, &repository.root, &["."]).unwrap(); assert!(first.contains("-dirty-")); assert!(second.contains("-dirty-")); @@ -271,11 +300,11 @@ mod tests { let repository = TestRepository::new("dangling-symlink"); let link = repository.root.join("untracked-link"); symlink("missing-one", &link).unwrap(); - let first = source_identity(&repository.root, &repository.root).unwrap(); + let first = source_identity(&repository.root, &repository.root, &["."]).unwrap(); fs::remove_file(&link).unwrap(); symlink("missing-two", &link).unwrap(); - let second = source_identity(&repository.root, &repository.root).unwrap(); + let second = source_identity(&repository.root, &repository.root, &["."]).unwrap(); assert_ne!(first, second); } @@ -285,6 +314,29 @@ mod tests { let repository = TestRepository::new("git-failure"); fs::write(repository.root.join(".git/index"), b"invalid index").unwrap(); - assert!(source_identity(&repository.root, &repository.root).is_err()); + assert!(source_identity(&repository.root, &repository.root, &["."]).is_err()); + } + + #[test] + fn identity_and_invalidation_share_the_same_source_scope() { + let repository = TestRepository::new("scoped"); + fs::create_dir(repository.root.join("source")).unwrap(); + fs::create_dir(repository.root.join("docs")).unwrap(); + fs::write(repository.root.join("source/input"), b"one").unwrap(); + fs::write(repository.root.join("docs/guide"), b"one").unwrap(); + run_git(&repository.root, &["add", "source", "docs"]); + run_git(&repository.root, &["commit", "-qm", "add scoped inputs"]); + let clean = source_identity(&repository.root, &repository.root, &["source"]).unwrap(); + + fs::write(repository.root.join("docs/guide"), b"two").unwrap(); + let docs_changed = + source_identity(&repository.root, &repository.root, &["source"]).unwrap(); + fs::write(repository.root.join("source/input"), b"two").unwrap(); + let source_changed = + source_identity(&repository.root, &repository.root, &["source"]).unwrap(); + + assert_eq!(docs_changed, clean); + assert!(source_changed.contains("-dirty-")); + assert_ne!(source_changed, clean); } } diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index ef3cf5ac939c..36326e9cf94a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -13,6 +13,8 @@ mod event_bus; mod model; mod mux; mod pairing; +#[cfg(unix)] +mod process_session; pub mod provider_management; mod short_id; mod surface; diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs new file mode 100644 index 000000000000..049aed94e2b4 --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -0,0 +1,173 @@ +//! Exact ownership helpers for Unix PTY sessions. +//! +//! `portable-pty` starts each terminal child as a session leader. Process +//! groups inside that session are transient job-control details, so shutdown +//! enumerates and signals every session member instead of guessing which +//! foreground or background groups still exist. + +use std::io; +use std::time::{Duration, Instant}; + +const SESSION_KILL_POLL: Duration = Duration::from_millis(5); + +pub(crate) fn signal(session: libc::pid_t, signal: libc::c_int) -> io::Result<()> { + validate_session_target(session)?; + for pid in members(session)? { + signal_if_still_member(pid, session, signal)?; + } + Ok(()) +} + +/// Repeatedly SIGKILL every member until the session contains only its +/// reserved leader (or is empty). Callers must keep the leader unreaped for +/// this entire operation, which prevents the session id from being reused. +pub(crate) fn kill_until_only_leader( + session: libc::pid_t, + leader: libc::pid_t, + deadline: Instant, + leader_exited: impl Fn() -> bool, +) -> io::Result { + validate_session_target(session)?; + loop { + let session_members = members(session)?; + for pid in &session_members { + signal_if_still_member(*pid, session, libc::SIGKILL)?; + } + // Once WNOWAIT has observed the leader's exit, no process can fork a + // new member between this complete session scan and success. + if leader_exited() && session_members.iter().all(|pid| *pid == leader) { + return Ok(true); + } + if Instant::now() >= deadline { + return Ok(false); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(SESSION_KILL_POLL), + ); + } +} + +fn validate_session_target(session: libc::pid_t) -> io::Result<()> { + if session <= 1 { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid PTY session id")); + } + // SAFETY: getsid(0) only queries the calling process. + if unsafe { libc::getsid(0) } == session { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "refusing to signal the mux process session", + )); + } + Ok(()) +} + +fn signal_if_still_member( + pid: libc::pid_t, + session: libc::pid_t, + signal: libc::c_int, +) -> io::Result<()> { + // Revalidate membership immediately before signaling. The reserved + // session leader prevents the session id itself from being recycled. + // SAFETY: getsid only queries process metadata. + let current_session = unsafe { libc::getsid(pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(()) } else { Err(error) }; + } + if current_session != session { + return Ok(()); + } + // SAFETY: membership was revalidated above and `signal` is a platform + // signal constant supplied by this module's callers. + if unsafe { libc::kill(pid, signal) } == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { Ok(()) } else { Err(error) } +} + +fn members(session: libc::pid_t) -> io::Result> { + let mut members = Vec::new(); + for pid in all_process_ids()? { + if pid <= 1 { + continue; + } + // SAFETY: getsid only queries process metadata. + let current_session = unsafe { libc::getsid(pid) }; + if current_session == session { + members.push(pid); + continue; + } + if current_session < 0 { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error); + } + } + } + members.sort_unstable(); + members.dedup(); + Ok(members) +} + +#[cfg(target_os = "macos")] +fn all_process_ids() -> io::Result> { + use std::ffi::c_void; + use std::mem::size_of; + + // SAFETY: a null buffer asks libproc for the current process count. + let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if count < 0 { + return Err(io::Error::last_os_error()); + } + let mut capacity = + usize::try_from(count).map_err(|_| io::Error::other("invalid process count"))? + 32; + for _ in 0..4 { + let mut pids = vec![0; capacity]; + let bytes = capacity + .checked_mul(size_of::()) + .and_then(|bytes| i32::try_from(bytes).ok()) + .ok_or_else(|| io::Error::other("process list buffer overflow"))?; + // SAFETY: `pids` owns a writable buffer of `bytes` bytes. + let listed = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast::(), bytes) }; + if listed < 0 { + return Err(io::Error::last_os_error()); + } + let listed = + usize::try_from(listed).map_err(|_| io::Error::other("invalid process count"))?; + if listed < capacity { + pids.truncate(listed); + return Ok(pids); + } + capacity = capacity + .checked_mul(2) + .ok_or_else(|| io::Error::other("process list buffer overflow"))?; + } + Err(io::Error::other("process list did not stabilize")) +} + +#[cfg(target_os = "linux")] +fn all_process_ids() -> io::Result> { + let mut pids = Vec::new(); + for entry in std::fs::read_dir("/proc")? { + let entry = match entry { + Ok(entry) => entry, + Err(error) if error.kind() == io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + if let Some(pid) = + entry.file_name().to_str().and_then(|name| name.parse::().ok()) + { + pids.push(pid); + } + } + Ok(pids) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn all_process_ids() -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "PTY session enumeration is unavailable on this platform", + )) +} diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 5ef4d8b071d4..9c2e3322f339 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -626,9 +626,9 @@ impl LocalProcess { } } - fn terminate_and_wait(&mut self, master: &dyn MasterPty, deadline: Instant) -> bool { + fn terminate_and_wait(&mut self, deadline: Instant) -> bool { match self { - Self::Owned(process) => process.terminate_and_wait(master, deadline), + Self::Owned(process) => process.terminate_and_wait(deadline), #[cfg(test)] Self::Untracked(killer) => killer.kill().is_ok(), } @@ -783,29 +783,32 @@ impl LocalPtyProcess { } #[cfg(unix)] - fn signal_terminal_process_groups(&self, master: &dyn MasterPty, signal: libc::c_int) { + fn signal_terminal_process_session(&self, signal: libc::c_int) -> std::io::Result<()> { let _signal = self.child_signal_lock.lock().unwrap(); if self.child_reaped.load(Ordering::Acquire) { - return; + return Ok(()); } - let mut groups = self - .pid - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .into_iter() - .chain(master.process_group_leader()) - .collect::>(); - groups.sort_unstable(); - groups.dedup(); - // portable-pty creates a new session for the shell. Signal its - // original group and the current foreground job group, while never - // addressing the mux's own process group. - let mux_group = unsafe { libc::getpgrp() }; - for group in groups.into_iter().filter(|group| *group > 0 && *group != mux_group) { - let _ = unsafe { libc::killpg(group, signal) }; + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return Err(std::io::Error::other("PTY child has no process id")); + }; + crate::process_session::signal(session, signal) + } + + #[cfg(unix)] + fn kill_terminal_process_session_until(&self, deadline: Instant) -> std::io::Result { + let _signal = self.child_signal_lock.lock().unwrap(); + if self.child_reaped.load(Ordering::Acquire) { + return Ok(true); } + let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return Err(std::io::Error::other("PTY child has no process id")); + }; + crate::process_session::kill_until_only_leader(leader, leader, deadline, || { + self.child_waitable.load(Ordering::Acquire) + }) } - fn terminate_and_wait(&self, master: &dyn MasterPty, deadline: Instant) -> bool { + fn terminate_and_wait(&self, deadline: Instant) -> bool { #[cfg(unix)] { { @@ -815,21 +818,25 @@ impl LocalPtyProcess { } self.termination_started.store(true, Ordering::Release); } - self.signal_terminal_process_groups(master, libc::SIGHUP); + if self.signal_terminal_process_session(libc::SIGHUP).is_err() { + return false; + } let _ = self.request_hangup(); self.wait_for_child_waitable_until( deadline.min(Instant::now() + Duration::from_millis(250)), ); - // SIGHUP is cooperative. Escalate every owned terminal group so - // an ignoring shell or foreground job cannot survive a successful - // shutdown acknowledgement. - self.signal_terminal_process_groups(master, libc::SIGKILL); + // Process groups are transient job-control details. Repeatedly + // kill every member of the owned PTY session while its leader is + // reserved with WNOWAIT, and do not acknowledge shutdown until no + // background group remains. + if !matches!(self.kill_terminal_process_session_until(deadline), Ok(true)) { + return false; + } self.group_escalation_complete.store(true, Ordering::Release); self.exited.1.notify_all(); } #[cfg(not(unix))] { - let _ = master; let _ = self.request_hangup(); } self.wait_for_exit_until(deadline) @@ -2387,9 +2394,7 @@ impl Surface { let _ = host.terminate_with_timeout(timeout); true } - PtyRuntime::Local { master, process, .. } => { - process.terminate_and_wait(master.as_ref(), deadline) - } + PtyRuntime::Local { process, .. } => process.terminate_and_wait(deadline), #[cfg(unix)] PtyRuntime::ExitedHosted => true, } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 1103d9082eb7..c03d43ad9166 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1846,39 +1846,31 @@ mod unix { self.pty_drained.load(Ordering::Acquire) } - fn signal_terminal_process_groups(&self, signal: libc::c_int) { - let mut groups = Vec::with_capacity(2); + fn signal_terminal_process_session(&self, signal: libc::c_int) -> std::io::Result<()> { // The wait thread observes exit with WNOWAIT, then takes this lock - // before reaping. While we hold it, `!child_reaped` means the - // original PID/PGID is still kernel-reserved and cannot have been - // reused between validation and killpg. + // before reaping. Holding it reserves the session leader and its + // numeric session id across enumeration and signaling. let _signal = self.child_signal_lock.lock().unwrap(); - let child_reserved = !self.child_reaped.load(Ordering::Acquire); - if child_reserved - && let Some(pid) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) - { - groups.push(pid); - } - // Query the PTY each time rather than trusting the original group: - // a foreground job or retained descendant may own a different - // group by the time explicit Terminate escalates. - if child_reserved - && let Some(foreground) = self.master.lock().unwrap().process_group_leader() - { - groups.push(foreground); + if self.child_reaped.load(Ordering::Acquire) { + return Ok(()); } - groups.sort_unstable(); - groups.dedup(); - // A portable-pty child starts as a new session/process-group - // leader. Signal both that durable group and any foreground job - // group, but never risk addressing the terminal-host's own group. - // SAFETY: getpgrp has no preconditions. - let host_group = unsafe { libc::getpgrp() }; - for group in groups.into_iter().filter(|group| *group > 0 && *group != host_group) { - // SAFETY: validated positive process-group ids owned by this - // PTY session; signal is a platform constant from this module. - let _ = unsafe { libc::killpg(group, signal) }; + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return Err(std::io::Error::other("PTY child has no process id")); + }; + crate::process_session::signal(session, signal) + } + + fn kill_terminal_process_session_until(&self, deadline: Instant) -> std::io::Result { + let _signal = self.child_signal_lock.lock().unwrap(); + if self.child_reaped.load(Ordering::Acquire) { + return Ok(true); } + let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return Err(std::io::Error::other("PTY child has no process id")); + }; + crate::process_session::kill_until_only_leader(leader, leader, deadline, || { + self.child_waitable.load(Ordering::Acquire) + }) } fn request_forced_pty_drain(&self) { @@ -1933,24 +1925,30 @@ mod unix { self.termination_started.store(true, Ordering::Release); } // ProcessSignaller only targets the direct child. Start with a - // graceful group hangup so foreground jobs and normal descendants - // can clean up too, then escalate after a strict bound. - self.signal_terminal_process_groups(libc::SIGHUP); + // graceful session hangup so foreground and background jobs can + // clean up too, then escalate after a strict bound. + if self.signal_terminal_process_session(libc::SIGHUP).is_err() { + self.termination_started.store(false, Ordering::Release); + return; + } if !self.child_waitable.load(Ordering::Acquire) { let _ = self.killer.lock().unwrap().kill(); } let _ = self.wait_for_child_waitable(HOST_TERMINATE_GRACE); let _ = self.wait_for_pty_drain(HOST_PTY_DRAIN_GRACE); - // The direct child may ignore SIGHUP, or it may already have - // exited while a descendant retains the PTY. Kill both the - // original session group and its current foreground job group. - // This escalation is mandatory even if Darwin reports PTY EOF as - // soon as the session leader exits: an HUP-ignoring descendant - // can still be alive in the now-invisible original group. - self.signal_terminal_process_groups(libc::SIGKILL); + // Process groups are transient job-control details. Repeatedly + // kill every member of the PTY session while WNOWAIT reserves the + // leader, then publish Exit only after every background group is + // gone. + let kill_deadline = Instant::now() + HOST_KILL_WAIT; + if !matches!(self.kill_terminal_process_session_until(kill_deadline), Ok(true)) { + self.termination_started.store(false, Ordering::Release); + return; + } self.finish_group_escalation(); - let child_exited = self.wait_for_child_exit(HOST_KILL_WAIT); + let child_exited = + self.wait_for_child_exit(kill_deadline.saturating_duration_since(Instant::now())); if child_exited && self.wait_for_pty_drain(HOST_PTY_DRAIN_GRACE) { return; } @@ -2087,21 +2085,36 @@ mod unix { if !self.shared.child_exited() { return; } - let mut removed_record = !self.published; - if self.published - && let Ok(bytes) = fs::read(&self.record_path) - && let Ok(current) = serde_json::from_slice::(&bytes) - && current.terminal_id == self.record.terminal_id - && current.incarnation == self.record.incarnation - && current.host_start_nonce == self.record.host_start_nonce - { - removed_record = fs::remove_file(&self.record_path).is_ok(); - } - let _ = fs::remove_file(&self.endpoint); - if removed_record { - let _ = fs::remove_file(&self.lease.path); - } + let owns_record = !self.published + || fs::read(&self.record_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .is_some_and(|current| { + current.terminal_id == self.record.terminal_id + && current.incarnation == self.record.incarnation + && current.host_start_nonce == self.record.host_start_nonce + }); + let endpoint_removed = match fs::remove_file(&self.endpoint) { + Ok(()) => true, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => true, + Err(_) => false, + }; + let lease_removed = if owns_record { + match fs::remove_file(&self.lease.path) { + Ok(()) => true, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => true, + Err(_) => false, + } + } else { + false + }; let _ = self.lease.file.sync_all(); + // Record removal is the final cleanup barrier. Once observers see + // it absent, both the transport endpoint and process-bound + // liveness path are already gone. + if self.published && owns_record && endpoint_removed && lease_removed { + let _ = fs::remove_file(&self.record_path); + } } } diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index be62b9aee6e7..d780eb8ad418 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -553,7 +553,7 @@ fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { // SAFETY: both fixture processes are live and owned by this test. let descendant_session = unsafe { libc::getsid(descendant_pid) }; - let host = adopt_terminal_host(record.clone(), record_path.clone()).unwrap(); + let host = adopt_terminal_host(record, record_path).unwrap(); host.terminate().unwrap(); host.disconnect(); wait_for_no_host_records(&harness.host_root()); diff --git a/cmux-tui/crates/ghostty-vt-sys/build.rs b/cmux-tui/crates/ghostty-vt-sys/build.rs index 1102ee54375b..1cef7b093247 100644 --- a/cmux-tui/crates/ghostty-vt-sys/build.rs +++ b/cmux-tui/crates/ghostty-vt-sys/build.rs @@ -27,6 +27,7 @@ fn main() { println!("cargo:rerun-if-env-changed=CMUX_GHOSTTY_VT_ZIG_CPU"); println!("cargo:rerun-if-changed={}", ghostty_dir.join("include").display()); println!("cargo:rerun-if-changed={}", ghostty_dir.join("build.zig").display()); + println!("cargo:rerun-if-changed={}", ghostty_dir.join("build.zig.zon").display()); println!("cargo:rerun-if-changed={}", ghostty_dir.join("src").display()); // Build libghostty-vt.a with zig. ReleaseFast regardless of the cargo From e682a59790894ce9e939ec0cbcdb1f64f0c5ad34 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:06:48 -0700 Subject: [PATCH 015/521] test(tui): cover applied legacy close errors --- cmux-tui/crates/cmux-tui/tests/cli.rs | 104 ++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 0b3e335776f5..b5eed10c03ea 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -292,7 +292,14 @@ fn legacy_server_process_helper() { .ok() .map(|value| value.parse::().unwrap()) .unwrap_or_else(std::process::id); - if matches!(scenario.as_str(), "success" | "kill-caller" | "cleanup-failure") { + if matches!( + scenario.as_str(), + "success" + | "kill-caller" + | "cleanup-failure" + | "applied-close-error" + | "persistent-close-error" + ) { let descendant = Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); fs::write( @@ -383,7 +390,11 @@ fn legacy_server_process_helper() { let mut snapshot_index = 0; loop { request.clear(); - reader.read_line(&mut request).unwrap(); + if reader.read_line(&mut request).unwrap() == 0 { + loop { + std::thread::park(); + } + } let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); if scenario == "cleanup-failure" { @@ -401,10 +412,13 @@ fn legacy_server_process_helper() { std::thread::park(); } } - assert!(matches!(scenario.as_str(), "success" | "kill-caller")); - let surfaces: &[u64] = match snapshot_index { - 0 => &[41], - 1 => &[42], + assert!(matches!( + scenario.as_str(), + "success" | "kill-caller" | "applied-close-error" | "persistent-close-error" + )); + let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { + ("persistent-close-error", _) | (_, 0) => &[41], + ("success" | "kill-caller", 1) => &[42], _ => &[], }; let tabs = surfaces @@ -435,16 +449,21 @@ fn legacy_server_process_helper() { let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); - writeln!( - stream, - "{}", - serde_json::json!({ - "id": close_request["id"], - "ok": true, - "data": {}, - }) - ) - .unwrap(); + let response = + if matches!(scenario.as_str(), "applied-close-error" | "persistent-close-error") { + serde_json::json!({ + "id": close_request["id"], + "ok": false, + "error": "surface cleanup reported an error", + }) + } else { + serde_json::json!({ + "id": close_request["id"], + "ok": true, + "data": {}, + }) + }; + writeln!(stream, "{response}").unwrap(); if scenario == "kill-caller" && expected_surface == 41 { let caller_pid = libc::pid_t::try_from(caller_pid).unwrap(); assert_eq!(unsafe { libc::kill(caller_pid, libc::SIGKILL) }, 0); @@ -1178,6 +1197,59 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_reconciles_an_applied_legacy_close_that_returned_an_error() { + let mut server = LegacyServerProcess::start( + "legacy-server-applied-close-error", + "applied-close-error", + None, + ); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); +} + +#[cfg(unix)] +#[test] +fn server_stop_refuses_when_a_legacy_close_error_leaves_the_surface_present() { + let mut server = LegacyServerProcess::start( + "legacy-server-persistent-close-error", + "persistent-close-error", + None, + ); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(server.child.try_wait().unwrap().is_none()); + assert!(process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn detached_legacy_stop_survives_the_calling_pane_process_exit() { From fc87a12c6c22cfeaa840ec4402bf7a152a1faadf Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:08:27 -0700 Subject: [PATCH 016/521] fix(tui): reconcile applied legacy closes --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 025bfadbaed9..cd2c5c54dcef 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -270,14 +270,12 @@ impl ServerLifecycle { &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), ) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_response(&mut self.reader, request_id)?; - if response.get("ok").and_then(Value::as_bool) != Some(true) { - let error = - response.get("error").and_then(Value::as_str).unwrap_or("close-surface failed"); - if !error.starts_with("unknown surface ") { - anyhow::bail!(crate::localization::catalog().server.legacy_close_failed); - } - } + let _response = read_response(&mut self.reader, request_id)?; + // Older servers can commit topology removal, then report a late + // runtime-cleanup error. Reconcile every close response against + // subsequent authoritative workspace snapshots. A genuinely + // failed close remains present for all bounded rounds, so the + // server is still never signaled on ambiguity. } Ok(surfaces.len()) } From 9047f8b47503dea45eda0ab1087bc5d3a013da8f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:10:46 -0700 Subject: [PATCH 017/521] test(tui): cover interrupted legacy closeout --- cmux-tui/crates/cmux-tui/tests/cli.rs | 45 +++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b5eed10c03ea..ae3a8cc9b320 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -298,6 +298,7 @@ fn legacy_server_process_helper() { | "kill-caller" | "cleanup-failure" | "applied-close-error" + | "applied-close-disconnect" | "persistent-close-error" ) { let descendant = @@ -388,7 +389,7 @@ fn legacy_server_process_helper() { let mut reader = BufReader::new(stream.try_clone_box().unwrap()); let mut request = String::new(); let mut snapshot_index = 0; - loop { + 'snapshots: loop { request.clear(); if reader.read_line(&mut request).unwrap() == 0 { loop { @@ -414,7 +415,11 @@ fn legacy_server_process_helper() { } assert!(matches!( scenario.as_str(), - "success" | "kill-caller" | "applied-close-error" | "persistent-close-error" + "success" + | "kill-caller" + | "applied-close-error" + | "applied-close-disconnect" + | "persistent-close-error" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { ("persistent-close-error", _) | (_, 0) => &[41], @@ -449,6 +454,15 @@ fn legacy_server_process_helper() { let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + if scenario == "applied-close-disconnect" { + drop(reader); + drop(stream); + stream = listener.accept().unwrap(); + serve_identify(&mut stream); + reader = BufReader::new(stream.try_clone_box().unwrap()); + snapshot_index += 1; + continue 'snapshots; + } let response = if matches!(scenario.as_str(), "applied-close-error" | "persistent-close-error") { serde_json::json!({ @@ -1224,6 +1238,33 @@ fn server_stop_reconciles_an_applied_legacy_close_that_returned_an_error() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_reconnects_after_an_applied_legacy_close_drops_the_control_stream() { + let mut server = LegacyServerProcess::start( + "legacy-server-applied-close-disconnect", + "applied-close-disconnect", + None, + ); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn server_stop_refuses_when_a_legacy_close_error_leaves_the_surface_present() { From 00c456cc8888a4928f23e5925903506c936eb446 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:12:13 -0700 Subject: [PATCH 018/521] fix(tui): recover interrupted legacy closeout --- .../crates/cmux-tui/src/server_lifecycle.rs | 58 ++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index cd2c5c54dcef..bc4d895f3d77 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -32,6 +32,20 @@ const SHUTDOWN_RESPONSE_TIMEOUT: Duration = type TransportReader = BufReader>; +#[cfg(unix)] +#[derive(Debug)] +struct LegacyConnectionInterrupted; + +#[cfg(unix)] +impl std::fmt::Display for LegacyConnectionInterrupted { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("legacy server control connection was interrupted") + } +} + +#[cfg(unix)] +impl std::error::Error for LegacyConnectionInterrupted {} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ReleaseMismatch { DistributionVersion, @@ -229,11 +243,22 @@ impl ServerLifecycle { } #[cfg(unix)] - fn close_legacy_surfaces_until_stable(&mut self) -> anyhow::Result<()> { + fn close_legacy_surfaces_until_stable( + &mut self, + expected: ProcessIdentity, + ) -> anyhow::Result<()> { let mut next_request_id = LEGACY_LIST_REQUEST_ID; let mut consecutive_empty_scans = 0; for _ in 0..LEGACY_MAX_SCAN_ROUNDS { - let closed = self.close_legacy_surface_snapshot(&mut next_request_id)?; + let closed = match self.close_legacy_surface_snapshot(&mut next_request_id) { + Ok(closed) => closed, + Err(error) if error.downcast_ref::().is_some() => { + self.reconnect_legacy_server(expected)?; + consecutive_empty_scans = 0; + continue; + } + Err(error) => return Err(error), + }; if closed == 0 { consecutive_empty_scans += 1; if consecutive_empty_scans == LEGACY_STABLE_EMPTY_SCANS { @@ -246,6 +271,23 @@ impl ServerLifecycle { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) } + #[cfg(unix)] + fn reconnect_legacy_server(&mut self, expected: ProcessIdentity) -> anyhow::Result<()> { + let replacement = Self::connect(self.path.clone()).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + if replacement.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + } + let actual = + verified_legacy_process(replacement.probe.identity.pid, replacement.peer_process_id)?; + if actual != expected { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + } + *self = replacement; + Ok(()) + } + #[cfg(unix)] fn close_legacy_surface_snapshot( &mut self, @@ -256,8 +298,9 @@ impl ServerLifecycle { self.reader.get_mut(), &json!({"id": list_request_id, "cmd": "list-workspaces"}), ) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_response(&mut self.reader, list_request_id)?; + .map_err(|_| LegacyConnectionInterrupted)?; + let response = read_response(&mut self.reader, list_request_id) + .map_err(|_| LegacyConnectionInterrupted)?; let data = response_data(&response)?; let mut surfaces = legacy_surface_ids(data); surfaces.sort_unstable(); @@ -269,8 +312,9 @@ impl ServerLifecycle { self.reader.get_mut(), &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), ) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let _response = read_response(&mut self.reader, request_id)?; + .map_err(|_| LegacyConnectionInterrupted)?; + let _response = read_response(&mut self.reader, request_id) + .map_err(|_| LegacyConnectionInterrupted)?; // Older servers can commit topology removal, then report a late // runtime-cleanup error. Reconcile every close response against // subsequent authoritative workspace snapshots. A genuinely @@ -345,7 +389,7 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - lifecycle.close_legacy_surfaces_until_stable().map_err(|_| { + lifecycle.close_legacy_surfaces_until_stable(actual).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; terminate_legacy_process_tree(actual)?; From 69822870d7c2f40173791f0527724f31e0fd8ef0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:18:15 -0700 Subject: [PATCH 019/521] test(tui): cover zombie legacy descendants --- cmux-tui/crates/cmux-tui/tests/cli.rs | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index ae3a8cc9b320..d9f501f53933 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -309,6 +309,28 @@ fn legacy_server_process_helper() { ) .unwrap(); drop(descendant); + } else if scenario == "zombie-child" { + let descendant = + Command::new("true").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let descendant_pid = descendant.id(); + fs::write( + std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), + descendant_pid.to_string(), + ) + .unwrap(); + drop(descendant); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let status = Command::new("ps") + .args(["-o", "stat=", "-p", &descendant_pid.to_string()]) + .output() + .unwrap(); + if String::from_utf8_lossy(&status.stdout).trim_start().starts_with('Z') { + break; + } + assert!(Instant::now() < deadline, "descendant did not become a zombie"); + std::thread::sleep(Duration::from_millis(10)); + } } let listener = transport::listen(&socket).unwrap(); let serve_identify = |stream: &mut Box| { @@ -420,8 +442,10 @@ fn legacy_server_process_helper() { | "applied-close-error" | "applied-close-disconnect" | "persistent-close-error" + | "zombie-child" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { + ("zombie-child", _) => &[], ("persistent-close-error", _) | (_, 0) => &[41], ("success" | "kill-caller", 1) => &[42], _ => &[], @@ -1211,6 +1235,29 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert!(!process_exists(descendant_pid)); } +#[cfg(target_os = "macos")] +#[test] +fn server_stop_handles_an_exited_unreaped_legacy_descendant() { + let mut server = LegacyServerProcess::start("legacy-server-zombie-child", "zombie-child", None); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn server_stop_reconciles_an_applied_legacy_close_that_returned_an_error() { From 88c9d5299d39671ddbca830dff33e9cb81135281 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:25:16 -0700 Subject: [PATCH 020/521] fix(tui): tolerate inert legacy descendants --- .../src/server_lifecycle/legacy_process.rs | 104 +++++++++++++++++- 1 file changed, 99 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index ef4ff851fe73..e1f0f341ce86 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -172,11 +172,9 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { libc::proc_pidinfo(pid, libc::PROC_PIDTBSDINFO, 0, (&raw mut info).cast(), expected) }; if result != expected { - // SAFETY: signal zero does not deliver a signal. - if unsafe { libc::kill(pid, 0) } != 0 - && io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) - { - return Ok(None); + match macos_process_status(pid)? { + None | Some(libc::SZOMB) => return Ok(None), + Some(_) => {} } return Err(io::Error::other("could not read process birth identity")); } @@ -187,10 +185,106 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { } let parent = libc::pid_t::try_from(info.pbi_ppid) .map_err(|_| io::Error::other("invalid parent process id"))?; + if info.pbi_status == libc::SZOMB { + return Ok(None); + } let started_at = (u128::from(info.pbi_start_tvsec) << 64) | u128::from(info.pbi_start_tvusec); Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent })) } +#[cfg(target_os = "macos")] +fn macos_process_status(pid: libc::pid_t) -> io::Result> { + use std::ffi::c_void; + use std::mem::{ManuallyDrop, offset_of, size_of}; + + #[allow(dead_code)] + #[repr(C)] + union ProcessStart { + links: [*mut c_void; 2], + started_at: ManuallyDrop, + } + + #[allow(dead_code)] + #[repr(C)] + struct ExternProcessPrefix { + start: ProcessStart, + vmspace: *mut c_void, + signal_actions: *mut c_void, + flags: libc::c_int, + status: libc::c_char, + pid: libc::pid_t, + } + + let mut query = [libc::CTL_KERN, libc::KERN_PROC, libc::KERN_PROC_PID, pid]; + let query_len = libc::c_uint::try_from(query.len()) + .map_err(|_| io::Error::other("process status query overflow"))?; + let mut record_len = 0; + // SAFETY: the query is a valid KERN_PROC_PID MIB and a null output + // buffer asks the kernel for the record size. + if unsafe { + libc::sysctl( + query.as_mut_ptr(), + query_len, + std::ptr::null_mut(), + &raw mut record_len, + std::ptr::null_mut(), + 0, + ) + } != 0 + { + let error = io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT)) { + return Ok(None); + } + return Err(error); + } + if record_len == 0 { + return Ok(None); + } + let mut record = vec![0_u8; record_len]; + // SAFETY: `record` owns a writable buffer of `record_len` bytes. + if unsafe { + libc::sysctl( + query.as_mut_ptr(), + query_len, + record.as_mut_ptr().cast(), + &raw mut record_len, + std::ptr::null_mut(), + 0, + ) + } != 0 + { + let error = io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(libc::ESRCH) | Some(libc::ENOENT)) { + return Ok(None); + } + return Err(error); + } + if record_len < size_of::() { + if record_len == 0 { + return Ok(None); + } + return Err(io::Error::other("process status record was truncated")); + } + // KERN_PROC_PID returns `kinfo_proc`, whose first field is + // `extern_proc`. Read only the stable prefix that contains status and + // PID, without depending on the private remainder of either structure. + let record_pid = unsafe { + record + .as_ptr() + .add(offset_of!(ExternProcessPrefix, pid)) + .cast::() + .read_unaligned() + }; + if record_pid != pid { + return Err(io::Error::other("process status id mismatch")); + } + let status = unsafe { + record.as_ptr().add(offset_of!(ExternProcessPrefix, status)).cast::().read() + }; + u32::try_from(status).map(Some).map_err(|_| io::Error::other("invalid process status")) +} + #[cfg(target_os = "macos")] fn direct_child_pids(parent: libc::pid_t) -> io::Result> { use std::ffi::c_void; From 7a1f5b95c9375dc21f1b1deea4f440a393ca7a64 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:31:52 -0700 Subject: [PATCH 021/521] test(tui): cover servers exiting during closeout --- .../src/server_lifecycle/legacy_process.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index e1f0f341ce86..fe6608dab8eb 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -424,6 +424,22 @@ mod tests { child.wait().unwrap(); } + #[test] + fn termination_accepts_a_verified_process_that_already_exited() { + let mut child = Command::new("sleep") + .arg("60") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let process = + ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + + terminate_process_tree(process).unwrap(); + } + #[cfg(target_os = "linux")] #[test] fn linux_child_enumeration_includes_children_spawned_by_worker_threads() { From d7b2f6863383b8f40ae8bea6c2e5aa0894f448ce Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:32:52 -0700 Subject: [PATCH 022/521] fix(tui): reconcile exits during closeout --- .../src/server_lifecycle/legacy_process.rs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index fe6608dab8eb..f5cf2ec39412 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -1,7 +1,10 @@ use std::collections::HashSet; use std::io; +use std::time::{Duration, Instant}; const PROCESS_TREE_MAX_ROUNDS: usize = 64; +const PROCESS_TREE_RETRY_INTERVAL: Duration = Duration::from_millis(10); +const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct ProcessIdentity { @@ -60,7 +63,27 @@ enum ExactSignalResult { } pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { - FrozenProcessTree::freeze(process).and_then(FrozenProcessTree::kill) + let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; + loop { + let error = match FrozenProcessTree::freeze(process) { + Ok(tree) => return tree.kill(), + Err(error) => error, + }; + match ProcessIdentity::capture(process.pid) { + Ok(None) => return Ok(()), + Ok(Some(current)) if current != process => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "server process identity changed during termination", + )); + } + Ok(Some(_)) | Err(_) => {} + } + if Instant::now() >= deadline { + return Err(error); + } + std::thread::sleep(PROCESS_TREE_RETRY_INTERVAL); + } } struct FrozenProcessTree { From ac9fe5c651d0d6e7e7a620954e2bab00534faa42 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:21:26 -0700 Subject: [PATCH 023/521] test(tui): cover atomic bounded shutdown --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 21 +++++++ cmux-tui/crates/cmux-tui-core/src/browser.rs | 38 ++++++++++++ .../cmux-tui-core/src/process_session.rs | 36 +++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 13 ++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 59 +++++++++++++++++++ .../cmux-tui/tests/terminal_host_recovery.rs | 52 +++++++++++++++- 6 files changed, 217 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 24e1854731fa..4f012049ac82 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -260,4 +260,25 @@ mod tests { assert!(!args.iter().any(|arg| arg == "--window-size=1280,900")); assert_eq!(args.last().map(String::as_str), Some("about:blank")); } + + #[test] + fn kill_until_confirms_the_owned_process_within_its_deadline() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let profile_dir = make_profile_dir().unwrap(); + let chrome = Chrome { + child: Mutex::new(Some(child)), + profile_dir, + profile_ephemeral: true, + web_socket_url: "ws://127.0.0.1/unused".to_string(), + }; + + assert!(chrome.kill_until(std::time::Instant::now() + Duration::from_secs(1))); + assert!(chrome.child.lock().unwrap().is_none()); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 95edecdba4b2..2107bf015103 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -2724,6 +2724,44 @@ mod tests { waiter.join().unwrap(); } + #[test] + fn server_shutdown_bounds_external_target_confirmation() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (close_tx, close_rx) = mpsc::channel(); + let (stop_tx, stop_rx) = mpsc::channel(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + let close = read_ws_json(&mut ws); + assert_eq!(close["method"], "Target.closeTarget"); + assert_eq!(close["params"]["targetId"], "target-1"); + close_tx.send(()).unwrap(); + let _ = stop_rx.recv(); + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + + let started = Instant::now(); + assert!(!runtime.close_surface_for_shutdown( + "target-1", + "session-1", + started + Duration::from_millis(50), + )); + assert!(started.elapsed() < Duration::from_millis(500)); + close_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + stop_tx.send(()).unwrap(); + server.join().unwrap(); + } + #[test] fn closed_surface_route_closes_its_cdp_target() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 049aed94e2b4..6eeb1a62f77a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -171,3 +171,39 @@ fn all_process_ids() -> io::Result> { "PTY session enumeration is unavailable on this platform", )) } + +#[cfg(test)] +mod tests { + #[cfg(target_os = "macos")] + use std::os::unix::process::CommandExt; + #[cfg(target_os = "macos")] + use std::process::{Command, Stdio}; + + #[cfg(target_os = "macos")] + use super::*; + + #[cfg(target_os = "macos")] + #[test] + fn macos_session_query_is_scoped_to_the_owned_session() { + let mut command = Command::new("sleep"); + command.arg("60").stdout(Stdio::null()).stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut child = command.spawn().unwrap(); + let session = libc::pid_t::try_from(child.id()).unwrap(); + + let members = macos_session_process_ids(session).unwrap(); + + let current = libc::pid_t::try_from(std::process::id()).unwrap(); + assert!(members.contains(&session)); + assert!(!members.contains(¤t)); + child.kill().unwrap(); + child.wait().unwrap(); + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 0a3063243e08..be0d3a7d475c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -5101,6 +5101,19 @@ mod tests { assert_eq!(response, json!({"id": 7, "ok": true, "data": {}})); } + #[test] + fn local_shutdown_exits_after_cleanup_when_the_peer_drops_before_ack() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let writer = test_writer(); + let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + writer.close(); + + assert!(!handle_message(&mux, client, r#"{"id":7,"cmd":"shutdown"}"#, &writer)); + assert!(mux.shutdown_requested()); + assert!(mux.surface(surface.id).is_none()); + } + #[test] fn websocket_clients_cannot_shutdown_the_server() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index d9f501f53933..fa000c2ccc28 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -292,6 +292,7 @@ fn legacy_server_process_helper() { .ok() .map(|value| value.parse::().unwrap()) .unwrap_or_else(std::process::id); + let mut reparent_on_close = None; if matches!( scenario.as_str(), "success" @@ -331,6 +332,30 @@ fn legacy_server_process_helper() { assert!(Instant::now() < deadline, "descendant did not become a zombie"); std::thread::sleep(Duration::from_millis(10)); } + } else if scenario == "reparent-on-close" { + let mut command = Command::new("/bin/bash"); + command + .args([ + "--noprofile", + "--norc", + "-c", + concat!( + "set -m; trap '' HUP; ", + "(trap '' HUP; while :; do sleep 60; done) & ", + "printf '%s' \"$!\" > \"$CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE\"; wait" + ), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + reparent_on_close = Some(command.spawn().unwrap()); } let listener = transport::listen(&socket).unwrap(); let serve_identify = |stream: &mut Box| { @@ -443,6 +468,7 @@ fn legacy_server_process_helper() { | "applied-close-disconnect" | "persistent-close-error" | "zombie-child" + | "reparent-on-close" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { ("zombie-child", _) => &[], @@ -502,6 +528,12 @@ fn legacy_server_process_helper() { }) }; writeln!(stream, "{response}").unwrap(); + if scenario == "reparent-on-close" + && let Some(mut direct) = reparent_on_close.take() + { + direct.kill().unwrap(); + direct.wait().unwrap(); + } if scenario == "kill-caller" && expected_surface == 41 { let caller_pid = libc::pid_t::try_from(caller_pid).unwrap(); assert_eq!(unsafe { libc::kill(caller_pid, libc::SIGKILL) }, 0); @@ -1235,6 +1267,33 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn legacy_stop_retains_pty_ownership_captured_before_surface_close() { + let mut server = + LegacyServerProcess::start("legacy-server-reparent-on-close", "reparent-on-close", None); + let descendant_pid = wait_for_pid_file(&server.descendant_pid_file, Duration::from_secs(5)); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + !process_exists(descendant_pid), + "surface close reparented a captured PTY job outside the final server tree" + ); +} + #[cfg(target_os = "macos")] #[test] fn server_stop_handles_an_exited_unreaped_legacy_descendant() { diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index d780eb8ad418..a18107d9a666 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -15,8 +15,9 @@ use cmux_tui_core::terminal_host::{ CAPABILITY_TOKEN_LEN, CapabilityRights, CapabilityToken, ClientHello, ClientRole, TerminalId, }; use cmux_tui_core::terminal_host_protocol::{ - FLAG_COLORS_FOLLOW, FLAG_VIEWER_SIZE_ACKS, Frame, MAX_FRAME_PAYLOAD, MessageKind, - PROTOCOL_VERSION, ProtocolError, RESIZE_ACK_CANONICAL_CHANGED, read_frame, write_frame, + FLAG_COLORS_FOLLOW, FLAG_TERMINATE_ONLY, FLAG_VIEWER_SIZE_ACKS, Frame, MAX_FRAME_PAYLOAD, + MessageKind, PROTOCOL_VERSION, ProtocolError, RESIZE_ACK_CANONICAL_CHANGED, read_frame, + write_frame, }; use cmux_tui_core::terminal_host_runtime::{ TerminalHostLiveness, adopt_terminal_host, decode_terminal_color_overrides, @@ -945,6 +946,53 @@ fn mint_capability_fences_prior_admin_input_before_renderer_input() { wait_for_no_host_records(&harness.host_root()); } +#[test] +fn terminate_only_owner_handshake_never_materializes_a_snapshot() { + let harness = RecoveryHarness::start("terminate-only"); + request( + &harness.socket, + serde_json::json!({ + "id": 1, + "cmd": "run", + "argv": ["/bin/cat"], + "new_workspace": true, + }), + ); + let (_, record) = wait_for_host_records(&harness.host_root(), 1).remove(0); + assert!(record.supports_terminate_only); + let mut stream = UnixStream::connect(&record.endpoint).unwrap(); + let hello = ClientHello { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + role: ClientRole::Admin, + requested_rights: CapabilityRights::TERMINATE, + terminal_id: TerminalId::from_bytes(decode_hex(&record.terminal_id).unwrap()), + token: CapabilityToken::from_bytes(decode_hex(&record.owner_token).unwrap()), + }; + let mut hello = hello.into_frame(1); + hello.flags = FLAG_TERMINATE_ONLY; + write_frame(&mut stream, &hello).unwrap(); + let response = read_frame(&mut stream, MAX_FRAME_PAYLOAD).unwrap().unwrap(); + assert_eq!(response.kind, MessageKind::HostHello); + assert_eq!(response.flags, FLAG_TERMINATE_ONLY); + + write_frame(&mut stream, &Frame::new(MessageKind::Terminate, Vec::new())).unwrap(); + stream.set_read_timeout(Some(Duration::from_millis(250))).unwrap(); + match read_frame(&mut stream, MAX_FRAME_PAYLOAD) { + Ok(None) => {} + Err(ProtocolError::Io(error)) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::ConnectionReset + ) => {} + Ok(Some(frame)) => panic!("terminate-only handshake emitted {:?}", frame.kind), + Err(error) => panic!("unexpected terminate-only close error: {error}"), + } + wait_for_no_host_records(&harness.host_root()); +} + #[test] fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { let mut harness = RecoveryHarness::start("sigkill-adopt"); From 697297e4d88aa30dc38fbee56bad886b94656bc9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:40:26 -0700 Subject: [PATCH 024/521] fix(tui): make shutdown atomic and bounded --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 36 +++- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 26 ++- cmux-tui/crates/cmux-tui-core/src/browser.rs | 48 ++++- cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 31 +++- .../cmux-tui-core/src/process_session.rs | 106 +++++++++-- cmux-tui/crates/cmux-tui-core/src/server.rs | 7 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 5 +- .../src/terminal_host_protocol.rs | 11 +- .../src/terminal_host_runtime.rs | 88 ++++++++- .../crates/cmux-tui/src/server_lifecycle.rs | 22 ++- .../src/server_lifecycle/legacy_process.rs | 173 +++++++++++++++++- 12 files changed, 509 insertions(+), 46 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 4f012049ac82..3e9bf787c86d 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -5,7 +5,7 @@ use std::process::{Child, Command, Stdio}; use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); @@ -85,8 +85,7 @@ impl Chrome { let web_socket_url = match rx.recv_timeout(Duration::from_secs(10)) { Ok(url) => url, Err(_) => { - let _ = child.kill(); - let _ = child.wait(); + let _ = kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)); if profile_ephemeral { let _ = std::fs::remove_dir_all(&profile_dir); } @@ -110,10 +109,18 @@ impl Chrome { } pub fn kill(&self) { - if let Some(mut child) = self.child.lock().unwrap().take() { - let _ = child.kill(); - let _ = child.wait(); + let _ = self.kill_until(Instant::now() + Duration::from_secs(1)); + } + + pub fn kill_until(&self, deadline: Instant) -> bool { + let mut slot = self.child.lock().unwrap(); + let Some(mut child) = slot.take() else { return true }; + drop(slot); + if kill_child_until(&mut child, deadline) { + return true; } + *self.child.lock().unwrap() = Some(child); + false } } @@ -126,6 +133,21 @@ impl Drop for Chrome { } } +fn kill_child_until(child: &mut Child, deadline: Instant) -> bool { + let _ = child.kill(); + loop { + match child.try_wait() { + Ok(Some(_)) => return true, + Ok(None) => {} + Err(_) => return false, + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + std::thread::sleep(remaining.min(Duration::from_millis(10))); + } +} + fn make_profile_dir() -> anyhow::Result { let seq = PROFILE_SEQ.fetch_add(1, Ordering::Relaxed); let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis(); @@ -278,7 +300,7 @@ mod tests { web_socket_url: "ws://127.0.0.1/unused".to_string(), }; - assert!(chrome.kill_until(std::time::Instant::now() + Duration::from_secs(1))); + assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); assert!(chrome.child.lock().unwrap().is_none()); } } diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 563d6eace098..a0b8b1958190 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -334,6 +334,16 @@ impl CdpClient { method: &str, params: Value, session_id: Option<&str>, + ) -> anyhow::Result { + self.call_with_timeout(method, params, session_id, self.inner.timeout) + } + + pub fn call_with_timeout( + &self, + method: &str, + params: Value, + session_id: Option<&str>, + timeout: Duration, ) -> anyhow::Result { let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); let (tx, rx) = channel(); @@ -353,7 +363,7 @@ impl CdpClient { return Err(e); } - match rx.recv_timeout(self.inner.timeout) { + match rx.recv_timeout(timeout) { Ok(Ok(value)) => Ok(value), Ok(Err(e)) => anyhow::bail!("{e}"), Err(_) => { @@ -402,6 +412,20 @@ impl CdpClient { self.call("Target.closeTarget", json!({ "targetId": target_id }), None).map(|_| ()) } + pub fn close_target_with_timeout( + &self, + target_id: &str, + timeout: Duration, + ) -> anyhow::Result<()> { + self.call_with_timeout( + "Target.closeTarget", + json!({ "targetId": target_id }), + None, + timeout, + ) + .map(|_| ()) + } + pub fn close_target_detached(&self, target_id: &str) -> anyhow::Result<()> { let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); let msg = json!({ diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 2107bf015103..43d0910fe423 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -517,12 +517,40 @@ impl BrowserRuntime { } } + fn close_surface_for_shutdown( + &self, + target_id: &str, + session_id: &str, + deadline: Instant, + ) -> bool { + self.unregister(target_id, session_id); + if self.source == BrowserSource::Launched { + if !self.is_closed() { + let _ = self.client.close_target_detached(target_id); + } + // The shared runtime confirms the owned browser process itself + // exited before shutdown can succeed. + return true; + } + if self.is_closed() { + return false; + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + self.client.close_target_with_timeout(target_id, remaining).is_ok() + } + pub fn shutdown(&self) { + let _ = self.shutdown_until(Instant::now() + Duration::from_secs(1)); + } + + pub(crate) fn shutdown_until(&self, deadline: Instant) -> bool { close_browser_runtime(self, "browser runtime shut down".to_string()); - let _ = self.client.flush_outbound(Duration::from_secs(1)); if let Some(chrome) = &self.chrome { - chrome.kill(); + return chrome.kill_until(deadline); } + true } } @@ -1176,6 +1204,22 @@ impl BrowserSurface { self.close_command_sender(); } + pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { + if self.dead.swap(true, Ordering::AcqRel) { + return true; + } + self.close_taps(); + let terminated = self.session.lock().unwrap().take().is_none_or(|session| { + session.runtime.close_surface_for_shutdown( + &session.target_id, + &session.session_id, + deadline, + ) + }); + self.close_command_sender(); + terminated + } + pub fn resize(&self, cols: u16, rows: u16) -> anyhow::Result { self.resize_reporting_acceptance(cols, rows, Box::new(|_| {})) .map(|reservation_id| reservation_id.is_some()) diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 36326e9cf94a..a9f1b6799f1f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -14,7 +14,7 @@ mod model; mod mux; mod pairing; #[cfg(unix)] -mod process_session; +pub mod process_session; pub mod provider_management; mod short_id; mod surface; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d08b2a3e54f9..cd524a5f83cc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -43,7 +43,7 @@ const WORKSPACE_NAME_MAX_BYTES: usize = 1_024; const PROVIDER_WORKSPACE_AUTHORITY_MIN_BYTES: usize = 32; const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); -const SHUTDOWN_FANOUT_WORKERS: usize = 128; +const SHUTDOWN_FANOUT_WORKERS: usize = 32; /// An opaque per-mux credential provisioned by the external machine /// provider. Debug output is deliberately redacted. @@ -3749,7 +3749,7 @@ impl Mux { surface.disconnect_for_daemon_shutdown(); } if let Some(runtime) = self.browser_runtime.lock().unwrap().take() { - runtime.shutdown(); + let _ = runtime.shutdown_until(deadline); } } @@ -3848,10 +3848,26 @@ impl Mux { }); let mut failed_surfaces = failed_surfaces.into_inner().unwrap(); failed_surfaces.sort_by_key(|surface| surface.id); + let browser_surface_failed = + failed_surfaces.iter().any(|surface| surface.as_browser().is_some()); let failed_surface_ids = failed_surfaces.iter().map(|surface| surface.id).collect::>(); *self.shutdown_surfaces.lock().unwrap() = failed_surfaces; + let browser_runtime = (!browser_surface_failed) + .then(|| self.browser_runtime.lock().unwrap().take()) + .flatten(); + let browser_runtime_failed = if let Some(runtime) = browser_runtime { + if runtime.shutdown_until(deadline) { + false + } else { + *self.browser_runtime.lock().unwrap() = Some(runtime); + true + } + } else { + false + }; + #[cfg(unix)] let failed_hosts = { let failed = Mutex::new(Vec::new()); @@ -3881,6 +3897,9 @@ impl Mux { .join(", ") )); } + if browser_runtime_failed { + failures.push("could not terminate the owned browser before the deadline".to_string()); + } #[cfg(unix)] if !failed_hosts.is_empty() { failures.push(format!( @@ -7631,6 +7650,14 @@ fn terminate_host_record_with_timeout( record_path: std::path::PathBuf, timeout: Duration, ) -> bool { + if record.supports_terminate_only { + return crate::terminal_host_runtime::terminate_terminal_host_with_timeout( + &record, + &record_path, + timeout, + ) + .is_ok(); + } if let Ok(host) = crate::terminal_host_runtime::adopt_terminal_host_with_timeout(record, record_path, timeout) { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 6eeb1a62f77a..907fc3281cf8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -28,15 +28,64 @@ pub(crate) fn kill_until_only_leader( leader_exited: impl Fn() -> bool, ) -> io::Result { validate_session_target(session)?; + let mut known_members = members(session)?; + signal_members(&known_members, session, None)?; loop { - let session_members = members(session)?; - for pid in &session_members { - signal_if_still_member(*pid, session, libc::SIGKILL)?; + let known_drained = known_members + .iter() + .copied() + .filter(|pid| *pid != leader) + .map(|pid| still_member(pid, session)) + .collect::>>()? + .into_iter() + .all(|alive| !alive); + if leader_exited() && known_drained { + let current = members(session)?; + if current.iter().all(|pid| *pid == leader) { + return Ok(true); + } + signal_members(¤t, session, None)?; + known_members = current; } - // Once WNOWAIT has observed the leader's exit, no process can fork a - // new member between this complete session scan and success. - if leader_exited() && session_members.iter().all(|pid| *pid == leader) { - return Ok(true); + if Instant::now() >= deadline { + return Ok(false); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(SESSION_KILL_POLL), + ); + } +} + +/// Drain a captured session while the caller keeps one exact member stopped. +/// The reserved process prevents session-id reuse and cannot fork, so global +/// membership is reconciled only after each known generation has exited. +pub fn kill_until_only_reserved( + session: libc::pid_t, + reserved: libc::pid_t, + deadline: Instant, +) -> io::Result { + validate_session_target(session)?; + if !still_member(reserved, session)? { + return Err(io::Error::new(io::ErrorKind::NotFound, "reserved PTY session process exited")); + } + let mut known_members = members(session)?; + signal_members(&known_members, session, Some(reserved))?; + loop { + let known_drained = known_members + .iter() + .copied() + .filter(|pid| *pid != reserved) + .map(|pid| still_member(pid, session)) + .collect::>>()? + .into_iter() + .all(|alive| !alive); + if known_drained { + let current = members(session)?; + if current.iter().all(|pid| *pid == reserved) { + return Ok(true); + } + signal_members(¤t, session, Some(reserved))?; + known_members = current; } if Instant::now() >= deadline { return Ok(false); @@ -47,6 +96,24 @@ pub(crate) fn kill_until_only_leader( } } +/// Read-only proof used after every pre-close process identity in a captured +/// legacy PTY session has already disappeared. +pub fn session_is_empty(session: libc::pid_t) -> io::Result { + validate_session_target(session)?; + members(session).map(|members| members.is_empty()) +} + +fn signal_members( + members: &[libc::pid_t], + session: libc::pid_t, + reserved: Option, +) -> io::Result<()> { + for pid in members.iter().copied().filter(|pid| Some(*pid) != reserved) { + signal_if_still_member(pid, session, libc::SIGKILL)?; + } + Ok(()) +} + fn validate_session_target(session: libc::pid_t) -> io::Result<()> { if session <= 1 { return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid PTY session id")); @@ -68,13 +135,7 @@ fn signal_if_still_member( ) -> io::Result<()> { // Revalidate membership immediately before signaling. The reserved // session leader prevents the session id itself from being recycled. - // SAFETY: getsid only queries process metadata. - let current_session = unsafe { libc::getsid(pid) }; - if current_session < 0 { - let error = io::Error::last_os_error(); - return if error.raw_os_error() == Some(libc::ESRCH) { Ok(()) } else { Err(error) }; - } - if current_session != session { + if !still_member(pid, session)? { return Ok(()); } // SAFETY: membership was revalidated above and `signal` is a platform @@ -86,6 +147,16 @@ fn signal_if_still_member( if error.raw_os_error() == Some(libc::ESRCH) { Ok(()) } else { Err(error) } } +fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { + // SAFETY: getsid only queries process metadata. + let current_session = unsafe { libc::getsid(pid) }; + if current_session >= 0 { + return Ok(current_session == session); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) } +} + fn members(session: libc::pid_t) -> io::Result> { let mut members = Vec::new(); for pid in all_process_ids()? { @@ -110,11 +181,18 @@ fn members(session: libc::pid_t) -> io::Result> { Ok(members) } +#[cfg(all(target_os = "macos", test))] +fn macos_session_process_ids(session: libc::pid_t) -> io::Result> { + members(session) +} + #[cfg(target_os = "macos")] fn all_process_ids() -> io::Result> { use std::ffi::c_void; use std::mem::size_of; + // Callers retain this snapshot and poll its members directly. A new full + // process-table scan happens only after that generation has drained. // SAFETY: a null buffer asks libproc for the current process count. let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; if count < 0 { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index be0d3a7d475c..a7e779ea54f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -1938,9 +1938,12 @@ fn handle_message(mux: &Arc, client: u64, message: &str, writer: &MessageWr disconnect_client(mux, client, true); return false; } - if shutdown && response_ok && sent { + if shutdown && response_ok { + // Cleanup is irreversible once the registry and runtimes have been + // drained. A peer that disconnects before the best-effort ACK must + // not leave an empty, permanently fenced daemon behind. mux.request_shutdown(); - return true; + return sent; } sent } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 9c2e3322f339..0b5b96eac75c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2399,10 +2399,7 @@ impl Surface { PtyRuntime::ExitedHosted => true, } } - Surface::Browser(browser) => { - browser.kill(); - true - } + Surface::Browser(browser) => browser.terminate_for_server_shutdown(deadline), } } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs index 8bc03fc7486a..9ddf35762671 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs @@ -21,6 +21,10 @@ pub const FLAG_COLORS_FOLLOW: u32 = 1 << 0; /// control responses. This handshake-only flag lets v1 peers negotiate the /// optimization without exposing an unknown ResizeAck to legacy renderers. pub const FLAG_VIEWER_SIZE_ACKS: u32 = 1 << 1; +/// ClientHello opt-in and HostHello acknowledgement for an authenticated +/// owner that will send exactly one Terminate control. This path deliberately +/// skips Snapshot registration and replay materialization. +pub const FLAG_TERMINATE_ONLY: u32 = 1 << 2; /// ResizeAck payload flag: this request changed the canonical grid and its /// sequenced Resized+Colors transition was enqueued immediately before the /// targeted acknowledgement. @@ -119,9 +123,10 @@ pub struct Frame { /// consumers stage the first frame and expose only the paired state. /// Snapshot keeps flags zero: its same-boundary Colors frame is a mandatory /// bootstrap rule rather than a live-stream transition. - /// ClientHello/HostHello may negotiate [`FLAG_VIEWER_SIZE_ACKS`]. Unknown - /// flags, flags on Colors or other message kinds, an unflagged Resized, and - /// a flagged live frame not followed by Colors are protocol errors. + /// ClientHello/HostHello may negotiate [`FLAG_VIEWER_SIZE_ACKS`] or + /// [`FLAG_TERMINATE_ONLY`]. Unknown flags, flags on Colors or other message + /// kinds, an unflagged Resized, and a flagged live frame not followed by + /// Colors are protocol errors. pub sequence: u64, pub payload: Vec, } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index c03d43ad9166..c0b1ba881ee1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -16,8 +16,8 @@ use crate::terminal_host::{ HostHello, HostIncarnation, HostReady, TerminalId, }; use crate::terminal_host_protocol::{ - FLAG_COLORS_FOLLOW, FLAG_VIEWER_SIZE_ACKS, Frame, MAX_FRAME_PAYLOAD, MessageKind, - PROTOCOL_VERSION, RESIZE_ACK_CANONICAL_CHANGED, read_frame, write_frame, + FLAG_COLORS_FOLLOW, FLAG_TERMINATE_ONLY, FLAG_VIEWER_SIZE_ACKS, Frame, MAX_FRAME_PAYLOAD, + MessageKind, PROTOCOL_VERSION, RESIZE_ACK_CANONICAL_CHANGED, read_frame, write_frame, }; const HOST_RECORD_VERSION: u32 = 2; @@ -74,6 +74,10 @@ pub struct TerminalHostRecord { /// hosts and must never receive the unknown SetDefaults message. #[serde(default)] pub supports_set_defaults: bool, + /// Additive handshake capability. Missing/false records require the + /// compatibility adoption path, which materializes a full Snapshot. + #[serde(default)] + pub supports_terminate_only: bool, } impl std::fmt::Debug for TerminalHostRecord { @@ -88,6 +92,7 @@ impl std::fmt::Debug for TerminalHostRecord { .field("host_start_nonce", &self.host_start_nonce) .field("workspace_key", &self.workspace_key) .field("supports_set_defaults", &self.supports_set_defaults) + .field("supports_terminate_only", &self.supports_terminate_only) .finish() } } @@ -1075,6 +1080,53 @@ mod unix { connect_record_with_timeout(record, record_path, timeout) } + pub(crate) fn terminate_terminal_host_with_timeout( + record: &TerminalHostRecord, + record_path: &Path, + timeout: Duration, + ) -> anyhow::Result<()> { + validate_terminal_host_record(record_path, record)?; + if !record.supports_terminate_only { + anyhow::bail!("terminal host does not advertise terminate-only handshakes"); + } + let terminal_id = TerminalId::from_bytes(decode_hex_array(&record.terminal_id)?); + let incarnation = HostIncarnation::from_bytes(decode_hex_array(&record.incarnation)?); + let owner_token = CapabilityToken::from_bytes(decode_hex_array(&record.owner_token)?); + let mut stream = connect_with_timeout(Path::new(&record.endpoint), timeout)?; + stream.set_read_timeout(Some(timeout))?; + stream.set_write_timeout(Some(timeout))?; + let hello = ClientHello { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + role: ClientRole::Admin, + requested_rights: CapabilityRights::TERMINATE, + terminal_id, + token: owner_token, + }; + let mut hello = hello.into_frame(1); + hello.flags = FLAG_TERMINATE_ONLY; + write_frame(&mut stream, &hello)?; + let response = read_required_frame(&mut stream, "terminate-only host hello")?; + if response.kind != MessageKind::HostHello + || response.flags != FLAG_TERMINATE_ONLY + || response.request_id != 1 + || response.sequence != 0 + { + anyhow::bail!("terminal host rejected terminate-only handshake"); + } + let response = HostHello::decode(&response.payload)?; + if response.selected_version != PROTOCOL_VERSION + || response.granted_rights != CapabilityRights::TERMINATE + || response.terminal_id != terminal_id + || response.incarnation != incarnation + { + anyhow::bail!("terminate-only host identity or rights changed"); + } + write_frame(&mut stream, &Frame::new(MessageKind::Terminate, Vec::new()))?; + stream.shutdown(std::net::Shutdown::Write)?; + Ok(()) + } + /// Validate a discovery record without trusting paths or alternate /// identity spellings supplied by its JSON payload. pub fn validate_terminal_host_record( @@ -1100,6 +1152,7 @@ mod unix { if record.host_pid != 0 || !record.host_start_nonce.is_empty() || record.supports_set_defaults + || record.supports_terminate_only { anyhow::bail!("legacy terminal-host record has unexpected liveness fields"); } @@ -2170,6 +2223,7 @@ mod unix { host_start_nonce: encode_hex(start_nonce.as_bytes()), workspace_key: String::new(), supports_set_defaults: true, + supports_terminate_only: true, }; let lease = HostLivenessLease::acquire(liveness_path(Path::new(&launch.record_path), &record))?; @@ -2450,14 +2504,19 @@ mod unix { let hello_frame = read_required_frame(&mut stream, "client hello")?; if hello_frame.kind != MessageKind::ClientHello || hello_frame.sequence != 0 - || hello_frame.flags & !FLAG_VIEWER_SIZE_ACKS != 0 + || hello_frame.flags & !(FLAG_VIEWER_SIZE_ACKS | FLAG_TERMINATE_ONLY) != 0 + || hello_frame.flags == (FLAG_VIEWER_SIZE_ACKS | FLAG_TERMINATE_ONLY) { anyhow::bail!("terminal-host client did not send ClientHello"); } let hello = ClientHello::decode(&hello_frame.payload)?; let response = authenticate_client(&host, &hello)?; + let terminate_only = hello_frame.flags == FLAG_TERMINATE_ONLY; if hello_frame.version != response.selected_version - || !response.granted_rights.contains(CapabilityRights::READ) + || (!terminate_only && !response.granted_rights.contains(CapabilityRights::READ)) + || (terminate_only + && (hello.role != ClientRole::Admin + || response.granted_rights != CapabilityRights::TERMINATE)) { anyhow::bail!("terminal-host capability denied"); } @@ -2467,10 +2526,27 @@ mod unix { let mut hello_response = Frame::new(MessageKind::HostHello, response.encode()); if viewer_size_acks { hello_response.flags = FLAG_VIEWER_SIZE_ACKS; + } else if terminate_only { + hello_response.flags = FLAG_TERMINATE_ONLY; } hello_response.request_id = hello_frame.request_id; write_frame(&mut stream, &hello_response)?; + if terminate_only { + stream.set_read_timeout(Some(HOST_HANDSHAKE_TIMEOUT))?; + let terminate = read_required_frame(&mut stream, "terminate-only request")?; + if terminate.kind != MessageKind::Terminate + || terminate.flags != 0 + || terminate.sequence != 0 + || terminate.request_id != 0 + || !terminate.payload.is_empty() + { + anyhow::bail!("terminate-only client sent an unexpected request"); + } + host.request_termination(); + return Ok(()); + } + let client = host.next_client.fetch_add(1, Ordering::Relaxed); let (sender, receiver) = sync_channel(256); let tap = HostTap { @@ -2977,6 +3053,7 @@ mod unix { host_start_nonce: encode_hex(nonce.as_bytes()), workspace_key: String::new(), supports_set_defaults: true, + supports_terminate_only: true, }; let record_path = record.record_path(&root); let lease = HostLivenessLease::acquire(liveness_path(&record_path, &record)).unwrap(); @@ -3095,6 +3172,7 @@ mod unix { legacy.host_pid = 0; legacy.host_start_nonce.clear(); legacy.supports_set_defaults = false; + legacy.supports_terminate_only = false; let legacy_path = legacy.record_path(root); write_record(&legacy_path, &legacy).unwrap(); @@ -3549,7 +3627,7 @@ pub use unix::{ #[cfg(unix)] pub(crate) use unix::{ adopt_terminal_host_with_timeout, launch_terminal_host_cancellable, - launch_terminal_host_with_identity_cancellable, + launch_terminal_host_with_identity_cancellable, terminate_terminal_host_with_timeout, }; #[cfg(not(unix))] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index bc4d895f3d77..2dab17a59f85 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -14,8 +14,10 @@ use serde_json::{Value, json}; #[cfg(unix)] mod legacy_process; +#[cfg(all(unix, test))] +use legacy_process::terminate_process_tree; #[cfg(unix)] -use legacy_process::{ProcessIdentity, terminate_process_tree}; +use legacy_process::{CapturedProcessTree, ProcessIdentity, capture_process_tree}; const PROBE_REQUEST_ID: u64 = 0; const SHUTDOWN_REQUEST_ID: u64 = 1; @@ -388,11 +390,12 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { if actual != expected { anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } + let captured = capture_legacy_process_tree(actual)?; lifecycle.close_legacy_surfaces_until_stable(actual).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - terminate_legacy_process_tree(actual)?; + terminate_captured_legacy_process_tree(captured)?; wait_for_disconnect(&mut lifecycle.reader) } @@ -417,12 +420,25 @@ fn verified_legacy_process( .ok_or_else(|| anyhow::anyhow!(messages.legacy_peer_mismatch)) } -#[cfg(unix)] +#[cfg(all(unix, test))] fn terminate_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result<()> { terminate_process_tree(process) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) } +#[cfg(unix)] +fn capture_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result { + capture_process_tree(process) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) +} + +#[cfg(unix)] +fn terminate_captured_legacy_process_tree(process: CapturedProcessTree) -> anyhow::Result<()> { + process + .terminate() + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) +} + pub(crate) fn validate_local_identity(data: &Value, path: &Path) -> anyhow::Result<()> { let probe = ServerProbe { identity: ServerIdentity::from_protocol_data(data)? }; probe.require_compatible(path) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index f5cf2ec39412..03ac61d91585 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -48,6 +48,33 @@ impl ProcessIdentity { Err(error) } } + + fn signal_in_session( + self, + session: libc::pid_t, + signal: libc::c_int, + ) -> io::Result { + let Some(current) = Self::capture(self.pid)? else { + return Ok(ExactSignalResult::Gone); + }; + if current != self { + return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); + } + // SAFETY: getsid only queries the exact process revalidated above. + let current_session = unsafe { libc::getsid(self.pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { + Ok(ExactSignalResult::Gone) + } else { + Err(error) + }; + } + if current_session != session { + return Ok(ExactSignalResult::Gone); + } + self.signal(signal) + } } #[derive(Clone, Copy, Debug)] @@ -63,14 +90,34 @@ enum ExactSignalResult { } pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { + match freeze_process_tree(process) { + Ok(tree) => tree.kill(), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +pub(super) fn capture_process_tree(process: ProcessIdentity) -> io::Result { + let tree = freeze_process_tree(process)?; + let sessions = tree.captured_sessions()?; + drop(tree); + Ok(CapturedProcessTree { root: process, sessions }) +} + +fn freeze_process_tree(process: ProcessIdentity) -> io::Result { let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; loop { let error = match FrozenProcessTree::freeze(process) { - Ok(tree) => return tree.kill(), + Ok(tree) => return Ok(tree), Err(error) => error, }; match ProcessIdentity::capture(process.pid) { - Ok(None) => return Ok(()), + Ok(None) => { + return Err(io::Error::new( + io::ErrorKind::NotFound, + "server exited before its process tree was fenced", + )); + } Ok(Some(current)) if current != process => { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -86,6 +133,91 @@ pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> } } +pub(super) struct CapturedProcessTree { + root: ProcessIdentity, + sessions: Vec, +} + +impl CapturedProcessTree { + pub(super) fn terminate(self) -> io::Result<()> { + let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; + for session in &self.sessions { + if !session.kill_until_empty(deadline)? { + return Err(io::Error::other( + "captured PTY session did not exit before the legacy shutdown deadline", + )); + } + } + terminate_process_tree(self.root) + } +} + +struct CapturedSession { + id: libc::pid_t, + members: Vec, +} + +impl CapturedSession { + fn kill_until_empty(&self, deadline: Instant) -> io::Result { + let Some(mut probe) = SessionProbe::acquire(self)? else { + if cmux_tui_core::process_session::session_is_empty(self.id)? { + return Ok(true); + } + return Err(io::Error::other( + "captured PTY session lost every exact process before termination", + )); + }; + if !cmux_tui_core::process_session::kill_until_only_reserved( + self.id, + probe.process.pid, + deadline, + )? { + return Ok(false); + } + probe.kill()?; + Ok(true) + } +} + +struct SessionProbe { + process: ProcessIdentity, + session: libc::pid_t, + armed: bool, +} + +impl SessionProbe { + fn acquire(session: &CapturedSession) -> io::Result> { + for process in &session.members { + if ProcessIdentity::capture(process.pid)? != Some(*process) { + continue; + } + if process.signal_in_session(session.id, libc::SIGSTOP)? != ExactSignalResult::Signaled + { + continue; + } + if ProcessIdentity::capture(process.pid)? == Some(*process) { + return Ok(Some(Self { process: *process, session: session.id, armed: true })); + } + let _ = process.signal(libc::SIGCONT); + } + Ok(None) + } + + fn kill(&mut self) -> io::Result<()> { + let _ = self.process.signal_in_session(self.session, libc::SIGKILL)?; + self.armed = false; + Ok(()) + } +} + +impl Drop for SessionProbe { + fn drop(&mut self) { + if self.armed { + let _ = self.process.signal_in_session(self.session, libc::SIGCONT); + } + } +} + struct FrozenProcessTree { root: ProcessIdentity, descendants: Vec, @@ -148,6 +280,43 @@ impl FrozenProcessTree { Err(io::Error::other("legacy process tree did not stabilize")) } + fn captured_sessions(&self) -> io::Result> { + // SAFETY: getsid only queries live, exact processes held stopped by + // this FrozenProcessTree. + let root_session = unsafe { libc::getsid(self.root.pid) }; + if root_session < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: getsid(0) only queries the detached helper. + let helper_session = unsafe { libc::getsid(0) }; + if helper_session < 0 { + return Err(io::Error::last_os_error()); + } + let mut sessions = Vec::::new(); + for process in &self.descendants { + // SAFETY: every descendant remains stopped and exact until this + // FrozenProcessTree is dropped. + let session = unsafe { libc::getsid(process.pid) }; + if session < 0 { + return Err(io::Error::last_os_error()); + } + if session > 1 && session != root_session && session != helper_session { + if let Some(captured) = sessions.iter_mut().find(|captured| captured.id == session) + { + captured.members.push(*process); + } else { + sessions.push(CapturedSession { id: session, members: vec![*process] }); + } + } + } + sessions.sort_by_key(|session| session.id); + for session in &mut sessions { + session.members.sort_by_key(|process| process.pid); + session.members.dedup(); + } + Ok(sessions) + } + fn kill(mut self) -> io::Result<()> { if !self.armed { return Ok(()); From bfa64e2639c4375862612ce190857bcad78401bf Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:44:10 -0700 Subject: [PATCH 025/521] test(tui): execute package upgrade identities --- .github/workflows/ci.yml | 3 + .../crates/cmux-tui/src/server_lifecycle.rs | 18 ++ tests/test_tui_npm_package_artifact.py | 7 +- tests/test_tui_package_launchers.py | 197 ++++++++++++++++++ 4 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 tests/test_tui_package_launchers.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d5bfdf1cd50..a941bedc755e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,9 @@ jobs: - name: Validate TUI npm package artifact transfer run: python3 tests/test_tui_npm_package_artifact.py + - name: Validate TUI package launchers + run: python3 tests/test_tui_package_launchers.py + - name: Validate Claude launch environment policy generation run: python3 scripts/generate-claude-launch-environment-policy.py --check diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 2dab17a59f85..a1c00d17dabd 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -657,6 +657,24 @@ mod tests { assert!(message.contains("Stopping exits pane processes.")); } + #[test] + fn replayable_launcher_override_survives_one_shot_package_shims() { + let argv0 = std::ffi::OsStr::new("/tmp/ephemeral/bin/cmux"); + + assert_eq!( + launcher_command_from(Some(std::ffi::OsStr::new("npx cmux")), Some(argv0)), + "npx cmux" + ); + assert_eq!( + launcher_command_from(Some(std::ffi::OsStr::new("uvx cmux")), Some(argv0)), + "uvx cmux" + ); + assert_eq!( + launcher_command_from(Some(std::ffi::OsStr::new("bad\ncommand")), Some(argv0)), + "/tmp/ephemeral/bin/cmux" + ); + } + #[cfg(unix)] #[test] fn legacy_stop_terminates_only_the_identified_process() { diff --git a/tests/test_tui_npm_package_artifact.py b/tests/test_tui_npm_package_artifact.py index a2b021aa0951..dc893373f9c4 100644 --- a/tests/test_tui_npm_package_artifact.py +++ b/tests/test_tui_npm_package_artifact.py @@ -84,7 +84,7 @@ def test_publish_workflows_restore_the_mode_preserving_archive() -> None: assert "--archive dist/npm-packages.tar.gz" in workflow -def test_raw_artifacts_do_not_inherit_the_packaging_version() -> None: +def test_build_workflow_executes_stamped_raw_and_packaged_artifacts() -> None: build = (ROOT / ".github/workflows/cmux-tui-build-package.yml").read_text() conditional_stamp = ( "CMUX_TUI_DISTRIBUTION_VERSION: " @@ -93,6 +93,9 @@ def test_raw_artifacts_do_not_inherit_the_packaging_version() -> None: assert "CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }}" not in build assert build.count(conditional_stamp) == 3 + assert build.count("verify_artifact_identity.py") == 4 + assert "dist/npm-smoke/node_modules/cmux/bin/cmux.js" in build + assert "/tmp/cmux-tui-wheel-smoke/bin/cmux" in build def main() -> None: @@ -101,7 +104,7 @@ def main() -> None: with tempfile.TemporaryDirectory() as directory: test_extract_rejects_paths_outside_package_root(Path(directory)) test_publish_workflows_restore_the_mode_preserving_archive() - test_raw_artifacts_do_not_inherit_the_packaging_version() + test_build_workflow_executes_stamped_raw_and_packaged_artifacts() if __name__ == "__main__": diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py new file mode 100644 index 000000000000..8ddb71afc17d --- /dev/null +++ b/tests/test_tui_package_launchers.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +VERIFY_SCRIPT = ROOT / "cmux-tui/dist/scripts/verify_artifact_identity.py" +NPM_LAUNCHER = ROOT / "cmux-tui/dist/npm/cmux/bin/cmux.js" +PYPI_PACKAGER = ROOT / "cmux-tui/dist/scripts/package_pypi.py" +EXPECTED_IDENTITY = "cmux-tui 1.2.3 (build123; ghostty ghost456)" + +NPM_PACKAGE_BY_PLATFORM = { + ("darwin", "arm64"): "cmux-tui-darwin-arm64", + ("darwin", "x86_64"): "cmux-tui-darwin-x64", + ("linux", "x86_64"): "cmux-tui-linux-x64", + ("linux", "aarch64"): "cmux-tui-linux-arm64", +} + +WHEEL_TAG_BY_PLATFORM = { + ("darwin", "arm64"): "macosx_11_0_arm64", + ("darwin", "x86_64"): "macosx_10_12_x86_64", + ("linux", "x86_64"): "manylinux_2_17_x86_64.manylinux2014_x86_64", + ("linux", "aarch64"): "manylinux_2_17_aarch64.manylinux2014_aarch64", +} + +RUST_TARGETS = ( + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", +) + + +def normalized_platform() -> tuple[str, str]: + machine = platform.machine().lower() + if machine in {"amd64", "x64"}: + machine = "x86_64" + elif machine == "arm64": + machine = "arm64" + return sys.platform, machine + + +def write_executable(path: Path, body: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"#!/bin/sh\n{body}\n") + path.chmod(0o755) + + +def run( + command: list[str | Path], + *, + cwd: Path = ROOT, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(part) for part in command], + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + + +def test_identity_verifier_executes_artifact_and_rejects_mismatch( + tmp_path: Path, +) -> None: + binary = tmp_path / "cmux-tui" + write_executable(binary, f"printf '%s\\n' '{EXPECTED_IDENTITY}'") + arguments = [ + sys.executable, + VERIFY_SCRIPT, + "--binary", + binary, + "--version", + "1.2.3", + "--build-commit", + "build123", + "--ghostty-commit", + "ghost456", + ] + + accepted = run(arguments) + assert accepted.returncode == 0, accepted.stderr + + rejected = run([*arguments[:-1], "wrong-ghostty"]) + assert rejected.returncode != 0 + assert "identity mismatch" in rejected.stderr + + +def test_npm_launcher_preserves_replayable_invocation(tmp_path: Path) -> None: + key = normalized_platform() + package = NPM_PACKAGE_BY_PLATFORM.get(key) + assert package is not None, f"unsupported test platform: {key}" + node = shutil.which("node") + assert node is not None, "node is required to exercise the npm launcher" + + launcher = tmp_path / "node_modules/cmux/bin/cmux.js" + launcher.parent.mkdir(parents=True) + shutil.copyfile(NPM_LAUNCHER, launcher) + launcher.chmod(0o755) + package_root = tmp_path / "node_modules" / package + package_root.mkdir(parents=True) + (package_root / "package.json").write_text( + f'{{"name":"{package}","version":"1.2.3"}}\n' + ) + write_executable( + package_root / "bin/cmux-tui", + "printf '%s\\n' \"$CMUX_TUI_LAUNCHER_COMMAND\"", + ) + + one_shot_env = os.environ.copy() + one_shot_env["npm_command"] = "exec" + one_shot = run([node, launcher], cwd=tmp_path, env=one_shot_env) + assert one_shot.returncode == 0, one_shot.stderr + assert one_shot.stdout.strip() == "npx cmux" + + installed_env = os.environ.copy() + installed_env.pop("npm_command", None) + installed = run([node, launcher], cwd=tmp_path, env=installed_env) + assert installed.returncode == 0, installed.stderr + assert installed.stdout.strip() == "cmux" + + +def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: + key = normalized_platform() + wheel_tag = WHEEL_TAG_BY_PLATFORM.get(key) + assert wheel_tag is not None, f"unsupported test platform: {key}" + + binaries = tmp_path / "binaries" + for target in RUST_TARGETS: + write_executable( + binaries / f"cmux-tui-{target}", + "printf '%s\\n' \"$CMUX_TUI_LAUNCHER_COMMAND\"", + ) + + wheels = tmp_path / "wheels" + packaged = run( + [ + sys.executable, + PYPI_PACKAGER, + "--binaries-dir", + binaries, + "--version", + "1.2.3", + "--out", + wheels, + ] + ) + assert packaged.returncode == 0, packaged.stderr + + wheel = wheels / f"cmux-1.2.3-py3-none-{wheel_tag}.whl" + installed = tmp_path / "installed" + with zipfile.ZipFile(wheel) as archive: + archive.extractall(installed) + (installed / "cmux_tui/bin/cmux-tui").chmod(0o755) + + environment = os.environ.copy() + environment["PYTHONPATH"] = str(installed) + launched = run( + [ + sys.executable, + "-c", + ( + "import sys; " + "sys.argv = ['/tmp/uv/archive-v0/hash/bin/cmux']; " + "from cmux_tui._main import main; " + "main()" + ), + ], + env=environment, + ) + assert launched.returncode == 0, launched.stderr + assert launched.stdout.strip() == "uvx cmux" + + +def main() -> None: + import tempfile + + with tempfile.TemporaryDirectory() as directory: + test_identity_verifier_executes_artifact_and_rejects_mismatch( + Path(directory) + ) + with tempfile.TemporaryDirectory() as directory: + test_npm_launcher_preserves_replayable_invocation(Path(directory)) + with tempfile.TemporaryDirectory() as directory: + test_pypi_launcher_preserves_uvx_invocation(Path(directory)) + + +if __name__ == "__main__": + main() From 4a451c23bc6a894cab82883214639070a1a1a0b7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:47:56 -0700 Subject: [PATCH 026/521] fix(tui): preserve replayable package launchers --- .github/workflows/cmux-tui-build-package.yml | 48 +++++++++++++--- .../crates/cmux-tui/src/server_lifecycle.rs | 36 ++++++++---- cmux-tui/dist/npm/cmux/bin/cmux.js | 5 ++ cmux-tui/dist/scripts/package_pypi.py | 16 ++++++ .../dist/scripts/verify_artifact_identity.py | 55 +++++++++++++++++++ 5 files changed, 140 insertions(+), 20 deletions(-) create mode 100644 cmux-tui/dist/scripts/verify_artifact_identity.py diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index d6259d9b4ab7..d2d7f1bfe624 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -136,6 +136,24 @@ jobs: cp "cmux-tui/target/${{ matrix.target }}/release/cmux-tui${{ matrix.ext }}" "dist/cmux-tui-${{ matrix.target }}${{ matrix.ext }}" ls -la dist + - name: Verify native binary identity + if: matrix.cross == false + shell: bash + env: + EXPECTED_VERSION: ${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }} + run: | + if [[ -z "$EXPECTED_VERSION" ]]; then + EXPECTED_VERSION="$( + cargo metadata --manifest-path cmux-tui/Cargo.toml --no-deps --format-version 1 | + python3 -c 'import json, sys; print(next(package["version"] for package in json.load(sys.stdin)["packages"] if package["name"] == "cmux-tui"))' + )" + fi + python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ + --binary "dist/cmux-tui-${{ matrix.target }}${{ matrix.ext }}" \ + --version "$EXPECTED_VERSION" \ + --build-commit "$(git rev-parse HEAD)" \ + --ghostty-commit "$(git -C ghostty rev-parse HEAD)" + - name: Upload binary artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -258,6 +276,15 @@ jobs: name: cmux-tui-aarch64-unknown-linux-gnu path: dist/binaries + - name: Verify downloaded binary identity + run: | + chmod +x dist/binaries/cmux-tui-x86_64-unknown-linux-gnu + python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ + --binary dist/binaries/cmux-tui-x86_64-unknown-linux-gnu \ + --version "$NPM_VERSION" \ + --build-commit "$(git rev-parse HEAD)" \ + --ghostty-commit "$(git rev-parse HEAD:ghostty)" + - name: Build npm package directories if: inputs.package_npm run: | @@ -308,9 +335,14 @@ jobs: if not binary.stat().st_mode & stat.S_IXUSR: raise SystemExit(f"{binary} is not executable") PY - chmod +x dist/binaries/cmux-tui-x86_64-unknown-linux-gnu - dist/binaries/cmux-tui-x86_64-unknown-linux-gnu --version >/tmp/cmux-tui-version.txt 2>&1 || \ - dist/binaries/cmux-tui-x86_64-unknown-linux-gnu --help >/tmp/cmux-tui-version.txt 2>&1 + mkdir -p dist/npm-smoke/node_modules + cp -R dist/npm-packages/cmux dist/npm-smoke/node_modules/cmux + cp -R dist/npm-packages/cmux-tui-linux-x64 dist/npm-smoke/node_modules/cmux-tui-linux-x64 + python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ + --binary dist/npm-smoke/node_modules/cmux/bin/cmux.js \ + --version "$NPM_VERSION" \ + --build-commit "$(git rev-parse HEAD)" \ + --ghostty-commit "$(git rev-parse HEAD:ghostty)" - name: Archive npm package directories with executable modes if: inputs.package_npm @@ -326,13 +358,13 @@ jobs: for wheel in dist/pypi-wheels/*.whl; do python3 -m zipfile -l "$wheel" >"/tmp/$(basename "$wheel").list" done - chmod +x dist/binaries/cmux-tui-x86_64-unknown-linux-gnu - dist/binaries/cmux-tui-x86_64-unknown-linux-gnu --version >/tmp/cmux-tui-version.txt 2>&1 || \ - dist/binaries/cmux-tui-x86_64-unknown-linux-gnu --help >/tmp/cmux-tui-version.txt 2>&1 python3 -m venv /tmp/cmux-tui-wheel-smoke /tmp/cmux-tui-wheel-smoke/bin/python -m pip install --no-index --find-links dist/pypi-wheels cmux=="$PYPI_VERSION" - /tmp/cmux-tui-wheel-smoke/bin/cmux --help >/tmp/cmux-help.txt - grep -i usage /tmp/cmux-help.txt + python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ + --binary /tmp/cmux-tui-wheel-smoke/bin/cmux \ + --version "$NPM_VERSION" \ + --build-commit "$(git rev-parse HEAD)" \ + --ghostty-commit "$(git rev-parse HEAD:ghostty)" - name: Upload npm package directories if: inputs.package_npm diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a1c00d17dabd..d301e281915d 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1,4 +1,5 @@ use std::collections::HashSet; +use std::ffi::OsStr; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; #[cfg(unix)] @@ -31,6 +32,8 @@ const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); const SHUTDOWN_RESPONSE_TIMEOUT: Duration = SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); +const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; +const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; type TransportReader = BufReader>; @@ -482,8 +485,23 @@ pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path } fn current_launcher_command() -> String { - std::env::args_os() - .next() + let override_command = std::env::var_os(LAUNCHER_COMMAND_ENV); + let argv0 = std::env::args_os().next(); + launcher_command_from(override_command.as_deref(), argv0.as_deref()) +} + +fn launcher_command_from(override_command: Option<&OsStr>, argv0: Option<&OsStr>) -> String { + if let Some(command) = + override_command.and_then(OsStr::to_str).map(str::trim).filter(|command| { + !command.is_empty() + && command.len() <= MAX_LAUNCHER_COMMAND_BYTES + && !command.chars().any(char::is_control) + }) + { + return command.to_string(); + } + + argv0 .filter(|value| !value.is_empty()) .map(PathBuf::from) .map(|path| shell_quote(&path)) @@ -659,18 +677,12 @@ mod tests { #[test] fn replayable_launcher_override_survives_one_shot_package_shims() { - let argv0 = std::ffi::OsStr::new("/tmp/ephemeral/bin/cmux"); + let argv0 = OsStr::new("/tmp/ephemeral/bin/cmux"); + assert_eq!(launcher_command_from(Some(OsStr::new("npx cmux")), Some(argv0)), "npx cmux"); + assert_eq!(launcher_command_from(Some(OsStr::new("uvx cmux")), Some(argv0)), "uvx cmux"); assert_eq!( - launcher_command_from(Some(std::ffi::OsStr::new("npx cmux")), Some(argv0)), - "npx cmux" - ); - assert_eq!( - launcher_command_from(Some(std::ffi::OsStr::new("uvx cmux")), Some(argv0)), - "uvx cmux" - ); - assert_eq!( - launcher_command_from(Some(std::ffi::OsStr::new("bad\ncommand")), Some(argv0)), + launcher_command_from(Some(OsStr::new("bad\ncommand")), Some(argv0)), "/tmp/ephemeral/bin/cmux" ); } diff --git a/cmux-tui/dist/npm/cmux/bin/cmux.js b/cmux-tui/dist/npm/cmux/bin/cmux.js index b00c4432ad01..6718930b3fb7 100644 --- a/cmux-tui/dist/npm/cmux/bin/cmux.js +++ b/cmux-tui/dist/npm/cmux/bin/cmux.js @@ -28,6 +28,7 @@ if (!pkg) { } const binName = process.platform === "win32" ? "cmux-tui.exe" : "cmux-tui"; +const launcherCommand = process.env.npm_command === "exec" ? "npx cmux" : "cmux"; let binPath; try { @@ -42,6 +43,10 @@ try { const result = spawnSync(binPath, process.argv.slice(2), { argv0: "cmux", + env: { + ...process.env, + CMUX_TUI_LAUNCHER_COMMAND: launcherCommand, + }, stdio: "inherit", }); diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index 0f4f1b41b51f..d46e2cc5ddf3 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -91,11 +91,27 @@ def wheel_bytes(version: str, tag: str, binary: bytes) -> list[tuple[str, bytes, import os import pathlib +import shlex import sys +def _launcher_command(argv0: str) -> str: + parts = pathlib.PurePath(argv0).parts + for index, part in enumerate(parts): + if part != "uv": + continue + if any( + candidate.startswith("archive-v") + and candidate.removeprefix("archive-v").isdigit() + for candidate in parts[index + 1 :] + ): + return "uvx cmux" + return shlex.quote(argv0) if argv0 else "cmux" + + def main() -> None: binary = pathlib.Path(__file__).resolve().parent / "bin" / "cmux-tui" + os.environ.setdefault("CMUX_TUI_LAUNCHER_COMMAND", _launcher_command(sys.argv[0])) os.execv(str(binary), ["cmux", *sys.argv[1:]]) """ ), diff --git a/cmux-tui/dist/scripts/verify_artifact_identity.py b/cmux-tui/dist/scripts/verify_artifact_identity.py new file mode 100644 index 000000000000..1dc12f2d12d6 --- /dev/null +++ b/cmux-tui/dist/scripts/verify_artifact_identity.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Execute a packaged cmux TUI artifact and verify its stamped identity.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--binary", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--build-commit", required=True) + parser.add_argument("--ghostty-commit", required=True) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + expected = ( + f"cmux-tui {args.version} " + f"({args.build_commit}; ghostty {args.ghostty_commit})" + ) + try: + completed = subprocess.run( + [str(args.binary), "--version"], + check=False, + capture_output=True, + text=True, + ) + except OSError as error: + raise SystemExit(f"artifact execution failed: {error}") from error + + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise SystemExit( + f"artifact execution failed with status {completed.returncode}: {detail}" + ) + + lines = completed.stdout.splitlines() + if lines != [expected] or completed.stderr: + actual = completed.stdout.rstrip("\r\n") + detail = f"; stderr: {completed.stderr.strip()}" if completed.stderr else "" + raise SystemExit( + f"identity mismatch: expected {expected!r}, got {actual!r}{detail}" + ) + + print(expected) + + +if __name__ == "__main__": + main() From 557b6128a442645f11116b345f0ed85c37c0e8a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:57:35 -0700 Subject: [PATCH 027/521] test(tui): cover transient legacy process metadata --- .../src/server_lifecycle/legacy_process.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 03ac61d91585..4860cb539fba 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -594,6 +594,8 @@ fn direct_child_pids(_parent: libc::pid_t) -> io::Result> { #[cfg(test)] mod tests { + #[cfg(any(target_os = "macos", target_os = "linux"))] + use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; #[cfg(target_os = "linux")] use std::sync::mpsc; @@ -632,6 +634,37 @@ mod tests { terminate_process_tree(process).unwrap(); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn captured_session_waits_for_an_unavailable_exact_member_to_exit() { + let mut command = Command::new("sleep"); + command.arg("60").stdout(Stdio::null()).stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut child = command.spawn().unwrap(); + let pid = libc::pid_t::try_from(child.id()).unwrap(); + let process = ProcessIdentity::capture(pid).unwrap().unwrap(); + let unavailable = + ProcessIdentity { pid: process.pid, started_at: process.started_at.wrapping_add(1) }; + let session = CapturedSession { id: pid, members: vec![unavailable] }; + let reaper = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + child.kill().unwrap(); + child.wait().unwrap() + }); + + let result = session.kill_until_empty(Instant::now() + Duration::from_secs(1)); + reaper.join().unwrap(); + + assert!(result.unwrap()); + } + #[cfg(target_os = "linux")] #[test] fn linux_child_enumeration_includes_children_spawned_by_worker_threads() { From a1f58c2f8bc3b6c20b136fe3528dfcaf46259b38 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:58:50 -0700 Subject: [PATCH 028/521] fix(tui): retry transient legacy process metadata --- .../src/server_lifecycle/legacy_process.rs | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 4860cb539fba..26772487b619 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -159,23 +159,38 @@ struct CapturedSession { impl CapturedSession { fn kill_until_empty(&self, deadline: Instant) -> io::Result { - let Some(mut probe) = SessionProbe::acquire(self)? else { - if cmux_tui_core::process_session::session_is_empty(self.id)? { - return Ok(true); + loop { + let error = match SessionProbe::acquire(self) { + Ok(Some(mut probe)) => { + match cmux_tui_core::process_session::kill_until_only_reserved( + self.id, + probe.process.pid, + deadline, + ) { + Ok(true) => match probe.kill() { + Ok(()) => return Ok(true), + Err(error) => error, + }, + Ok(false) => return Ok(false), + Err(error) => error, + } + } + Ok(None) => match cmux_tui_core::process_session::session_is_empty(self.id) { + Ok(true) => return Ok(true), + Ok(false) => io::Error::other( + "captured PTY session lost every exact process before termination", + ), + Err(error) => error, + }, + Err(error) => error, + }; + if Instant::now() >= deadline { + return Err(error); } - return Err(io::Error::other( - "captured PTY session lost every exact process before termination", - )); - }; - if !cmux_tui_core::process_session::kill_until_only_reserved( - self.id, - probe.process.pid, - deadline, - )? { - return Ok(false); + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(PROCESS_TREE_RETRY_INTERVAL), + ); } - probe.kill()?; - Ok(true) } } @@ -195,10 +210,7 @@ impl SessionProbe { { continue; } - if ProcessIdentity::capture(process.pid)? == Some(*process) { - return Ok(Some(Self { process: *process, session: session.id, armed: true })); - } - let _ = process.signal(libc::SIGCONT); + return Ok(Some(Self { process: *process, session: session.id, armed: true })); } Ok(None) } From bffb59c13fc9140233a38cf34ae6552b7c3906ad Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:01:52 -0700 Subject: [PATCH 029/521] test(tui): cover reset shutdown connections --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index d301e281915d..3ca38af67030 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -687,6 +687,18 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn shutdown_read_error_accepts_a_socket_that_stopped_listening() { + let path = std::env::temp_dir().join(format!( + "cmux-tui-shutdown-disconnect-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + + assert!(shutdown_read_error(&path).is_ok()); + } + #[cfg(unix)] #[test] fn legacy_stop_terminates_only_the_identified_process() { From c9dcdb96228db059c74eddcbd09491d9fb2fc10e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:03:37 -0700 Subject: [PATCH 030/521] fix(tui): accept verified shutdown disconnects --- .../crates/cmux-tui/src/server_lifecycle.rs | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 3ca38af67030..9ecc3d013c50 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -226,7 +226,7 @@ impl ServerLifecycle { Err(error) => return Err(error), }; if response.get("ok").and_then(Value::as_bool) == Some(true) { - return wait_for_disconnect(&mut self.reader); + return wait_for_disconnect(&mut self.reader, &self.path); } if let Some(error) = response.get("error").and_then(Value::as_str) { anyhow::bail!("{}: {error}", crate::localization::catalog().server.shutdown_failed); @@ -399,7 +399,7 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; terminate_captured_legacy_process_tree(captured)?; - wait_for_disconnect(&mut lifecycle.reader) + wait_for_disconnect(&mut lifecycle.reader, &lifecycle.path) } #[cfg(unix)] @@ -598,7 +598,7 @@ fn legacy_surface_ids(data: &Value) -> Vec { surfaces } -fn wait_for_disconnect(reader: &mut TransportReader) -> anyhow::Result<()> { +fn wait_for_disconnect(reader: &mut TransportReader, path: &Path) -> anyhow::Result<()> { let deadline = Instant::now() + RESPONSE_TIMEOUT; let mut line = String::new(); loop { @@ -616,13 +616,23 @@ fn wait_for_disconnect(reader: &mut TransportReader) -> anyhow::Result<()> { std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut ) => { - anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out) + if connection_is_gone(path) { + return Ok(()); + } + anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out); } - Err(_) => anyhow::bail!(crate::localization::catalog().server.transport_failed), + Err(_) => return shutdown_read_error(path), } } } +fn shutdown_read_error(path: &Path) -> anyhow::Result<()> { + if connection_is_gone(path) { + return Ok(()); + } + anyhow::bail!(crate::localization::catalog().server.transport_failed) +} + fn connection_is_gone(path: &Path) -> bool { transport::connect(path).is_err() } @@ -690,13 +700,18 @@ mod tests { #[cfg(unix)] #[test] fn shutdown_read_error_accepts_a_socket_that_stopped_listening() { - let path = std::env::temp_dir().join(format!( + let path = PathBuf::from("/tmp").join(format!( "cmux-tui-shutdown-disconnect-{}-{}.sock", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() )); assert!(shutdown_read_error(&path).is_ok()); + + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + assert!(shutdown_read_error(&path).is_err()); + drop(listener); + std::fs::remove_file(path).unwrap(); } #[cfg(unix)] From a662979c1e542ee789123e6202abbb4a3094983b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:07:28 -0700 Subject: [PATCH 031/521] test(tui): cover post-freeze metadata races --- .../src/server_lifecycle/legacy_process.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 26772487b619..6438067eb339 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -646,6 +646,38 @@ mod tests { terminate_process_tree(process).unwrap(); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn process_tree_termination_retries_post_freeze_metadata_errors() { + let mut child = Command::new("sleep") + .arg("60") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let process = + ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); + let mut attempts = 0; + + let result = retry_process_tree_termination( + process, + Instant::now() + Duration::from_secs(1), + |_| { + attempts += 1; + if attempts == 1 { + Err(io::Error::other("transient process metadata read")) + } else { + Ok(()) + } + }, + ); + child.kill().unwrap(); + child.wait().unwrap(); + + assert!(result.is_ok()); + assert_eq!(attempts, 2); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn captured_session_waits_for_an_unavailable_exact_member_to_exit() { From b0e410fa92097bd2d93418e7ad201ad8b1855b7c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:08:44 -0700 Subject: [PATCH 032/521] fix(tui): retry complete legacy termination --- .../src/server_lifecycle/legacy_process.rs | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 6438067eb339..f941c8538763 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -90,10 +90,39 @@ enum ExactSignalResult { } pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { - match freeze_process_tree(process) { - Ok(tree) => tree.kill(), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error), + let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; + retry_process_tree_termination(process, deadline, |_| { + let tree = FrozenProcessTree::freeze(process)?; + tree.kill() + }) +} + +fn retry_process_tree_termination( + process: ProcessIdentity, + deadline: Instant, + mut attempt: impl FnMut(Instant) -> io::Result<()>, +) -> io::Result<()> { + loop { + let error = match attempt(deadline) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + match ProcessIdentity::capture(process.pid) { + Ok(None) => return Ok(()), + Ok(Some(current)) if current != process => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "server process identity changed during termination", + )); + } + Ok(Some(_)) | Err(_) => {} + } + if Instant::now() >= deadline { + return Err(error); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(PROCESS_TREE_RETRY_INTERVAL), + ); } } From d353a53fba54cdbf41005d7a4086b8e55aaf4467 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:51:14 -0700 Subject: [PATCH 033/521] test(tui): cover shutdown ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 76 ++++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 22 ++++++ 2 files changed, 98 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index cd524a5f83cc..ac86f1c46679 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10031,6 +10031,47 @@ mod tests { assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); } + #[test] + fn ordinary_surface_close_retains_failed_process_ownership_for_shutdown() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + owned.set_server_shutdown_failure_for_test(true); + + assert!(mux.close_surface(surface.id).unwrap()); + assert!(mux.surface(surface.id).is_none()); + assert_eq!(mux.shutdown_surfaces.lock().unwrap().len(), 1); + + owned.set_server_shutdown_failure_for_test(false); + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); + } + + #[cfg(unix)] + #[test] + fn server_shutdown_rejects_malformed_host_records_before_removing_topology() { + let root = std::env::temp_dir().join(format!( + "cmux-shutdown-host-record-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join("00000000000040008000000000000031.json"), b"{not-valid-json") + .unwrap(); + let mux = Mux::new_for_test( + "strict-shutdown-records", + SurfaceOptions { terminal_host_root: Some(root.clone()), ..SurfaceOptions::default() }, + ); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + + let result = mux.close_all_surfaces_for_shutdown(); + let topology_retained = mux.surface(surface.id).is_some(); + let _ = std::fs::remove_dir_all(&root); + + let error = result.unwrap_err(); + assert!(format!("{error:#}").contains("load terminal hosts for server shutdown")); + assert!(topology_retained); + } + #[test] fn server_shutdown_waits_for_async_browser_bootstrap() { let mux = test_mux(); @@ -10117,6 +10158,41 @@ mod tests { assert!(observed.into_iter().all(|item_deadline| item_deadline == deadline)); } + #[test] + fn shutdown_fanout_does_not_claim_another_batch_after_the_deadline() { + let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(SHUTDOWN_FANOUT_WORKERS + 1); + let deadline = Instant::now() + Duration::from_millis(250); + let worker = std::thread::spawn({ + let gate = gate.clone(); + move || { + bounded_shutdown_fanout( + &vec![(); SHUTDOWN_FANOUT_WORKERS + 1], + deadline, + |_, _| { + started_tx.send(()).unwrap(); + let (released, wake) = &*gate; + let released = wake + .wait_while(released.lock().unwrap(), |released| !*released) + .unwrap(); + drop(released); + }, + ); + } + }); + + for _ in 0..SHUTDOWN_FANOUT_WORKERS { + started_rx.recv_timeout(Duration::from_millis(200)).unwrap(); + } + std::thread::sleep(deadline.saturating_duration_since(Instant::now())); + let (released, wake) = &*gate; + *released.lock().unwrap() = true; + wake.notify_all(); + worker.join().unwrap(); + + assert_eq!(started_rx.try_iter().count(), 0); + } + #[test] fn surface_creation_fence_has_a_bounded_wait() { let gate = SurfaceCreationGate::default(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 0b5b96eac75c..b3d2d5da0710 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2925,6 +2925,28 @@ mod tests { assert_eq!(signals.load(Ordering::Relaxed), 0); } + #[test] + fn local_shutdown_does_not_wait_for_the_pty_writer_lock() { + let mux = Mux::new_for_test("shutdown-writer-lock", SurfaceOptions::default()); + let surface = + Surface::spawn_for_test(1, SurfaceOptions::default(), Arc::downgrade(&mux)).unwrap(); + let locked = surface.clone(); + let (held_tx, held_rx) = sync_channel(1); + let holder = std::thread::spawn(move || { + let pty = locked.as_pty().unwrap(); + let _runtime = pty.runtime.lock().unwrap(); + held_tx.send(()).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }); + held_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + let started = Instant::now(); + assert!(surface.terminate_for_server_shutdown(started + Duration::from_millis(25))); + assert!(started.elapsed() < Duration::from_millis(100)); + + holder.join().unwrap(); + } + #[cfg(unix)] #[test] fn hosted_mirror_never_answers_terminal_queries() { From c95938d0f192ccfee2af02e50c040bea8915010c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:54:45 -0700 Subject: [PATCH 034/521] fix(tui): retain shutdown process ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 84 ++++++++++++++++--- cmux-tui/crates/cmux-tui-core/src/server.rs | 38 +++++++-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 54 +++++++++--- .../src/terminal_host_runtime.rs | 43 +++++++++- 4 files changed, 185 insertions(+), 34 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ac86f1c46679..487cf46ca450 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3474,7 +3474,7 @@ impl Mux { }; if let Some(surface) = removed { self.purge_surface_side_tables(surface.id); - surface.kill(); + self.retire_surface_runtime(surface); if let Some(delta) = delta { self.emit_tree_delta(delta, selection_resync); } else { @@ -3723,6 +3723,20 @@ impl Mux { self.reconcile_latest_client_size(&sizing, &attached_clients); } + /// Finish removed runtimes outside the topology lock. If the bounded + /// attempt cannot prove process death, retain the complete surface as + /// ownership escrow for the atomic server-shutdown retry. + fn retire_surface_runtime(&self, surface: Arc) { + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + if surface.terminate_for_server_shutdown(deadline) { + return; + } + let mut retained = self.shutdown_surfaces.lock().unwrap(); + if !retained.iter().any(|candidate| candidate.id == surface.id) { + retained.push(surface); + } + } + pub fn list_agents( &self, surface: Option, @@ -3781,10 +3795,44 @@ impl Mux { #[cfg(unix)] let terminal_host_records = { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + let required = self + .state + .lock() + .unwrap() + .surfaces + .values() + .filter(|surface| !surface.is_dead()) + .filter_map(|surface| surface.terminal_host_identity()) + .collect::>(); match root { - Some(root) => crate::terminal_host_runtime::load_terminal_host_records(&root) - .context("load terminal hosts for server shutdown")?, - None => Vec::new(), + Some(root) => { + let records = + crate::terminal_host_runtime::load_terminal_host_records_strict(&root) + .context("load terminal hosts for server shutdown")?; + let available = records + .iter() + .map(|(_, record)| { + (record.terminal_id.as_str(), record.incarnation.as_str()) + }) + .collect::>(); + for identity in required { + if !available.contains(&( + identity.terminal_id.as_str(), + identity.incarnation.as_str(), + )) { + anyhow::bail!( + "load terminal hosts for server shutdown: missing record for {}:{}", + identity.terminal_id, + identity.incarnation + ); + } + } + records + } + None if required.is_empty() => Vec::new(), + None => anyhow::bail!( + "load terminal hosts for server shutdown: terminal-host root is unavailable" + ), } }; @@ -3841,13 +3889,15 @@ impl Mux { } let failed_surfaces = Mutex::new(Vec::>::new()); - bounded_shutdown_fanout(&surfaces, deadline, |surface, deadline| { + let started_surfaces = bounded_shutdown_fanout(&surfaces, deadline, |surface, deadline| { if !surface.terminate_for_server_shutdown(deadline) { failed_surfaces.lock().unwrap().push(surface.clone()); } }); let mut failed_surfaces = failed_surfaces.into_inner().unwrap(); + failed_surfaces.extend(surfaces[started_surfaces..].iter().cloned()); failed_surfaces.sort_by_key(|surface| surface.id); + failed_surfaces.dedup_by_key(|surface| surface.id); let browser_surface_failed = failed_surfaces.iter().any(|surface| surface.as_browser().is_some()); let failed_surface_ids = @@ -3871,7 +3921,7 @@ impl Mux { #[cfg(unix)] let failed_hosts = { let failed = Mutex::new(Vec::new()); - bounded_shutdown_fanout( + let started_hosts = bounded_shutdown_fanout( &terminal_host_records, deadline, |(path, record), deadline| { @@ -3881,7 +3931,13 @@ impl Mux { }, ); let mut failed = failed.into_inner().unwrap(); + failed.extend( + terminal_host_records[started_hosts..] + .iter() + .map(|(_, record)| record.terminal_id.clone()), + ); failed.sort(); + failed.dedup(); failed }; @@ -3985,7 +4041,7 @@ impl Mux { if let Some(surface) = old_surface.and_then(|id| self.state.lock().unwrap().surfaces.remove(&id)) { - surface.kill(); + self.retire_surface_runtime(surface.clone()); self.emit(MuxEvent::SurfaceExited(surface.id)); } } @@ -5658,7 +5714,7 @@ impl Mux { }; if let Some(surface) = &removed { self.purge_surface_side_tables(surface.id); - surface.kill(); + self.retire_surface_runtime(surface.clone()); } if let Some(delta) = delta { self.emit_tree_delta(delta, selection_resync); @@ -5792,7 +5848,7 @@ impl Mux { }; for surface in removed { self.purge_surface_side_tables(surface.id); - surface.kill(); + self.retire_surface_runtime(surface); } if tree_removed { self.emit_tree_delta(delta, selection_resync); @@ -6078,7 +6134,7 @@ impl Mux { drop(registry); for surface in removed { self.purge_surface_side_tables(surface.id); - surface.kill(); + self.retire_surface_runtime(surface); } self.terminate_tombstoned_workspace_hosts(&closed_workspace_key); self.emit_tree_delta(delta, selection_resync); @@ -7757,9 +7813,9 @@ fn bounded_shutdown_fanout( items: &[T], deadline: Instant, operation: impl Fn(&T, Instant) + Sync, -) { +) -> usize { if items.is_empty() { - return; + return 0; } let next = AtomicUsize::new(0); let workers = items.len().min(SHUTDOWN_FANOUT_WORKERS); @@ -7769,6 +7825,9 @@ fn bounded_shutdown_fanout( let operation = &operation; scope.spawn(move || { loop { + if Instant::now() >= deadline { + break; + } let index = next.fetch_add(1, Ordering::Relaxed); let Some(item) = items.get(index) else { break }; operation(item, deadline); @@ -7776,6 +7835,7 @@ fn bounded_shutdown_fanout( }); } }); + next.load(Ordering::Relaxed).min(items.len()) } fn insert_surface_checked(state: &mut State, surface: Arc) -> anyhow::Result<()> { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index a7e779ea54f4..e1f3dee3bcba 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2932,6 +2932,28 @@ fn detach_committed_attach(mux: &Mux, client: u64, surface: SurfaceId, stream: u } } +fn advertised_capabilities() -> Vec<&'static str> { + // Atomic shutdown is advertised only where PTY session descendants can + // be enumerated and confirmed dead. + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + vec![ + ATTACH_INITIAL_SIZE_CAPABILITY, + WORKSPACE_REGISTRY_CAPABILITY, + PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, + SERVER_SHUTDOWN_CAPABILITY, + ] + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + vec![ + ATTACH_INITIAL_SIZE_CAPABILITY, + WORKSPACE_REGISTRY_CAPABILITY, + PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, + ] + } +} + fn handle_command( mux: &Arc, client: u64, @@ -2948,12 +2970,7 @@ fn handle_command( "build_commit": release.build_commit, "ghostty_commit": release.ghostty_commit, "protocol": release.protocol, - "capabilities": [ - ATTACH_INITIAL_SIZE_CAPABILITY, - WORKSPACE_REGISTRY_CAPABILITY, - PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, - SERVER_SHUTDOWN_CAPABILITY - ], + "capabilities": advertised_capabilities(), "session": mux.session, "pid": std::process::id(), "registry_id": registry_id, @@ -6498,10 +6515,17 @@ mod tests { "attach-initial-size", "workspace-registry-v1", PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, - SERVER_SHUTDOWN_CAPABILITY, ] { assert!(capabilities.iter().any(|value| value.as_str() == Some(expected))); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + assert!( + capabilities.iter().any(|value| value.as_str() == Some(SERVER_SHUTDOWN_CAPABILITY)) + ); + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + assert!( + capabilities.iter().all(|value| value.as_str() != Some(SERVER_SHUTDOWN_CAPABILITY)) + ); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index b3d2d5da0710..d914a59a85c0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -14,7 +14,7 @@ use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{ Receiver, RecvError, RecvTimeoutError, SyncSender, TryRecvError, TrySendError, sync_channel, }; -use std::sync::{Arc, Condvar, Mutex, TryLockError, Weak}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, TryLockError, Weak}; use std::time::{Duration, Instant}; use ghostty_vt::{ @@ -558,6 +558,9 @@ pub struct PtySurface { term: Mutex, mouse_encoders: Mutex, runtime: Mutex, + /// Local process ownership is separate from PTY writes so shutdown never + /// waits behind a blocked writer. + local_process: Option>, host_identity: Option, pid: Option, command: Vec, @@ -603,7 +606,6 @@ enum PtyRuntime { Local { writer: Box, master: Box, - process: LocalProcess, }, #[cfg(unix)] Hosted(Box), @@ -1038,11 +1040,8 @@ impl Surface { meta: SurfaceMeta { id, name: Mutex::new(None), selection: Mutex::new(None) }, term: Mutex::new(term), mouse_encoders: Mutex::new(mouse_encoders), - runtime: Mutex::new(PtyRuntime::Local { - writer, - master: pty.master, - process: LocalProcess::Owned(process.clone()), - }), + runtime: Mutex::new(PtyRuntime::Local { writer, master: pty.master }), + local_process: Some(Mutex::new(LocalProcess::Owned(process.clone()))), host_identity: None, pid, command: argv, @@ -1211,6 +1210,7 @@ impl Surface { term: Mutex::new(term), mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::Hosted(Box::new(attachment))), + local_process: None, host_identity: Some(host_identity), pid: snapshot.pid, command: snapshot.command, @@ -1702,6 +1702,7 @@ impl Surface { term: Mutex::new(term), mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::ExitedHosted), + local_process: None, host_identity: Some(identity), pid: None, command, @@ -1776,8 +1777,8 @@ impl Surface { pixel_height: 0, }), }), - process: LocalProcess::Untracked(Box::new(TestChildKiller)), }), + local_process: Some(Mutex::new(LocalProcess::Untracked(Box::new(TestChildKiller)))), host_identity: None, pid: Some(id as u32), command: opts.command.unwrap_or_else(|| vec![platform::default_shell()]), @@ -1808,8 +1809,8 @@ impl Surface { #[cfg(test)] pub(crate) fn set_server_shutdown_failure_for_test(&self, fail: bool) { let Self::Pty(pty) = self else { return }; - let mut runtime = pty.runtime.lock().unwrap(); - let PtyRuntime::Local { process, .. } = &mut *runtime else { return }; + let Some(process) = &pty.local_process else { return }; + let mut process = process.lock().unwrap(); *process = LocalProcess::Untracked(if fail { Box::new(TestFailingChildKiller) } else { @@ -2357,9 +2358,14 @@ impl Surface { fn terminate_runtime(&self) -> bool { match self { Surface::Pty(pty) => { + if let Some(process) = &pty.local_process { + return process.lock().unwrap().request_hangup(); + } let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { - PtyRuntime::Local { process, .. } => process.request_hangup(), + PtyRuntime::Local { .. } => { + unreachable!("local PTY runtime is missing process ownership") + } #[cfg(unix)] PtyRuntime::Hosted(host) => { // The host owns record cleanup and removes it only @@ -2384,7 +2390,15 @@ impl Surface { pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { match self { Surface::Pty(pty) => { - let mut runtime = pty.runtime.lock().unwrap(); + if let Some(process) = &pty.local_process { + let Some(mut process) = lock_mutex_until(process, deadline) else { + return false; + }; + return process.terminate_and_wait(deadline); + } + let Some(mut runtime) = lock_mutex_until(&pty.runtime, deadline) else { + return false; + }; match &mut *runtime { #[cfg(unix)] PtyRuntime::Hosted(host) => { @@ -2394,7 +2408,9 @@ impl Surface { let _ = host.terminate_with_timeout(timeout); true } - PtyRuntime::Local { process, .. } => process.terminate_and_wait(deadline), + PtyRuntime::Local { .. } => { + unreachable!("local PTY runtime is missing process ownership") + } #[cfg(unix)] PtyRuntime::ExitedHosted => true, } @@ -2539,6 +2555,18 @@ impl Surface { } } +fn lock_mutex_until(mutex: &Mutex, deadline: Instant) -> Option> { + loop { + match mutex.try_lock() { + Ok(guard) => return Some(guard), + Err(TryLockError::Poisoned(error)) => return Some(error.into_inner()), + Err(TryLockError::WouldBlock) => {} + } + let remaining = deadline.checked_duration_since(Instant::now())?; + std::thread::sleep(remaining.min(Duration::from_millis(1))); + } +} + #[cfg(test)] struct TestMasterPty { size: Mutex, diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index c0b1ba881ee1..3c17a7d075fc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1346,6 +1346,44 @@ mod unix { Ok(records) } + /// Load every discovery record without omission. Adoption tolerates + /// transient or stale files, but an atomic server shutdown must account + /// for every record before it discards topology. + pub fn load_terminal_host_records_strict( + root: &Path, + ) -> anyhow::Result> { + let mut records = Vec::new(); + let mut identities = HashSet::new(); + let entries = match fs::read_dir(root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(records), + Err(error) => return Err(error.into()), + }; + for entry in entries { + let entry = entry?; + let path = entry.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = fs::read(&path) + .with_context(|| format!("read terminal-host record {}", path.display()))?; + let record = serde_json::from_slice::(&bytes) + .with_context(|| format!("decode terminal-host record {}", path.display()))?; + validate_terminal_host_record(&path, &record) + .with_context(|| format!("validate terminal-host record {}", path.display()))?; + if !identities.insert((record.terminal_id.clone(), record.incarnation.clone())) { + anyhow::bail!( + "duplicate terminal-host identity {}:{}", + record.terminal_id, + record.incarnation + ); + } + records.push((path, record)); + } + records.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(records) + } + fn connect_record( record: TerminalHostRecord, record_path: PathBuf, @@ -3621,8 +3659,9 @@ mod unix { pub use unix::{ HostAttachment, adopt_terminal_host, isolate_terminal_host_process_fds, launch_terminal_host, launch_terminal_host_with_identity, load_terminal_host_records, - remove_stale_terminal_host_record, serve_terminal_host_stdio, terminal_host_record_liveness, - terminal_host_root, validate_terminal_host_record, + load_terminal_host_records_strict, remove_stale_terminal_host_record, + serve_terminal_host_stdio, terminal_host_record_liveness, terminal_host_root, + validate_terminal_host_record, }; #[cfg(unix)] pub(crate) use unix::{ From 0666934630a441a6d0d540aec246b1316d9a12de Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:57:05 -0700 Subject: [PATCH 035/521] test(tui): cover retryable browser shutdown --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 97 ++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 43d0910fe423..a9cf6f5ef44f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -2806,6 +2806,103 @@ mod tests { server.join().unwrap(); } + #[test] + fn server_shutdown_retries_an_unconfirmed_external_target() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (close_tx, close_rx) = mpsc::channel(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + + let first = read_ws_json(&mut ws); + assert_eq!(first["method"], "Target.closeTarget"); + close_tx.send(1).unwrap(); + + let second = read_ws_json(&mut ws); + assert_eq!(second["method"], "Target.closeTarget"); + assert_eq!(second["params"]["targetId"], "target-1"); + write_ws_json(&mut ws, json!({"id": second["id"], "result": {"success": true}})); + close_tx.send(2).unwrap(); + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + let surface = test_surface(); + let browser = surface.as_browser().unwrap(); + *browser.session.lock().unwrap() = Some(BrowserSession { + runtime: runtime.clone(), + target_id: "target-1".to_string(), + session_id: "session-1".to_string(), + }); + + assert!(!browser.terminate_for_server_shutdown(Instant::now() + Duration::from_millis(50))); + assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 1); + assert!(browser.session.lock().unwrap().is_some()); + assert!(browser.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1))); + assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); + assert!(browser.session.lock().unwrap().is_none()); + + runtime.shutdown(); + server.join().unwrap(); + } + + #[test] + fn runtime_shutdown_waits_for_a_backpressured_detached_target_close() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let (close_tx, close_rx) = mpsc::channel(); + let (read_tx, read_rx) = mpsc::channel(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + + if read_rx.recv_timeout(Duration::from_secs(2)).is_err() { + return; + } + let close = read_ws_json(&mut ws); + assert_eq!(close["method"], "Target.closeTarget"); + assert_eq!(close["params"]["targetId"].as_str().map(str::len), Some(8 * 1024 * 1024)); + close_tx.send(()).unwrap(); + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + + runtime.close_surface_detached(&"x".repeat(8 * 1024 * 1024), "session-1"); + let (shutdown_tx, shutdown_rx) = mpsc::channel(); + let shutdown_runtime = runtime.clone(); + let shutdown = thread::spawn(move || { + shutdown_tx + .send(shutdown_runtime.shutdown_until(Instant::now() + Duration::from_secs(2))) + .unwrap(); + }); + assert!( + shutdown_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "runtime shutdown returned before its queued target close was flushed" + ); + read_tx.send(()).unwrap(); + assert!(shutdown_rx.recv_timeout(Duration::from_secs(2)).unwrap()); + close_rx + .recv_timeout(Duration::from_secs(1)) + .expect("runtime shutdown dropped the detached target close"); + + shutdown.join().unwrap(); + server.join().unwrap(); + } + #[test] fn closed_surface_route_closes_its_cdp_target() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); From 826afc6db42b33fe374f0805c41610a3cc7d37a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:57:54 -0700 Subject: [PATCH 036/521] fix(tui): preserve browser shutdown retries --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 47 ++++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index a9cf6f5ef44f..e16871ba74c3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -546,11 +546,20 @@ impl BrowserRuntime { } pub(crate) fn shutdown_until(&self, deadline: Instant) -> bool { + let outbound_flushed = if self.source == BrowserSource::External { + if self.is_closed() { + false + } else { + deadline + .checked_duration_since(Instant::now()) + .is_some_and(|remaining| self.client.flush_outbound(remaining).is_ok()) + } + } else { + true + }; close_browser_runtime(self, "browser runtime shut down".to_string()); - if let Some(chrome) = &self.chrome { - return chrome.kill_until(deadline); - } - true + let browser_stopped = self.chrome.as_ref().is_none_or(|chrome| chrome.kill_until(deadline)); + outbound_flushed && browser_stopped } } @@ -1205,18 +1214,28 @@ impl BrowserSurface { } pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { - if self.dead.swap(true, Ordering::AcqRel) { + if !self.dead.swap(true, Ordering::AcqRel) { + self.close_taps(); + self.close_command_sender(); + } + let Some(session) = self.session.lock().unwrap().clone() else { return true; + }; + let terminated = session.runtime.close_surface_for_shutdown( + &session.target_id, + &session.session_id, + deadline, + ); + if terminated { + let mut current = self.session.lock().unwrap(); + if current.as_ref().is_some_and(|candidate| { + candidate.target_id == session.target_id + && candidate.session_id == session.session_id + && Arc::ptr_eq(&candidate.runtime, &session.runtime) + }) { + current.take(); + } } - self.close_taps(); - let terminated = self.session.lock().unwrap().take().is_none_or(|session| { - session.runtime.close_surface_for_shutdown( - &session.target_id, - &session.session_id, - deadline, - ) - }); - self.close_command_sender(); terminated } From 099383880722633c08c637b767f77bc5dc7ee9bd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:58:55 -0700 Subject: [PATCH 037/521] test(tui): satisfy shutdown lint gate --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index e16871ba74c3..94701538e2d5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -2902,7 +2902,7 @@ mod tests { runtime.close_surface_detached(&"x".repeat(8 * 1024 * 1024), "session-1"); let (shutdown_tx, shutdown_rx) = mpsc::channel(); - let shutdown_runtime = runtime.clone(); + let shutdown_runtime = runtime; let shutdown = thread::spawn(move || { shutdown_tx .send(shutdown_runtime.shutdown_until(Instant::now() + Duration::from_secs(2))) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 487cf46ca450..ad6952f344ba 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10227,7 +10227,7 @@ mod tests { let gate = gate.clone(); move || { bounded_shutdown_fanout( - &vec![(); SHUTDOWN_FANOUT_WORKERS + 1], + &[(); SHUTDOWN_FANOUT_WORKERS + 1], deadline, |_, _| { started_tx.send(()).unwrap(); From 3ff87f91c8cbb021bb86a0d00d1fe1f8aad05444 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:29:41 -0700 Subject: [PATCH 038/521] test(tui): cover lightweight close ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 64 ++++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 24 ++++++++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ad6952f344ba..98698ef1bf09 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10107,6 +10107,51 @@ mod tests { assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); } + #[test] + fn ordinary_surface_close_does_not_retain_terminal_render_state() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + owned.set_server_shutdown_failure_for_test(true); + let before_close = Arc::strong_count(&owned); + + assert!(mux.close_surface(surface.id).unwrap()); + + assert!( + Arc::strong_count(&owned) < before_close, + "shutdown escrow retained the heavyweight Surface" + ); + } + + #[test] + fn bulk_surface_close_uses_one_shared_termination_deadline() { + let mux = test_mux(); + let first = mux.new_workspace(None, Some((80, 24))).unwrap(); + let pane = mux.with_state(|state| state.pane_of(first.id).unwrap()); + for _ in 1..64 { + mux.new_tab(Some(pane), None, None).unwrap(); + } + let surfaces = mux.with_state(|state| { + state.panes[&pane] + .tabs + .iter() + .map(|surface| state.surfaces[surface].clone()) + .collect::>() + }); + for surface in &surfaces { + surface.set_server_shutdown_delay_for_test(Duration::from_millis(50)); + } + + let started = Instant::now(); + assert!(mux.close_pane(pane).unwrap()); + + assert!( + started.elapsed() < Duration::from_millis(500), + "bulk close multiplied the per-surface shutdown timeout: {:?}", + started.elapsed() + ); + } + #[cfg(unix)] #[test] fn server_shutdown_rejects_malformed_host_records_before_removing_topology() { @@ -10226,18 +10271,13 @@ mod tests { let worker = std::thread::spawn({ let gate = gate.clone(); move || { - bounded_shutdown_fanout( - &[(); SHUTDOWN_FANOUT_WORKERS + 1], - deadline, - |_, _| { - started_tx.send(()).unwrap(); - let (released, wake) = &*gate; - let released = wake - .wait_while(released.lock().unwrap(), |released| !*released) - .unwrap(); - drop(released); - }, - ); + bounded_shutdown_fanout(&[(); SHUTDOWN_FANOUT_WORKERS + 1], deadline, |_, _| { + started_tx.send(()).unwrap(); + let (released, wake) = &*gate; + let released = + wake.wait_while(released.lock().unwrap(), |released| !*released).unwrap(); + drop(released); + }); } }); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index d914a59a85c0..22b64367579c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1818,6 +1818,14 @@ impl Surface { }); } + #[cfg(test)] + pub(crate) fn set_server_shutdown_delay_for_test(&self, delay: Duration) { + let Self::Pty(pty) = self else { return }; + let Some(process) = &pty.local_process else { return }; + *process.lock().unwrap() = + LocalProcess::Untracked(Box::new(TestSlowFailingChildKiller(delay))); + } + fn as_pty(&self) -> Option<&PtySurface> { match self { Surface::Pty(surface) => Some(surface), @@ -2637,6 +2645,22 @@ impl ChildKiller for TestFailingChildKiller { } } +#[cfg(test)] +#[derive(Debug)] +struct TestSlowFailingChildKiller(Duration); + +#[cfg(test)] +impl ChildKiller for TestSlowFailingChildKiller { + fn kill(&mut self) -> std::io::Result<()> { + std::thread::sleep(self.0); + Err(std::io::Error::other("forced delayed shutdown failure")) + } + + fn clone_killer(&self) -> Box { + Box::new(Self(self.0)) + } +} + impl PtySurface { /// Snapshot sparse colors and the host-resolved cursor visual without /// touching the shared renderer or consuming its damage. From f56c225a50f878e7d5838c5b454ef1bbfb7897a8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:38:07 -0700 Subject: [PATCH 039/521] fix(tui): retain lightweight shutdown owners --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 18 ++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 251 ++++++++---------- cmux-tui/crates/cmux-tui-core/src/surface.rs | 160 +++++++---- .../src/terminal_host_runtime.rs | 55 +++- 4 files changed, 298 insertions(+), 186 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 94701538e2d5..9f303bf558ef 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -104,6 +104,15 @@ struct BrowserSession { session_id: String, } +#[derive(Clone)] +pub(crate) struct BrowserShutdownOwner(BrowserSession); + +impl BrowserShutdownOwner { + pub(crate) fn terminate_until(&self, deadline: Instant) -> bool { + self.0.runtime.close_surface_for_shutdown(&self.0.target_id, &self.0.session_id, deadline) + } +} + struct BrowserState { latest_frame: Option, // Latest-wins attach frame taps. Broadcast overwrites each slot and @@ -1152,6 +1161,14 @@ fn emit_browser_failure(mux: &Weak, id: SurfaceId, message: String) { } impl BrowserSurface { + pub(crate) fn shutdown_owner(&self) -> Option { + if !self.dead.swap(true, Ordering::AcqRel) { + self.close_taps(); + self.close_command_sender(); + } + self.session.lock().unwrap().clone().map(BrowserShutdownOwner) + } + pub fn latest_frame(&self) -> Option { let state = self.state.lock().unwrap(); if matches!(state.status, BrowserStatus::Failed(_)) { @@ -1213,6 +1230,7 @@ impl BrowserSurface { self.close_command_sender(); } + #[cfg(test)] pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { if !self.dead.swap(true, Ordering::AcqRel) { self.close_taps(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 98698ef1bf09..102907d7769f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -20,7 +20,7 @@ use crate::event_bus::{MuxEventBroadcaster, MuxEventReceiver}; use crate::layout::{Rect, layout_screen}; use crate::model::{Node, Pane, Screen, State, Workspace}; use crate::pairing::PairingBroker; -use crate::surface::{DefaultColors, Surface, SurfaceOptions}; +use crate::surface::{DefaultColors, Surface, SurfaceOptions, SurfaceShutdownOwner}; use crate::terminal_host::TerminalId; use crate::terminal_host_runtime::TerminalHostIdentity; #[cfg(unix)] @@ -45,6 +45,22 @@ const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); const SHUTDOWN_FANOUT_WORKERS: usize = 32; +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +enum ShutdownOwnerKey { + Surface(SurfaceId), + Hosted { terminal_id: String, incarnation: String }, +} + +fn shutdown_owner_key(surface: SurfaceId, owner: &SurfaceShutdownOwner) -> ShutdownOwnerKey { + owner + .hosted_identity() + .map(|(terminal_id, incarnation)| ShutdownOwnerKey::Hosted { + terminal_id: terminal_id.to_string(), + incarnation: incarnation.to_string(), + }) + .unwrap_or(ShutdownOwnerKey::Surface(surface)) +} + /// An opaque per-mux credential provisioned by the external machine /// provider. Debug output is deliberately redacted. #[derive(PartialEq, Eq)] @@ -852,7 +868,7 @@ pub struct Mux { surface_creations: SurfaceCreationGate, async_surface_creations: AsyncSurfaceCreationGate, shutdown_coordinator: Mutex<()>, - shutdown_surfaces: Mutex>>, + shutdown_owners: Mutex>, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -1085,7 +1101,7 @@ impl Mux { surface_creations: SurfaceCreationGate::default(), async_surface_creations: AsyncSurfaceCreationGate::default(), shutdown_coordinator: Mutex::new(()), - shutdown_surfaces: Mutex::new(Vec::new()), + shutdown_owners: Mutex::new(HashMap::new()), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -3723,17 +3739,37 @@ impl Mux { self.reconcile_latest_client_size(&sizing, &attached_clients); } - /// Finish removed runtimes outside the topology lock. If the bounded - /// attempt cannot prove process death, retain the complete surface as - /// ownership escrow for the atomic server-shutdown retry. + /// Finish removed runtimes outside the topology lock. One shared deadline + /// bounds an entire pane/workspace close, and failures retain only the + /// minimal process/target owner needed for a later retry. fn retire_surface_runtime(&self, surface: Arc) { - let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; - if surface.terminate_for_server_shutdown(deadline) { + self.retire_surface_runtimes([surface]); + } + + fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { + let mut owners = self.shutdown_owners.lock().unwrap().drain().collect::>(); + for surface in surfaces { + if let Some(owner) = surface.shutdown_owner() { + owners.insert(shutdown_owner_key(surface.id, &owner), owner); + } + } + let owners = owners.into_iter().collect::>(); + if owners.is_empty() { return; } - let mut retained = self.shutdown_surfaces.lock().unwrap(); - if !retained.iter().any(|candidate| candidate.id == surface.id) { - retained.push(surface); + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let failed = Mutex::new(HashSet::new()); + let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { + if !owner.terminate_until(deadline) { + failed.lock().unwrap().insert(key.clone()); + } + }); + let failed = failed.into_inner().unwrap(); + let mut retained = self.shutdown_owners.lock().unwrap(); + for (index, (key, owner)) in owners.into_iter().enumerate() { + if index >= started || failed.contains(&key) { + retained.insert(key, owner); + } } } @@ -3756,12 +3792,20 @@ impl Mux { let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; let _ = self.surface_creations.stop_and_wait_until(deadline); let _ = self.async_surface_creations.stop_and_wait_until(deadline); - let mut surfaces = - self.state.lock().unwrap().surfaces.values().cloned().collect::>(); - surfaces.extend(self.shutdown_surfaces.lock().unwrap().drain(..)); + let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); for surface in surfaces { surface.disconnect_for_daemon_shutdown(); } + let retired = self + .shutdown_owners + .lock() + .unwrap() + .drain() + .map(|(_, owner)| owner) + .collect::>(); + bounded_shutdown_fanout(&retired, deadline, |owner, deadline| { + let _ = owner.terminate_until(deadline); + }); if let Some(runtime) = self.browser_runtime.lock().unwrap().take() { let _ = runtime.shutdown_until(deadline); } @@ -3836,7 +3880,7 @@ impl Mux { } }; - let (mut surfaces, tree_changed) = { + let (surfaces, tree_changed) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); let mut registry = self.workspace_registry.lock().unwrap(); let terminals = registry @@ -3871,8 +3915,6 @@ impl Mux { } (surfaces, tree_changed) }; - surfaces.extend(self.shutdown_surfaces.lock().unwrap().drain(..)); - self.pending_workspace_surfaces.lock().unwrap().clear(); self.agent_records.lock().unwrap().clear(); self.surface_notifications.lock().unwrap().clear(); @@ -3888,21 +3930,59 @@ impl Mux { self.emit(MuxEvent::TreeChanged); } - let failed_surfaces = Mutex::new(Vec::>::new()); - let started_surfaces = bounded_shutdown_fanout(&surfaces, deadline, |surface, deadline| { - if !surface.terminate_for_server_shutdown(deadline) { - failed_surfaces.lock().unwrap().push(surface.clone()); + let mut owners = self.shutdown_owners.lock().unwrap().drain().collect::>(); + let closed_count = surfaces.len() + owners.len(); + for surface in surfaces { + if let Some(owner) = surface.shutdown_owner() { + owners.insert(shutdown_owner_key(surface.id, &owner), owner); + } + } + #[cfg(unix)] + for (record_path, record) in terminal_host_records { + let owner = SurfaceShutdownOwner::hosted(record, record_path); + let key = owner + .hosted_identity() + .map(|(terminal_id, incarnation)| ShutdownOwnerKey::Hosted { + terminal_id: terminal_id.to_string(), + incarnation: incarnation.to_string(), + }) + .expect("hosted owner has a terminal identity"); + owners.entry(key).or_insert(owner); + } + let owners = owners.into_iter().collect::>(); + let failed_keys = Mutex::new(HashSet::new()); + let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { + if !owner.terminate_until(deadline) { + failed_keys.lock().unwrap().insert(key.clone()); } }); - let mut failed_surfaces = failed_surfaces.into_inner().unwrap(); - failed_surfaces.extend(surfaces[started_surfaces..].iter().cloned()); - failed_surfaces.sort_by_key(|surface| surface.id); - failed_surfaces.dedup_by_key(|surface| surface.id); + let mut failed_keys = failed_keys.into_inner().unwrap(); + failed_keys.extend(owners[started..].iter().map(|(key, _)| key.clone())); let browser_surface_failed = - failed_surfaces.iter().any(|surface| surface.as_browser().is_some()); - let failed_surface_ids = - failed_surfaces.iter().map(|surface| surface.id).collect::>(); - *self.shutdown_surfaces.lock().unwrap() = failed_surfaces; + owners.iter().any(|(key, owner)| failed_keys.contains(key) && owner.is_browser()); + let mut failed_surface_ids = failed_keys + .iter() + .filter_map(|key| match key { + ShutdownOwnerKey::Surface(surface) => Some(*surface), + ShutdownOwnerKey::Hosted { .. } => None, + }) + .collect::>(); + failed_surface_ids.sort_unstable(); + let mut failed_hosts = failed_keys + .iter() + .filter_map(|key| match key { + ShutdownOwnerKey::Hosted { terminal_id, .. } => Some(terminal_id.clone()), + ShutdownOwnerKey::Surface(_) => None, + }) + .collect::>(); + failed_hosts.sort(); + let mut retained = self.shutdown_owners.lock().unwrap(); + for (key, owner) in owners { + if failed_keys.contains(&key) { + retained.insert(key, owner); + } + } + drop(retained); let browser_runtime = (!browser_surface_failed) .then(|| self.browser_runtime.lock().unwrap().take()) @@ -3918,29 +3998,6 @@ impl Mux { false }; - #[cfg(unix)] - let failed_hosts = { - let failed = Mutex::new(Vec::new()); - let started_hosts = bounded_shutdown_fanout( - &terminal_host_records, - deadline, - |(path, record), deadline| { - if !terminate_and_confirm_terminal_host_record(record, path, deadline) { - failed.lock().unwrap().push(record.terminal_id.clone()); - } - }, - ); - let mut failed = failed.into_inner().unwrap(); - failed.extend( - terminal_host_records[started_hosts..] - .iter() - .map(|(_, record)| record.terminal_id.clone()), - ); - failed.sort(); - failed.dedup(); - failed - }; - let mut failures = Vec::new(); if !failed_surface_ids.is_empty() { failures.push(format!( @@ -3956,7 +4013,6 @@ impl Mux { if browser_runtime_failed { failures.push("could not terminate the owned browser before the deadline".to_string()); } - #[cfg(unix)] if !failed_hosts.is_empty() { failures.push(format!( "could not terminate {} terminal host(s): {}", @@ -3968,7 +4024,7 @@ impl Mux { anyhow::bail!("{}", failures.join("; ")); } - Ok(surfaces.len()) + Ok(closed_count) } fn lock_shutdown_coordinator_until( @@ -5846,10 +5902,10 @@ impl Mux { else { return Ok(false); }; - for surface in removed { + for surface in &removed { self.purge_surface_side_tables(surface.id); - self.retire_surface_runtime(surface); } + self.retire_surface_runtimes(removed); if tree_removed { self.emit_tree_delta(delta, selection_resync); for screen in changed_screens { @@ -6132,10 +6188,10 @@ impl Mux { self.emit_terminal_registry_changed(®istry, terminal_revision_after); } drop(registry); - for surface in removed { + for surface in &removed { self.purge_surface_side_tables(surface.id); - self.retire_surface_runtime(surface); } + self.retire_surface_runtimes(removed); self.terminate_tombstoned_workspace_hosts(&closed_workspace_key); self.emit_tree_delta(delta, selection_resync); self.emit_empty_if_current(empty_revision); @@ -7700,31 +7756,6 @@ fn terminate_host_record( } } -#[cfg(unix)] -fn terminate_host_record_with_timeout( - record: crate::terminal_host_runtime::TerminalHostRecord, - record_path: std::path::PathBuf, - timeout: Duration, -) -> bool { - if record.supports_terminate_only { - return crate::terminal_host_runtime::terminate_terminal_host_with_timeout( - &record, - &record_path, - timeout, - ) - .is_ok(); - } - if let Ok(host) = - crate::terminal_host_runtime::adopt_terminal_host_with_timeout(record, record_path, timeout) - { - let terminated = host.terminate_with_timeout(timeout).is_ok(); - host.disconnect(); - terminated - } else { - false - } -} - #[cfg(unix)] fn terminal_host_record_liveness( record_path: &Path, @@ -7763,52 +7794,6 @@ fn cleanup_terminal_host_record( } } -#[cfg(unix)] -fn terminate_and_confirm_terminal_host_record( - record: &crate::terminal_host_runtime::TerminalHostRecord, - record_path: &Path, - deadline: Instant, -) -> bool { - let mut termination_sent = false; - let mut last_terminate_attempt = None; - loop { - match terminal_host_record_liveness(record_path, record) { - TerminalHostLiveness::Dead => { - return match crate::terminal_host_runtime::remove_stale_terminal_host_record( - record_path, - record, - ) { - Ok(removed) => removed, - Err(_) if !record_path.exists() => true, - Err(_) => false, - }; - } - TerminalHostLiveness::Live | TerminalHostLiveness::Indeterminate => {} - } - - let now = Instant::now(); - if !termination_sent - && last_terminate_attempt.is_none_or(|attempt: Instant| { - now.duration_since(attempt) >= Duration::from_millis(100) - }) - { - termination_sent = terminate_host_record_with_timeout( - record.clone(), - record_path.to_path_buf(), - deadline - .saturating_duration_since(Instant::now()) - .min(SHUTDOWN_TERMINATION_TIMEOUT), - ); - last_terminate_attempt = Some(Instant::now()); - } - let now = Instant::now(); - if now >= deadline { - return false; - } - std::thread::sleep(deadline.saturating_duration_since(now).min(Duration::from_millis(10))); - } -} - fn bounded_shutdown_fanout( items: &[T], deadline: Instant, @@ -10084,11 +10069,11 @@ mod tests { assert!(error.to_string().contains("could not terminate 1 surface process")); assert!(mux.surface(surface.id).is_none()); - assert_eq!(mux.shutdown_surfaces.lock().unwrap().len(), 1); + assert_eq!(mux.shutdown_owners.lock().unwrap().len(), 1); owned.set_server_shutdown_failure_for_test(false); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); - assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); + assert!(mux.shutdown_owners.lock().unwrap().is_empty()); } #[test] @@ -10100,11 +10085,11 @@ mod tests { assert!(mux.close_surface(surface.id).unwrap()); assert!(mux.surface(surface.id).is_none()); - assert_eq!(mux.shutdown_surfaces.lock().unwrap().len(), 1); + assert_eq!(mux.shutdown_owners.lock().unwrap().len(), 1); owned.set_server_shutdown_failure_for_test(false); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); - assert!(mux.shutdown_surfaces.lock().unwrap().is_empty()); + assert!(mux.shutdown_owners.lock().unwrap().is_empty()); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 22b64367579c..1cf1c6843f1a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -14,7 +14,7 @@ use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{ Receiver, RecvError, RecvTimeoutError, SyncSender, TryRecvError, TrySendError, sync_channel, }; -use std::sync::{Arc, Condvar, Mutex, MutexGuard, TryLockError, Weak}; +use std::sync::{Arc, Condvar, Mutex, TryLockError, Weak}; use std::time::{Duration, Instant}; use ghostty_vt::{ @@ -29,7 +29,9 @@ use crate::{Mux, MuxEvent, SurfaceId}; pub use crate::browser::{ BrowserAttachState, BrowserFrame, BrowserFrameStream, BrowserSource, BrowserStatus, }; -use crate::browser::{BrowserResizeWaiter, BrowserSurface, PendingBrowserResize}; +use crate::browser::{ + BrowserResizeWaiter, BrowserShutdownOwner, BrowserSurface, PendingBrowserResize, +}; #[cfg(unix)] use crate::terminal_host_protocol::{FLAG_COLORS_FOLLOW, Frame, MessageKind, PROTOCOL_VERSION}; use cmux_tui_cdp::BrowserMode; @@ -561,6 +563,8 @@ pub struct PtySurface { /// Local process ownership is separate from PTY writes so shutdown never /// waits behind a blocked writer. local_process: Option>, + #[cfg(unix)] + hosted_shutdown_owner: Option>, host_identity: Option, pid: Option, command: Vec, @@ -613,30 +617,93 @@ enum PtyRuntime { ExitedHosted, } +#[derive(Clone)] enum LocalProcess { Owned(Arc), #[cfg(test)] - Untracked(Box), + Untracked(Arc>>), } impl LocalProcess { - fn request_hangup(&mut self) -> bool { + #[cfg(test)] + fn untracked(killer: Box) -> Self { + Self::Untracked(Arc::new(Mutex::new(killer))) + } + + fn request_hangup(&self) -> bool { match self { Self::Owned(process) => process.request_hangup(), #[cfg(test)] - Self::Untracked(killer) => killer.kill().is_ok(), + Self::Untracked(killer) => killer.lock().unwrap().kill().is_ok(), } } - fn terminate_and_wait(&mut self, deadline: Instant) -> bool { + fn terminate_and_wait(&self, deadline: Instant) -> bool { match self { Self::Owned(process) => process.terminate_and_wait(deadline), #[cfg(test)] - Self::Untracked(killer) => killer.kill().is_ok(), + Self::Untracked(killer) => killer.lock().unwrap().kill().is_ok(), } } } +#[cfg(unix)] +#[derive(Clone)] +struct HostedShutdownOwner { + record: crate::terminal_host_runtime::TerminalHostRecord, + record_path: PathBuf, +} + +pub(crate) struct SurfaceShutdownOwner { + kind: SurfaceShutdownOwnerKind, +} + +enum SurfaceShutdownOwnerKind { + Local(LocalProcess), + #[cfg(unix)] + Hosted(HostedShutdownOwner), + Browser(BrowserShutdownOwner), +} + +impl SurfaceShutdownOwner { + pub(crate) fn terminate_until(&self, deadline: Instant) -> bool { + match &self.kind { + SurfaceShutdownOwnerKind::Local(process) => process.terminate_and_wait(deadline), + #[cfg(unix)] + SurfaceShutdownOwnerKind::Hosted(owner) => { + crate::terminal_host_runtime::terminate_and_confirm_terminal_host_record( + &owner.record, + &owner.record_path, + deadline, + ) + } + SurfaceShutdownOwnerKind::Browser(owner) => owner.terminate_until(deadline), + } + } + + pub(crate) fn hosted_identity(&self) -> Option<(&str, &str)> { + match &self.kind { + #[cfg(unix)] + SurfaceShutdownOwnerKind::Hosted(owner) => { + Some((&owner.record.terminal_id, &owner.record.incarnation)) + } + SurfaceShutdownOwnerKind::Local(_) | SurfaceShutdownOwnerKind::Browser(_) => None, + } + } + + pub(crate) fn is_browser(&self) -> bool { + matches!(self.kind, SurfaceShutdownOwnerKind::Browser(_)) + } + + #[cfg(unix)] + pub(crate) fn hosted( + record: crate::terminal_host_runtime::TerminalHostRecord, + record_path: PathBuf, + ) -> Self { + Self { kind: SurfaceShutdownOwnerKind::Hosted(HostedShutdownOwner { record, record_path }) } + } +} + struct LocalPtyProcess { killer: Mutex>, #[cfg(unix)] @@ -1042,6 +1109,8 @@ impl Surface { mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::Local { writer, master: pty.master }), local_process: Some(Mutex::new(LocalProcess::Owned(process.clone()))), + #[cfg(unix)] + hosted_shutdown_owner: None, host_identity: None, pid, command: argv, @@ -1203,6 +1272,7 @@ impl Surface { mouse_encoders.sync_from_terminal(&term); let sequence_boundary = snapshot.sequence_boundary; let host_identity = attachment.identity(); + let (host_record, host_record_path) = attachment.discovery_record(); let render_state = RenderState::new()?; let (frame_requests, frame_rx) = sync_channel(1); let surface = Arc::new(Surface::Pty(PtySurface { @@ -1211,6 +1281,10 @@ impl Surface { mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::Hosted(Box::new(attachment))), local_process: None, + hosted_shutdown_owner: Some(Box::new(HostedShutdownOwner { + record: host_record, + record_path: host_record_path, + })), host_identity: Some(host_identity), pid: snapshot.pid, command: snapshot.command, @@ -1703,6 +1777,7 @@ impl Surface { mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::ExitedHosted), local_process: None, + hosted_shutdown_owner: None, host_identity: Some(identity), pid: None, command, @@ -1778,7 +1853,9 @@ impl Surface { }), }), }), - local_process: Some(Mutex::new(LocalProcess::Untracked(Box::new(TestChildKiller)))), + local_process: Some(Mutex::new(LocalProcess::untracked(Box::new(TestChildKiller)))), + #[cfg(unix)] + hosted_shutdown_owner: None, host_identity: None, pid: Some(id as u32), command: opts.command.unwrap_or_else(|| vec![platform::default_shell()]), @@ -1810,20 +1887,19 @@ impl Surface { pub(crate) fn set_server_shutdown_failure_for_test(&self, fail: bool) { let Self::Pty(pty) = self else { return }; let Some(process) = &pty.local_process else { return }; - let mut process = process.lock().unwrap(); - *process = LocalProcess::Untracked(if fail { - Box::new(TestFailingChildKiller) - } else { - Box::new(TestChildKiller) - }); + let process = process.lock().unwrap(); + let LocalProcess::Untracked(killer) = &*process else { return }; + *killer.lock().unwrap() = + if fail { Box::new(TestFailingChildKiller) } else { Box::new(TestChildKiller) }; } #[cfg(test)] pub(crate) fn set_server_shutdown_delay_for_test(&self, delay: Duration) { let Self::Pty(pty) = self else { return }; let Some(process) = &pty.local_process else { return }; - *process.lock().unwrap() = - LocalProcess::Untracked(Box::new(TestSlowFailingChildKiller(delay))); + let process = process.lock().unwrap(); + let LocalProcess::Untracked(killer) = &*process else { return }; + *killer.lock().unwrap() = Box::new(TestSlowFailingChildKiller(delay)); } fn as_pty(&self) -> Option<&PtySurface> { @@ -2395,38 +2471,32 @@ impl Surface { /// Start runtime termination. Durable terminal-host death is confirmed /// separately through the process-bound discovery record. - pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { + pub(crate) fn shutdown_owner(&self) -> Option { match self { Surface::Pty(pty) => { if let Some(process) = &pty.local_process { - let Some(mut process) = lock_mutex_until(process, deadline) else { - return false; - }; - return process.terminate_and_wait(deadline); + return Some(SurfaceShutdownOwner { + kind: SurfaceShutdownOwnerKind::Local(process.lock().unwrap().clone()), + }); } - let Some(mut runtime) = lock_mutex_until(&pty.runtime, deadline) else { - return false; - }; - match &mut *runtime { - #[cfg(unix)] - PtyRuntime::Hosted(host) => { - let timeout = deadline - .saturating_duration_since(Instant::now()) - .min(Duration::from_millis(100)); - let _ = host.terminate_with_timeout(timeout); - true - } - PtyRuntime::Local { .. } => { - unreachable!("local PTY runtime is missing process ownership") - } - #[cfg(unix)] - PtyRuntime::ExitedHosted => true, + #[cfg(unix)] + if let Some(owner) = &pty.hosted_shutdown_owner { + return Some(SurfaceShutdownOwner { + kind: SurfaceShutdownOwnerKind::Hosted(owner.as_ref().clone()), + }); } + None } - Surface::Browser(browser) => browser.terminate_for_server_shutdown(deadline), + Surface::Browser(browser) => browser.shutdown_owner().map(|owner| { + SurfaceShutdownOwner { kind: SurfaceShutdownOwnerKind::Browser(owner) } + }), } } + pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { + self.shutdown_owner().is_none_or(|owner| owner.terminate_until(deadline)) + } + pub(crate) fn disconnect_for_daemon_shutdown(&self) { match self { #[cfg(unix)] @@ -2563,18 +2633,6 @@ impl Surface { } } -fn lock_mutex_until(mutex: &Mutex, deadline: Instant) -> Option> { - loop { - match mutex.try_lock() { - Ok(guard) => return Some(guard), - Err(TryLockError::Poisoned(error)) => return Some(error.into_inner()), - Err(TryLockError::WouldBlock) => {} - } - let remaining = deadline.checked_duration_since(Instant::now())?; - std::thread::sleep(remaining.min(Duration::from_millis(1))); - } -} - #[cfg(test)] struct TestMasterPty { size: Mutex, diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 3c17a7d075fc..92f4754b6720 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1384,6 +1384,57 @@ mod unix { Ok(records) } + /// Terminate one exact host incarnation and retain its discovery record + /// until process-bound liveness proves that host is dead. + pub(crate) fn terminate_and_confirm_terminal_host_record( + record: &TerminalHostRecord, + record_path: &Path, + deadline: Instant, + ) -> bool { + let mut last_terminate_attempt = None; + loop { + match terminal_host_record_liveness(record_path, record) + .unwrap_or(TerminalHostLiveness::Indeterminate) + { + TerminalHostLiveness::Dead => { + return match remove_stale_terminal_host_record(record_path, record) { + Ok(removed) => removed, + Err(_) if !record_path.exists() => true, + Err(_) => false, + }; + } + TerminalHostLiveness::Live | TerminalHostLiveness::Indeterminate => {} + } + + let now = Instant::now(); + if last_terminate_attempt.is_none_or(|attempt: Instant| { + now.duration_since(attempt) >= Duration::from_millis(100) + }) { + let timeout = deadline + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(100)); + if !timeout.is_zero() { + if record.supports_terminate_only { + let _ = terminate_terminal_host_with_timeout(record, record_path, timeout); + } else if let Ok(host) = adopt_terminal_host_with_timeout( + record.clone(), + record_path.to_path_buf(), + timeout, + ) { + let _ = host.terminate_with_timeout(timeout); + host.disconnect(); + } + last_terminate_attempt = Some(Instant::now()); + } + } + let now = Instant::now(); + if now >= deadline { + return false; + } + thread::sleep(deadline.saturating_duration_since(now).min(Duration::from_millis(10))); + } + } + fn connect_record( record: TerminalHostRecord, record_path: PathBuf, @@ -3665,8 +3716,8 @@ pub use unix::{ }; #[cfg(unix)] pub(crate) use unix::{ - adopt_terminal_host_with_timeout, launch_terminal_host_cancellable, - launch_terminal_host_with_identity_cancellable, terminate_terminal_host_with_timeout, + launch_terminal_host_cancellable, launch_terminal_host_with_identity_cancellable, + terminate_and_confirm_terminal_host_record, }; #[cfg(not(unix))] From 17af4a18af0d3b3bc5dcd9aaeede9491177e66c7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:40:51 -0700 Subject: [PATCH 040/521] test(tui): cover recoverable browser shutdown --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 123 +++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 9f303bf558ef..25dbc5c308de 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -2890,6 +2890,129 @@ mod tests { server.join().unwrap(); } + #[test] + fn server_shutdown_reconnects_to_confirm_a_target_after_transport_loss() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + ws.close(None).unwrap(); + drop(ws); + + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + let reconnect = loop { + match listener.accept() { + Ok((stream, _)) => break Some(stream), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + if Instant::now() >= deadline { + break None; + } + thread::sleep(Duration::from_millis(5)); + } + Err(error) => panic!("failed to accept shutdown reconnect: {error}"), + } + }; + let Some(stream) = reconnect else { return false }; + stream.set_nonblocking(false).unwrap(); + let mut ws = accept(stream).unwrap(); + let query = read_ws_json(&mut ws); + assert_eq!(query["method"], "Target.getTargets"); + write_ws_json(&mut ws, json!({"id": query["id"], "result": {"targetInfos": []}})); + true + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + let surface = test_surface(); + let browser = surface.as_browser().unwrap(); + *browser.session.lock().unwrap() = Some(BrowserSession { + runtime: runtime.clone(), + target_id: "target-1".to_string(), + session_id: "session-1".to_string(), + }); + let owner = browser.shutdown_owner().unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + while !runtime.is_closed() && Instant::now() < deadline { + thread::yield_now(); + } + assert!(runtime.is_closed(), "initial CDP transport did not close"); + + let terminated = owner.terminate_until(Instant::now() + Duration::from_millis(500)); + + runtime.shutdown(); + let reconnected = server.join().unwrap(); + assert!(terminated, "closed runtime made target cleanup permanently unretryable"); + assert!(reconnected, "shutdown did not reconnect to query the target"); + } + + #[test] + fn server_shutdown_treats_a_missing_target_as_already_closed() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + + let first = read_ws_json(&mut ws); + if first["method"] == "Target.getTargets" { + write_ws_json( + &mut ws, + json!({ + "id": first["id"], + "result": { + "targetInfos": [{"targetId": "target-1"}] + } + }), + ); + let close = read_ws_json(&mut ws); + assert_eq!(close["method"], "Target.closeTarget"); + assert_eq!(close["params"]["targetId"], "target-1"); + // Simulate Chrome applying the close while its response is + // lost. A post-timeout query must make the retry idempotent. + let query = read_ws_json(&mut ws); + assert_eq!(query["method"], "Target.getTargets"); + write_ws_json(&mut ws, json!({"id": query["id"], "result": {"targetInfos": []}})); + true + } else { + assert_eq!(first["method"], "Target.closeTarget"); + false + } + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + let surface = test_surface(); + let browser = surface.as_browser().unwrap(); + *browser.session.lock().unwrap() = Some(BrowserSession { + runtime: runtime.clone(), + target_id: "target-1".to_string(), + session_id: "session-1".to_string(), + }); + let owner = browser.shutdown_owner().unwrap(); + + let terminated = owner.terminate_until(Instant::now() + Duration::from_millis(400)); + + runtime.shutdown(); + let queried_after_close = server.join().unwrap(); + assert!(terminated, "an already-absent target remained a permanent shutdown failure"); + assert!(queried_after_close, "shutdown did not confirm target absence after close timeout"); + } + #[test] fn runtime_shutdown_waits_for_a_backpressured_detached_target_close() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); From eed326c38b6108f44e1ad932bd7d725b2f7e0c6c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:46:30 -0700 Subject: [PATCH 041/521] fix(tui): recover browser shutdown ownership --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 34 +++- cmux-tui/crates/cmux-tui-core/src/browser.rs | 160 +++++++++++++------ 2 files changed, 145 insertions(+), 49 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index a0b8b1958190..ade9e3914d31 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -275,15 +275,26 @@ enum Outbound { impl CdpClient { pub fn connect(web_socket_url: &str, events: SyncSender) -> anyhow::Result { + Self::connect_with_timeout(web_socket_url, events, Duration::from_secs(5)) + } + + pub fn connect_with_timeout( + web_socket_url: &str, + events: SyncSender, + timeout: Duration, + ) -> anyhow::Result { + if timeout.is_zero() { + anyhow::bail!("CDP connection deadline expired"); + } let endpoint = WsEndpoint::parse(web_socket_url)?; let mut addrs = (endpoint.host.as_str(), endpoint.port).to_socket_addrs()?; let addr = addrs.next().ok_or_else(|| { anyhow::anyhow!("no socket address for {}:{}", endpoint.host, endpoint.port) })?; - let stream = TcpStream::connect_timeout(&addr, Duration::from_secs(5))?; + let stream = TcpStream::connect_timeout(&addr, timeout)?; stream.set_nodelay(true)?; - stream.set_read_timeout(Some(Duration::from_secs(5)))?; - stream.set_write_timeout(Some(Duration::from_secs(5)))?; + stream.set_read_timeout(Some(timeout))?; + stream.set_write_timeout(Some(timeout))?; let request = web_socket_url.into_client_request()?; let (ws, _) = client(request, stream)?; // The reader thread owns the socket and drains queued outbound @@ -291,7 +302,7 @@ impl CdpClient { // read starts can wait for this window, but writers never contend // on the socket itself. ws.get_ref().set_read_timeout(Some(Duration::from_millis(20)))?; - ws.get_ref().set_write_timeout(Some(Duration::from_secs(5)))?; + ws.get_ref().set_write_timeout(Some(timeout))?; let (outbound_tx, outbound_rx) = channel(); let event_queue = Arc::new(EventQueue::new()); let client = CdpClient { @@ -426,6 +437,21 @@ impl CdpClient { .map(|_| ()) } + pub fn target_exists_with_timeout( + &self, + target_id: &str, + timeout: Duration, + ) -> anyhow::Result { + let result = self.call_with_timeout("Target.getTargets", json!({}), None, timeout)?; + let targets = result + .get("targetInfos") + .and_then(Value::as_array) + .ok_or_else(|| anyhow::anyhow!("Target.getTargets response missing targetInfos"))?; + Ok(targets + .iter() + .any(|target| target.get("targetId").and_then(Value::as_str) == Some(target_id))) + } + pub fn close_target_detached(&self, target_id: &str) -> anyhow::Result<()> { let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed); let msg = json!({ diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 25dbc5c308de..92bba282fc0b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -237,6 +237,7 @@ struct BrowserWorkerErrorState { pub struct BrowserRuntime { client: CdpClient, + web_socket_url: String, chrome: Option, source: BrowserSource, stealth_user_agent: Option, @@ -267,6 +268,12 @@ struct QueuedSurfaceEvent { retained_bytes: usize, } +enum ExternalTargetShutdown { + Closed, + StillOpen, + Unreachable, +} + impl SurfaceRoute { fn new() -> Self { Self { state: Mutex::new(SurfaceRouteState::default()), ready: Condvar::new() } @@ -405,6 +412,7 @@ impl BrowserRuntime { }; let runtime = Arc::new(BrowserRuntime { client, + web_socket_url: web_socket_url.to_string(), chrome, source, stealth_user_agent, @@ -541,13 +549,64 @@ impl BrowserRuntime { // exited before shutdown can succeed. return true; } - if self.is_closed() { - return false; + if !self.is_closed() { + match Self::close_external_target_until(&self.client, target_id, deadline) { + ExternalTargetShutdown::Closed => return true, + ExternalTargetShutdown::StillOpen => return false, + ExternalTargetShutdown::Unreachable => {} + } } let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false; }; - self.client.close_target_with_timeout(target_id, remaining).is_ok() + let (event_tx, _event_rx) = sync_channel(CDP_EVENT_QUEUE_CAPACITY); + let Ok(client) = CdpClient::connect_with_timeout(&self.web_socket_url, event_tx, remaining) + else { + return false; + }; + matches!( + Self::close_external_target_until(&client, target_id, deadline), + ExternalTargetShutdown::Closed + ) + } + + fn close_external_target_until( + client: &CdpClient, + target_id: &str, + deadline: Instant, + ) -> ExternalTargetShutdown { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return ExternalTargetShutdown::Unreachable; + }; + let query_timeout = remaining / 3; + if query_timeout.is_zero() { + return ExternalTargetShutdown::Unreachable; + } + match client.target_exists_with_timeout(target_id, query_timeout) { + Ok(false) => return ExternalTargetShutdown::Closed, + Ok(true) => {} + Err(_) => return ExternalTargetShutdown::Unreachable, + } + + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return ExternalTargetShutdown::StillOpen; + }; + let close_timeout = remaining / 3; + if close_timeout.is_zero() { + return ExternalTargetShutdown::StillOpen; + } + if client.close_target_with_timeout(target_id, close_timeout).is_ok() { + return ExternalTargetShutdown::Closed; + } + + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return ExternalTargetShutdown::Unreachable; + }; + match client.target_exists_with_timeout(target_id, remaining) { + Ok(false) => ExternalTargetShutdown::Closed, + Ok(true) => ExternalTargetShutdown::StillOpen, + Err(_) => ExternalTargetShutdown::Unreachable, + } } pub fn shutdown(&self) { @@ -557,7 +616,7 @@ impl BrowserRuntime { pub(crate) fn shutdown_until(&self, deadline: Instant) -> bool { let outbound_flushed = if self.source == BrowserSource::External { if self.is_closed() { - false + true } else { deadline .checked_duration_since(Instant::now()) @@ -1166,7 +1225,7 @@ impl BrowserSurface { self.close_taps(); self.close_command_sender(); } - self.session.lock().unwrap().clone().map(BrowserShutdownOwner) + self.session.lock().unwrap().take().map(BrowserShutdownOwner) } pub fn latest_frame(&self) -> Option { @@ -1224,39 +1283,12 @@ impl BrowserSurface { return; } self.close_taps(); - if let Some(session) = self.session.lock().unwrap().take() { + if let Some(session) = self.session.lock().unwrap().clone() { session.runtime.close_surface_detached(&session.target_id, &session.session_id); } self.close_command_sender(); } - #[cfg(test)] - pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { - if !self.dead.swap(true, Ordering::AcqRel) { - self.close_taps(); - self.close_command_sender(); - } - let Some(session) = self.session.lock().unwrap().clone() else { - return true; - }; - let terminated = session.runtime.close_surface_for_shutdown( - &session.target_id, - &session.session_id, - deadline, - ); - if terminated { - let mut current = self.session.lock().unwrap(); - if current.as_ref().is_some_and(|candidate| { - candidate.target_id == session.target_id - && candidate.session_id == session.session_id - && Arc::ptr_eq(&candidate.runtime, &session.runtime) - }) { - current.take(); - } - } - terminated - } - pub fn resize(&self, cols: u16, rows: u16) -> anyhow::Result { self.resize_reporting_acceptance(cols, rows, Box::new(|_| {})) .map(|reservation_id| reservation_id.is_some()) @@ -2817,6 +2849,15 @@ mod tests { let discover = read_ws_json(&mut ws); assert_eq!(discover["method"], "Target.setDiscoverTargets"); write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + let query = read_ws_json(&mut ws); + assert_eq!(query["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": query["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); let close = read_ws_json(&mut ws); assert_eq!(close["method"], "Target.closeTarget"); assert_eq!(close["params"]["targetId"], "target-1"); @@ -2834,7 +2875,7 @@ mod tests { assert!(!runtime.close_surface_for_shutdown( "target-1", "session-1", - started + Duration::from_millis(50), + started + Duration::from_millis(150), )); assert!(started.elapsed() < Duration::from_millis(500)); close_rx.recv_timeout(Duration::from_secs(1)).unwrap(); @@ -2855,14 +2896,41 @@ mod tests { assert_eq!(discover["method"], "Target.setDiscoverTargets"); write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); - let first = read_ws_json(&mut ws); - assert_eq!(first["method"], "Target.closeTarget"); + let first_query = read_ws_json(&mut ws); + assert_eq!(first_query["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": first_query["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + let first_close = read_ws_json(&mut ws); + assert_eq!(first_close["method"], "Target.closeTarget"); + let first_confirmation = read_ws_json(&mut ws); + assert_eq!(first_confirmation["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": first_confirmation["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); close_tx.send(1).unwrap(); - let second = read_ws_json(&mut ws); - assert_eq!(second["method"], "Target.closeTarget"); - assert_eq!(second["params"]["targetId"], "target-1"); - write_ws_json(&mut ws, json!({"id": second["id"], "result": {"success": true}})); + let second_query = read_ws_json(&mut ws); + assert_eq!(second_query["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": second_query["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + let second_close = read_ws_json(&mut ws); + assert_eq!(second_close["method"], "Target.closeTarget"); + assert_eq!(second_close["params"]["targetId"], "target-1"); + write_ws_json(&mut ws, json!({"id": second_close["id"], "result": {"success": true}})); close_tx.send(2).unwrap(); }); let runtime = super::BrowserRuntime::connect_to_endpoint( @@ -2878,13 +2946,13 @@ mod tests { target_id: "target-1".to_string(), session_id: "session-1".to_string(), }); + let owner = browser.shutdown_owner().unwrap(); - assert!(!browser.terminate_for_server_shutdown(Instant::now() + Duration::from_millis(50))); + assert!(!owner.terminate_until(Instant::now() + Duration::from_millis(200))); assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 1); - assert!(browser.session.lock().unwrap().is_some()); - assert!(browser.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1))); - assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); assert!(browser.session.lock().unwrap().is_none()); + assert!(owner.terminate_until(Instant::now() + Duration::from_secs(1))); + assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); runtime.shutdown(); server.join().unwrap(); @@ -2947,10 +3015,11 @@ mod tests { let terminated = owner.terminate_until(Instant::now() + Duration::from_millis(500)); - runtime.shutdown(); + let runtime_released = runtime.shutdown_until(Instant::now() + Duration::from_millis(100)); let reconnected = server.join().unwrap(); assert!(terminated, "closed runtime made target cleanup permanently unretryable"); assert!(reconnected, "shutdown did not reconnect to query the target"); + assert!(runtime_released, "closed external runtime remained permanently retained"); } #[test] @@ -3102,6 +3171,7 @@ mod tests { .recv_timeout(Duration::from_secs(1)) .expect("closed surface route did not close its CDP target"); assert!(browser.is_dead()); + assert!(browser.shutdown_owner().is_some()); assert!(browser.session.lock().unwrap().is_none()); runtime.shutdown(); From 907afe9581e6e70b5a238c978394c393e641d5e6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:47:05 -0700 Subject: [PATCH 042/521] test(tui): require versioned package replay --- tests/test_tui_package_launchers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index 8ddb71afc17d..d3be686c79f9 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -105,6 +105,9 @@ def test_npm_launcher_preserves_replayable_invocation(tmp_path: Path) -> None: launcher.parent.mkdir(parents=True) shutil.copyfile(NPM_LAUNCHER, launcher) launcher.chmod(0o755) + (launcher.parent.parent / "package.json").write_text( + '{"name":"cmux","version":"1.2.3"}\n' + ) package_root = tmp_path / "node_modules" / package package_root.mkdir(parents=True) (package_root / "package.json").write_text( @@ -119,7 +122,7 @@ def test_npm_launcher_preserves_replayable_invocation(tmp_path: Path) -> None: one_shot_env["npm_command"] = "exec" one_shot = run([node, launcher], cwd=tmp_path, env=one_shot_env) assert one_shot.returncode == 0, one_shot.stderr - assert one_shot.stdout.strip() == "npx cmux" + assert one_shot.stdout.strip() == "npx cmux@1.2.3" installed_env = os.environ.copy() installed_env.pop("npm_command", None) @@ -177,7 +180,7 @@ def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: env=environment, ) assert launched.returncode == 0, launched.stderr - assert launched.stdout.strip() == "uvx cmux" + assert launched.stdout.strip() == "uvx cmux==1.2.3" def main() -> None: From b3fe9dce93314c82e2b5b3bb0ee913d7ce8ebcb0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 20:47:50 -0700 Subject: [PATCH 043/521] fix(tui): pin package replay versions --- cmux-tui/dist/npm/cmux/bin/cmux.js | 4 +++- cmux-tui/dist/scripts/package_pypi.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmux-tui/dist/npm/cmux/bin/cmux.js b/cmux-tui/dist/npm/cmux/bin/cmux.js index 6718930b3fb7..abc47aaf7128 100644 --- a/cmux-tui/dist/npm/cmux/bin/cmux.js +++ b/cmux-tui/dist/npm/cmux/bin/cmux.js @@ -8,6 +8,7 @@ // signals so cmux behaves exactly like the native binary. const { spawnSync } = require("child_process"); +const { version } = require("../package.json"); const PACKAGE_BY_PLATFORM = { "darwin-arm64": "cmux-tui-darwin-arm64", @@ -28,7 +29,8 @@ if (!pkg) { } const binName = process.platform === "win32" ? "cmux-tui.exe" : "cmux-tui"; -const launcherCommand = process.env.npm_command === "exec" ? "npx cmux" : "cmux"; +const launcherCommand = + process.env.npm_command === "exec" ? `npx cmux@${version}` : "cmux"; let binPath; try { diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index d46e2cc5ddf3..817ace6630d1 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -87,13 +87,15 @@ def wheel_bytes(version: str, tag: str, binary: bytes) -> list[tuple[str, bytes, ( f"{PACKAGE_NAME}/_main.py", text_bytes( - """from __future__ import annotations + f"""from __future__ import annotations import os import pathlib import shlex import sys +PACKAGE_VERSION = {version!r} + def _launcher_command(argv0: str) -> str: parts = pathlib.PurePath(argv0).parts @@ -105,7 +107,7 @@ def _launcher_command(argv0: str) -> str: and candidate.removeprefix("archive-v").isdigit() for candidate in parts[index + 1 :] ): - return "uvx cmux" + return "uvx cmux==" + PACKAGE_VERSION return shlex.quote(argv0) if argv0 else "cmux" From 3d6c29f6a504942b755beb81d872ba41f0950c25 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:44:49 -0700 Subject: [PATCH 044/521] test(tui): cover cleanup completion ownership --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 55 +++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/browser.rs | 51 +++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 +++++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 43 +++++++++++++-- tests/test_tui_package_launchers.py | 1 + 5 files changed, 171 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 3e9bf787c86d..57b17b31fa0f 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -9,6 +9,11 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); +#[cfg(test)] +thread_local! { + static FORCE_KILL_TIMEOUT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + #[derive(Debug, Clone)] pub struct ChromeLaunchOptions { pub binary: PathBuf, @@ -135,6 +140,10 @@ impl Drop for Chrome { fn kill_child_until(child: &mut Child, deadline: Instant) -> bool { let _ = child.kill(); + #[cfg(test)] + if FORCE_KILL_TIMEOUT.get() { + return false; + } loop { match child.try_wait() { Ok(Some(_)) => return true, @@ -303,4 +312,50 @@ mod tests { assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); assert!(chrome.child.lock().unwrap().is_none()); } + + #[cfg(unix)] + #[test] + fn drop_transfers_an_unconfirmed_child_to_a_reaper() { + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + } + + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + let chrome = Chrome { + child: Mutex::new(Some(child)), + profile_dir: make_profile_dir().unwrap(), + profile_ephemeral: true, + web_socket_url: "ws://127.0.0.1/unused".to_string(), + }; + + FORCE_KILL_TIMEOUT.set(true); + drop(chrome); + FORCE_KILL_TIMEOUT.set(false); + let deadline = Instant::now() + Duration::from_secs(1); + let reaped = loop { + if unsafe { kill(pid, 0) } != 0 { + break true; + } + if Instant::now() >= deadline { + break false; + } + std::thread::sleep(Duration::from_millis(10)); + }; + if !reaped { + let mut status = 0; + unsafe { + waitpid(pid, &mut status, 0); + } + } + + assert!(reaped, "dropping Chrome discarded an unconfirmed child without reaping it"); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 92bba282fc0b..6b3bce84b227 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -2884,6 +2884,57 @@ mod tests { server.join().unwrap(); } + #[test] + fn server_shutdown_retains_a_target_when_close_reports_false() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_millis(500))).unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + let query = read_ws_json(&mut ws); + assert_eq!(query["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": query["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + let close = read_ws_json(&mut ws); + assert_eq!(close["method"], "Target.closeTarget"); + write_ws_json(&mut ws, json!({"id": close["id"], "result": {"success": false}})); + let confirmation = read_ws_json(&mut ws); + assert_eq!(confirmation["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": confirmation["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + + let terminated = runtime.close_surface_for_shutdown( + "target-1", + "session-1", + Instant::now() + Duration::from_secs(1), + ); + + runtime.shutdown(); + server.join().unwrap(); + assert!(!terminated, "Target.closeTarget success=false discarded the target owner"); + } + #[test] fn server_shutdown_retries_an_unconfirmed_external_target() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 102907d7769f..ad227e594032 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10092,6 +10092,32 @@ mod tests { assert!(mux.shutdown_owners.lock().unwrap().is_empty()); } + #[test] + fn server_shutdown_cannot_pass_an_in_flight_surface_retirement() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let surface_id = surface.id; + let owned = mux.surface(surface.id).unwrap(); + owned.set_server_shutdown_delay_for_test(Duration::from_millis(150)); + let close = std::thread::spawn({ + let mux = mux.clone(); + move || mux.close_surface(surface_id).unwrap() + }); + let removal_deadline = Instant::now() + Duration::from_secs(1); + while mux.surface(surface_id).is_some() && Instant::now() < removal_deadline { + std::thread::yield_now(); + } + assert!(mux.surface(surface_id).is_none(), "ordinary close did not remove its surface"); + + let shutdown = mux.close_all_surfaces_for_shutdown(); + + assert!(close.join().unwrap()); + owned.set_server_shutdown_failure_for_test(false); + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + let error = shutdown.expect_err("server shutdown passed an untracked retirement owner"); + assert!(error.to_string().contains("could not terminate 1 surface process")); + } + #[test] fn ordinary_surface_close_does_not_retain_terminal_render_state() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 9ecc3d013c50..6baf61ddfe5f 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -699,19 +699,52 @@ mod tests { #[cfg(unix)] #[test] - fn shutdown_read_error_accepts_a_socket_that_stopped_listening() { + fn modern_shutdown_requires_a_success_response_before_disconnect() { let path = PathBuf::from("/tmp").join(format!( "cmux-tui-shutdown-disconnect-{}-{}.sock", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let identify: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(identify["cmd"], "identify"); + write_json_line( + &mut stream, + &json!({ + "id": identify["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "version": "test", + "protocol": PROTOCOL_VERSION, + "pid": std::process::id(), + "capabilities": [SERVER_SHUTDOWN_CAPABILITY], + }, + }), + ) + .unwrap(); + line.clear(); + reader.read_line(&mut line).unwrap(); + let shutdown: Value = serde_json::from_str(&line).unwrap(); + assert_eq!(shutdown["cmd"], "shutdown"); + }); - assert!(shutdown_read_error(&path).is_ok()); + let result = ServerLifecycle::connect(path.clone()).unwrap().stop(); - let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - assert!(shutdown_read_error(&path).is_err()); - drop(listener); + server.join().unwrap(); std::fs::remove_file(path).unwrap(); + let error = result.expect_err("disconnect without a success response was accepted"); + assert!( + error.to_string().contains(crate::localization::catalog().server.response_closed) + || error + .to_string() + .contains(crate::localization::catalog().server.transport_failed) + ); } #[cfg(unix)] diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index d3be686c79f9..c2fa2cc0fcf1 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -166,6 +166,7 @@ def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: environment = os.environ.copy() environment["PYTHONPATH"] = str(installed) + environment["CMUX_TUI_LAUNCHER_COMMAND"] = "uvx cmux==0.9.0" launched = run( [ sys.executable, From e4dc60e0ce1099a9275dcb1cae7c5c03714f6875 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:52:14 -0700 Subject: [PATCH 045/521] fix(tui): retain cleanup ownership until confirmed --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 51 ++++- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 17 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 182 +++++++++++------- .../cmux-tui-core/tests/browser_runtime.rs | 22 +++ .../crates/cmux-tui/src/server_lifecycle.rs | 6 +- cmux-tui/dist/scripts/package_pypi.py | 2 +- 6 files changed, 193 insertions(+), 87 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 57b17b31fa0f..110cb07a5dc6 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -90,9 +90,12 @@ impl Chrome { let web_socket_url = match rx.recv_timeout(Duration::from_secs(10)) { Ok(url) => url, Err(_) => { - let _ = kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)); - if profile_ephemeral { - let _ = std::fs::remove_dir_all(&profile_dir); + if kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)) { + if profile_ephemeral { + let _ = std::fs::remove_dir_all(&profile_dir); + } + } else { + reap_child_detached(child, profile_ephemeral.then(|| profile_dir.clone())); } anyhow::bail!( "Chrome did not publish a DevTools endpoint within 10s (binary: {})", @@ -131,13 +134,51 @@ impl Chrome { impl Drop for Chrome { fn drop(&mut self) { - self.kill(); - if self.profile_ephemeral { + if self.kill_until(Instant::now() + Duration::from_secs(1)) { + if self.profile_ephemeral { + let _ = std::fs::remove_dir_all(&self.profile_dir); + } + return; + } + let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); + if let Some(child) = child { + reap_child_detached(child, self.profile_ephemeral.then(|| self.profile_dir.clone())); + } else if self.profile_ephemeral { let _ = std::fs::remove_dir_all(&self.profile_dir); } } } +struct ReapRequest { + child: Child, + profile_dir: Option, +} + +fn reap_child_detached(child: Child, profile_dir: Option) { + let request = std::sync::Arc::new(Mutex::new(Some(ReapRequest { child, profile_dir }))); + let worker_request = request.clone(); + let spawned = + std::thread::Builder::new().name("cmux-tui-cdp-chrome-reaper".into()).spawn(move || { + if let Some(request) = worker_request.lock().unwrap().take() { + reap_child(request); + } + }); + if spawned.is_err() + && let Some(request) = request.lock().unwrap().take() + { + reap_child(request); + } +} + +fn reap_child(mut request: ReapRequest) { + let _ = request.child.kill(); + if request.child.wait().is_ok() + && let Some(profile_dir) = request.profile_dir + { + let _ = std::fs::remove_dir_all(profile_dir); + } +} + fn kill_child_until(child: &mut Child, deadline: Instant) -> bool { let _ = child.kill(); #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index ade9e3914d31..af81db4fd03e 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -420,7 +420,8 @@ impl CdpClient { } pub fn close_target(&self, target_id: &str) -> anyhow::Result<()> { - self.call("Target.closeTarget", json!({ "targetId": target_id }), None).map(|_| ()) + let result = self.call("Target.closeTarget", json!({ "targetId": target_id }), None)?; + require_target_close_success(&result) } pub fn close_target_with_timeout( @@ -428,13 +429,13 @@ impl CdpClient { target_id: &str, timeout: Duration, ) -> anyhow::Result<()> { - self.call_with_timeout( + let result = self.call_with_timeout( "Target.closeTarget", json!({ "targetId": target_id }), None, timeout, - ) - .map(|_| ()) + )?; + require_target_close_success(&result) } pub fn target_exists_with_timeout( @@ -671,6 +672,14 @@ impl CdpClient { } } +fn require_target_close_success(result: &Value) -> anyhow::Result<()> { + match result.get("success").and_then(Value::as_bool) { + Some(true) => Ok(()), + Some(false) => anyhow::bail!("Target.closeTarget reported success=false"), + None => anyhow::bail!("Target.closeTarget response missing success"), + } +} + pub fn resolve_browser_ws_url(input: &str) -> anyhow::Result { let trimmed = input.trim(); if trimmed.starts_with("ws://") { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ad227e594032..99ba532bd471 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -61,6 +61,50 @@ fn shutdown_owner_key(surface: SurfaceId, owner: &SurfaceShutdownOwner) -> Shutd .unwrap_or(ShutdownOwnerKey::Surface(surface)) } +#[derive(Default)] +struct ShutdownOwnerLedger { + owners: Mutex>>, +} + +impl ShutdownOwnerLedger { + fn stage(&self, key: ShutdownOwnerKey, owner: SurfaceShutdownOwner) { + self.owners.lock().unwrap().entry(key).or_insert_with(|| Arc::new(owner)); + } + + fn stage_surface(&self, surface: &Arc) { + if let Some(owner) = surface.shutdown_owner() { + self.stage(shutdown_owner_key(surface.id, &owner), owner); + } + } + + fn snapshot(&self) -> Vec<(ShutdownOwnerKey, Arc)> { + self.owners + .lock() + .unwrap() + .iter() + .map(|(key, owner)| (key.clone(), owner.clone())) + .collect() + } + + fn remove_confirmed(&self, confirmed: &[(ShutdownOwnerKey, Arc)]) { + let mut owners = self.owners.lock().unwrap(); + for (key, confirmed) in confirmed { + if owners.get(key).is_some_and(|owner| Arc::ptr_eq(owner, confirmed)) { + owners.remove(key); + } + } + } + + fn len(&self) -> usize { + self.owners.lock().unwrap().len() + } + + #[cfg(test)] + fn is_empty(&self) -> bool { + self.owners.lock().unwrap().is_empty() + } +} + /// An opaque per-mux credential provisioned by the external machine /// provider. Debug output is deliberately redacted. #[derive(PartialEq, Eq)] @@ -868,7 +912,7 @@ pub struct Mux { surface_creations: SurfaceCreationGate, async_surface_creations: AsyncSurfaceCreationGate, shutdown_coordinator: Mutex<()>, - shutdown_owners: Mutex>, + shutdown_owners: ShutdownOwnerLedger, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -1101,7 +1145,7 @@ impl Mux { surface_creations: SurfaceCreationGate::default(), async_surface_creations: AsyncSurfaceCreationGate::default(), shutdown_coordinator: Mutex::new(()), - shutdown_owners: Mutex::new(HashMap::new()), + shutdown_owners: ShutdownOwnerLedger::default(), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -3568,12 +3612,12 @@ impl Mux { if split_index_dirty { Self::rebuild_split_screen_index(&mut state); } - removed.or_else(|| state.surfaces.remove(&surface.id)) + removed.or_else(|| take_surface_for_retirement(self, &mut state, surface.id)) }; drop(registry); - if removed.is_some() { + if let Some(removed) = removed { self.purge_surface_side_tables(surface.id); - surface.kill(); + self.retire_surface_runtime(removed); } Ok(()) } @@ -3747,30 +3791,30 @@ impl Mux { } fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { - let mut owners = self.shutdown_owners.lock().unwrap().drain().collect::>(); for surface in surfaces { - if let Some(owner) = surface.shutdown_owner() { - owners.insert(shutdown_owner_key(surface.id, &owner), owner); - } + self.shutdown_owners.stage_surface(&surface); } - let owners = owners.into_iter().collect::>(); + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; + self.terminate_staged_shutdown_owners_until(deadline); + } + + fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) { + let owners = self.shutdown_owners.snapshot(); if owners.is_empty() { return; } - let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; let failed = Mutex::new(HashSet::new()); let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { if !owner.terminate_until(deadline) { failed.lock().unwrap().insert(key.clone()); } }); - let failed = failed.into_inner().unwrap(); - let mut retained = self.shutdown_owners.lock().unwrap(); - for (index, (key, owner)) in owners.into_iter().enumerate() { - if index >= started || failed.contains(&key) { - retained.insert(key, owner); - } - } + let mut failed = failed.into_inner().unwrap(); + failed.extend(owners[started..].iter().map(|(key, _)| key.clone())); + let confirmed = + owners.into_iter().filter(|(key, _)| !failed.contains(key)).collect::>(); + self.shutdown_owners.remove_confirmed(&confirmed); } pub fn list_agents( @@ -3790,22 +3834,14 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; let _ = self.surface_creations.stop_and_wait_until(deadline); let _ = self.async_surface_creations.stop_and_wait_until(deadline); let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); - for surface in surfaces { + for surface in &surfaces { surface.disconnect_for_daemon_shutdown(); } - let retired = self - .shutdown_owners - .lock() - .unwrap() - .drain() - .map(|(_, owner)| owner) - .collect::>(); - bounded_shutdown_fanout(&retired, deadline, |owner, deadline| { - let _ = owner.terminate_until(deadline); - }); + self.terminate_staged_shutdown_owners_until(deadline); if let Some(runtime) = self.browser_runtime.lock().unwrap().take() { let _ = runtime.shutdown_until(deadline); } @@ -3880,7 +3916,7 @@ impl Mux { } }; - let (surfaces, tree_changed) = { + let (surfaces, retained_count, tree_changed) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); let mut registry = self.workspace_registry.lock().unwrap(); let terminals = registry @@ -3891,13 +3927,17 @@ impl Mux { .collect::>(); let batch = registry.close_terminals_atomically(&mutation, &terminals)?; - let (surfaces, tree_changed) = { + let (surfaces, retained_count, tree_changed) = { let mut state = self.state.lock().unwrap(); + let retained_count = self.shutdown_owners.len(); let tree_changed = !state.surfaces.is_empty() || !state.panes.is_empty() || state.workspaces.iter().any(|workspace| !workspace.screens.is_empty()); let surfaces = state.surfaces.drain().map(|(_, surface)| surface).collect::>(); + for surface in &surfaces { + self.shutdown_owners.stage_surface(surface); + } for workspace in &mut state.workspaces { workspace.screens.clear(); workspace.active_screen = 0; @@ -3907,13 +3947,13 @@ impl Mux { } state.panes.clear(); state.split_screens.clear(); - (surfaces, tree_changed) + (surfaces, retained_count, tree_changed) }; if batch.closed != 0 { self.emit_terminal_registry_changed(®istry, batch.revision); } - (surfaces, tree_changed) + (surfaces, retained_count, tree_changed) }; self.pending_workspace_surfaces.lock().unwrap().clear(); self.agent_records.lock().unwrap().clear(); @@ -3930,13 +3970,8 @@ impl Mux { self.emit(MuxEvent::TreeChanged); } - let mut owners = self.shutdown_owners.lock().unwrap().drain().collect::>(); - let closed_count = surfaces.len() + owners.len(); - for surface in surfaces { - if let Some(owner) = surface.shutdown_owner() { - owners.insert(shutdown_owner_key(surface.id, &owner), owner); - } - } + let closed_count = surfaces.len() + retained_count; + drop(surfaces); #[cfg(unix)] for (record_path, record) in terminal_host_records { let owner = SurfaceShutdownOwner::hosted(record, record_path); @@ -3947,42 +3982,27 @@ impl Mux { incarnation: incarnation.to_string(), }) .expect("hosted owner has a terminal identity"); - owners.entry(key).or_insert(owner); + self.shutdown_owners.stage(key, owner); } - let owners = owners.into_iter().collect::>(); - let failed_keys = Mutex::new(HashSet::new()); - let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { - if !owner.terminate_until(deadline) { - failed_keys.lock().unwrap().insert(key.clone()); - } - }); - let mut failed_keys = failed_keys.into_inner().unwrap(); - failed_keys.extend(owners[started..].iter().map(|(key, _)| key.clone())); - let browser_surface_failed = - owners.iter().any(|(key, owner)| failed_keys.contains(key) && owner.is_browser()); - let mut failed_surface_ids = failed_keys + self.terminate_staged_shutdown_owners_until(deadline); + let retained = self.shutdown_owners.snapshot(); + let browser_surface_failed = retained.iter().any(|(_, owner)| owner.is_browser()); + let mut failed_surface_ids = retained .iter() - .filter_map(|key| match key { + .filter_map(|(key, _)| match key { ShutdownOwnerKey::Surface(surface) => Some(*surface), ShutdownOwnerKey::Hosted { .. } => None, }) .collect::>(); failed_surface_ids.sort_unstable(); - let mut failed_hosts = failed_keys + let mut failed_hosts = retained .iter() - .filter_map(|key| match key { + .filter_map(|(key, _)| match key { ShutdownOwnerKey::Hosted { terminal_id, .. } => Some(terminal_id.clone()), ShutdownOwnerKey::Surface(_) => None, }) .collect::>(); failed_hosts.sort(); - let mut retained = self.shutdown_owners.lock().unwrap(); - for (key, owner) in owners { - if failed_keys.contains(&key) { - retained.insert(key, owner); - } - } - drop(retained); let browser_runtime = (!browser_surface_failed) .then(|| self.browser_runtime.lock().unwrap().take()) @@ -4094,9 +4114,11 @@ impl Mux { runtime.retry_at = None; runtime.surface.take() }; - if let Some(surface) = - old_surface.and_then(|id| self.state.lock().unwrap().surfaces.remove(&id)) - { + let retired = old_surface.and_then(|id| { + let mut state = self.state.lock().unwrap(); + take_surface_for_retirement(self, &mut state, id) + }); + if let Some(surface) = retired { self.retire_surface_runtime(surface.clone()); self.emit(MuxEvent::SurfaceExited(surface.id)); } @@ -6165,7 +6187,9 @@ impl Mux { for pane_id in pane_ids { if let Some(pane) = state.remove_pane(pane_id) { for surface in pane.tabs { - if let Some(surface) = state.surfaces.remove(&surface) { + if let Some(surface) = + take_surface_for_retirement(self, &mut state, surface) + { removed.push(surface); } } @@ -8140,7 +8164,7 @@ fn close_workspace_delta( /// split ownership or positional indexes changed. Runs under the state lock. fn remove_surface(mux: &Mux, state: &mut State, target: SurfaceId) -> (Option>, bool) { let previous_active = state.active_pane(); - let removed = state.surfaces.remove(&target); + let removed = take_surface_for_retirement(mux, state, target); let Some(pane_id) = state.pane_of(target) else { return (removed, false); }; @@ -8219,6 +8243,20 @@ fn remove_surface(mux: &Mux, state: &mut State, target: SurfaceId) -> (Option Option> { + let removed = state.surfaces.remove(&target); + if let Some(surface) = &removed { + mux.shutdown_owners.stage_surface(surface); + } + removed +} + fn collapse_empty_pane(mux: &Mux, state: &mut State, pane_id: PaneId) { state.remove_pane(pane_id); let Some((wi, si)) = state.screen_of(pane_id) else { @@ -10069,11 +10107,11 @@ mod tests { assert!(error.to_string().contains("could not terminate 1 surface process")); assert!(mux.surface(surface.id).is_none()); - assert_eq!(mux.shutdown_owners.lock().unwrap().len(), 1); + assert_eq!(mux.shutdown_owners.len(), 1); owned.set_server_shutdown_failure_for_test(false); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); - assert!(mux.shutdown_owners.lock().unwrap().is_empty()); + assert!(mux.shutdown_owners.is_empty()); } #[test] @@ -10085,11 +10123,11 @@ mod tests { assert!(mux.close_surface(surface.id).unwrap()); assert!(mux.surface(surface.id).is_none()); - assert_eq!(mux.shutdown_owners.lock().unwrap().len(), 1); + assert_eq!(mux.shutdown_owners.len(), 1); owned.set_server_shutdown_failure_for_test(false); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); - assert!(mux.shutdown_owners.lock().unwrap().is_empty()); + assert!(mux.shutdown_owners.is_empty()); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 3b3f83c594f3..42456fde472d 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -27,6 +27,16 @@ fn write_json(ws: &mut tungstenite::WebSocket, value: Value ws.send(Message::Text(value.to_string().into())).unwrap(); } +fn write_target_list( + ws: &mut tungstenite::WebSocket, + id: Value, + targets: &[&str], +) { + let target_infos = + targets.iter().map(|target_id| json!({"targetId": target_id})).collect::>(); + write_json(ws, json!({"id": id, "result": {"targetInfos": target_infos}})); +} + fn rpc(path: &std::path::Path, mut cmd: Value) -> Value { let mut stream = UnixStream::connect(path).unwrap(); if cmd.get("id").is_none() { @@ -275,6 +285,9 @@ fn socket_browser_attach_streams_frames_input_and_cell_pixels() { } write_json(&mut ws, json!({"id": id, "result": {}})); } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1", "target-popup"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); closed += 1; @@ -614,6 +627,9 @@ fn wedged_browser_navigate_does_not_block_same_socket_connection() { // sit in CdpClient::call until timeout, but this mux // socket connection must remain usable. } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); break; @@ -775,6 +791,9 @@ fn queued_back_and_forward_do_not_collapse_while_worker_is_blocked() { "Page.navigateToHistoryEntry" => { write_json(&mut ws, json!({"id": id, "result": {}})); } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); break; @@ -885,6 +904,9 @@ fn control_command_reports_backpressure_when_worker_queue_is_full() { "Page.reload" | "Page.navigateToHistoryEntry" => { write_json(&mut ws, json!({"id": id, "result": {}})); } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); break; diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 6baf61ddfe5f..57d5d122c5be 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -220,11 +220,7 @@ impl ServerLifecycle { ) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = match read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID) { - Ok(response) => response, - Err(_error) if connection_is_gone(&self.path) => return Ok(()), - Err(error) => return Err(error), - }; + let response = read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID)?; if response.get("ok").and_then(Value::as_bool) == Some(true) { return wait_for_disconnect(&mut self.reader, &self.path); } diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index 817ace6630d1..806b5f6b2169 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -113,7 +113,7 @@ def _launcher_command(argv0: str) -> str: def main() -> None: binary = pathlib.Path(__file__).resolve().parent / "bin" / "cmux-tui" - os.environ.setdefault("CMUX_TUI_LAUNCHER_COMMAND", _launcher_command(sys.argv[0])) + os.environ["CMUX_TUI_LAUNCHER_COMMAND"] = _launcher_command(sys.argv[0]) os.execv(str(binary), ["cmux", *sys.argv[1:]]) """ ), From 2def8b97dcd66204c1960cf9ccd23e2000ae074b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:33:54 -0700 Subject: [PATCH 046/521] test(tui): cover bounded shutdown completion --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 ++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 87 +++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 27 ++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 20 +++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 30 ++++++- 5 files changed, 182 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e35f82f0bdc3..6a8d05a9449b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10149,6 +10149,25 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn ordinary_surface_close_does_not_retry_unrelated_retained_owners() { + let mux = test_mux(); + let retained = mux.new_workspace(None, Some((80, 24))).unwrap(); + let retained_owner = mux.surface(retained.id).unwrap(); + let attempts = retained_owner.count_server_shutdown_attempts_for_test(); + + assert!(mux.close_surface(retained.id).unwrap()); + assert_eq!(attempts.load(Ordering::Acquire), 1); + + let current = mux.new_workspace(None, Some((80, 24))).unwrap(); + assert!(mux.close_surface(current.id).unwrap()); + assert_eq!( + attempts.load(Ordering::Acquire), + 1, + "an unrelated close retried retained shutdown ownership" + ); + } + #[test] fn server_shutdown_cannot_pass_an_in_flight_surface_retirement() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 334875adf399..6f642850b98c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -733,6 +733,10 @@ trait MessageSink: Send + Sync { fn send_initial(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn send_stream(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn send_control(&self, value: &Value) -> std::io::Result<()>; + #[cfg(test)] + fn send_control_confirmed(&self, value: &Value, _timeout: Duration) -> std::io::Result<()> { + self.send_control(value) + } fn send_terminal(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn set_write_timeout(&self, _timeout: Option) -> std::io::Result<()> { Ok(()) @@ -5123,6 +5127,89 @@ mod tests { assert_eq!(response, json!({"id": 7, "ok": true, "data": {}})); } + struct ConfirmedControlSink { + entered: std::sync::mpsc::SyncSender<()>, + release: Mutex>, + open: AtomicBool, + } + + impl MessageSink for ConfirmedControlSink { + fn send_initial(&self, _value: &Value, _stream: &OutboundStream) -> std::io::Result<()> { + Ok(()) + } + + fn send_stream(&self, _value: &Value, _stream: &OutboundStream) -> std::io::Result<()> { + Ok(()) + } + + fn send_control(&self, _value: &Value) -> std::io::Result<()> { + Ok(()) + } + + fn send_control_confirmed( + &self, + _value: &Value, + _timeout: Duration, + ) -> std::io::Result<()> { + self.entered + .send(()) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "test closed"))?; + self.release + .lock() + .unwrap() + .recv() + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "test closed")) + } + + fn send_terminal(&self, _value: &Value, _stream: &OutboundStream) -> std::io::Result<()> { + Ok(()) + } + + fn is_open(&self) -> bool { + self.open.load(Ordering::Acquire) + } + + fn close(&self) { + self.open.store(false, Ordering::Release); + } + } + + #[test] + fn local_shutdown_waits_for_the_acknowledgement_write() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let writer = MessageWriter::new(ConfirmedControlSink { + entered: entered_tx, + release: Mutex::new(release_rx), + open: AtomicBool::new(true), + }); + let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let request = std::thread::spawn({ + let mux = mux.clone(); + let writer = writer.clone(); + move || { + done_tx + .send(handle_message(&mux, client, r#"{"id":7,"cmd":"shutdown"}"#, &writer)) + .unwrap(); + } + }); + + entered_rx + .recv_timeout(Duration::from_secs(1)) + .expect("shutdown did not use the confirmed control path"); + assert!(!mux.shutdown_requested()); + assert!(mux.surface(surface.id).is_none()); + assert!(done_rx.recv_timeout(Duration::from_millis(50)).is_err()); + + release_tx.send(()).unwrap(); + assert!(done_rx.recv_timeout(Duration::from_secs(1)).unwrap()); + request.join().unwrap(); + assert!(mux.shutdown_requested()); + } + #[test] fn local_shutdown_exits_after_cleanup_when_the_peer_drops_before_ack() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 1cf1c6843f1a..bd575d0a725f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1902,6 +1902,17 @@ impl Surface { *killer.lock().unwrap() = Box::new(TestSlowFailingChildKiller(delay)); } + #[cfg(test)] + pub(crate) fn count_server_shutdown_attempts_for_test(&self) -> Arc { + let attempts = Arc::new(AtomicUsize::new(0)); + let Self::Pty(pty) = self else { return attempts }; + let Some(process) = &pty.local_process else { return attempts }; + let process = process.lock().unwrap(); + let LocalProcess::Untracked(killer) = &*process else { return attempts }; + *killer.lock().unwrap() = Box::new(TestCountingFailingChildKiller(attempts.clone())); + attempts + } + fn as_pty(&self) -> Option<&PtySurface> { match self { Surface::Pty(surface) => Some(surface), @@ -2719,6 +2730,22 @@ impl ChildKiller for TestSlowFailingChildKiller { } } +#[cfg(test)] +#[derive(Debug)] +struct TestCountingFailingChildKiller(Arc); + +#[cfg(test)] +impl ChildKiller for TestCountingFailingChildKiller { + fn kill(&mut self) -> std::io::Result<()> { + self.0.fetch_add(1, Ordering::AcqRel); + Err(std::io::Error::other("forced counted shutdown failure")) + } + + fn clone_killer(&self) -> Box { + Box::new(Self(self.0.clone())) + } +} + impl PtySurface { /// Snapshot sparse colors and the host-resolved cursor visual without /// touching the shared renderer or consuming its damage. diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 57d5d122c5be..e822fa44cf72 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -779,4 +779,24 @@ mod tests { assert_eq!(legacy_surface_ids(&data), [11, 12, 13]); } + + #[cfg(unix)] + #[test] + fn response_deadline_overrides_a_longer_stream_timeout() { + let (client, _server) = std::os::unix::net::UnixStream::pair().unwrap(); + client.set_read_timeout(Some(Duration::from_millis(500))).unwrap(); + let mut reader = BufReader::new(Box::new(client) as Box); + let started = Instant::now(); + + let error = + read_matching_response_with_timeout(&mut reader, 7, false, Duration::from_millis(50)) + .unwrap_err(); + + assert!( + started.elapsed() < Duration::from_millis(250), + "response deadline was hidden by the stream timeout: {:?}", + started.elapsed() + ); + assert_eq!(error.to_string(), crate::localization::catalog().server.transport_failed); + } } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 798a72f5291d..3505afca24d9 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -301,6 +301,7 @@ fn legacy_server_process_helper() { | "applied-close-error" | "applied-close-disconnect" | "persistent-close-error" + | "browser-surface" ) { let descendant = Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); @@ -469,6 +470,7 @@ fn legacy_server_process_helper() { | "persistent-close-error" | "zombie-child" | "reparent-on-close" + | "browser-surface" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { ("zombie-child", _) => &[], @@ -478,7 +480,13 @@ fn legacy_server_process_helper() { }; let tabs = surfaces .iter() - .map(|surface| serde_json::json!({"surface": surface})) + .map(|surface| { + if scenario == "browser-surface" { + serde_json::json!({"surface": surface, "kind": "browser"}) + } else { + serde_json::json!({"surface": surface, "kind": "pty"}) + } + }) .collect::>(); writeln!( stream, @@ -1482,6 +1490,26 @@ fn server_stop_does_not_signal_an_older_server_when_pane_cleanup_fails() { assert!(process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_refuses_an_older_browser_without_target_confirmation() { + let mut server = LegacyServerProcess::start("legacy-server-browser", "browser-surface", None); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(server.child.try_wait().unwrap().is_none()); +} + #[cfg(unix)] #[test] fn server_stop_refuses_a_legacy_server_that_spoofs_another_process_id() { From 30512152d360be997b1a41f46a3e0451bd0c27b6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:45:25 -0700 Subject: [PATCH 047/521] fix(tui): scope and bound shutdown completion --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 48 ++- .../cmux-tui-core/src/process_session.rs | 7 + cmux-tui/crates/cmux-tui-core/src/server.rs | 162 +++++++--- .../crates/cmux-tui/src/server_lifecycle.rs | 278 +++++++++++++++--- .../src/server_lifecycle/legacy_process.rs | 114 ++++++- cmux-tui/crates/cmux-tui/tests/cli.rs | 48 ++- 6 files changed, 548 insertions(+), 109 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 6a8d05a9449b..01cf55fbd963 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -67,14 +67,27 @@ struct ShutdownOwnerLedger { } impl ShutdownOwnerLedger { - fn stage(&self, key: ShutdownOwnerKey, owner: SurfaceShutdownOwner) { - self.owners.lock().unwrap().entry(key).or_insert_with(|| Arc::new(owner)); + fn stage( + &self, + key: ShutdownOwnerKey, + owner: SurfaceShutdownOwner, + ) -> (ShutdownOwnerKey, Arc) { + let staged = self + .owners + .lock() + .unwrap() + .entry(key.clone()) + .or_insert_with(|| Arc::new(owner)) + .clone(); + (key, staged) } - fn stage_surface(&self, surface: &Arc) { - if let Some(owner) = surface.shutdown_owner() { - self.stage(shutdown_owner_key(surface.id, &owner), owner); - } + fn stage_surface( + &self, + surface: &Arc, + ) -> Option<(ShutdownOwnerKey, Arc)> { + let owner = surface.shutdown_owner()?; + Some(self.stage(shutdown_owner_key(surface.id, &owner), owner)) } fn snapshot(&self) -> Vec<(ShutdownOwnerKey, Arc)> { @@ -3800,16 +3813,25 @@ impl Mux { } fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { - for surface in surfaces { - self.shutdown_owners.stage_surface(&surface); - } + let owners = surfaces + .into_iter() + .filter_map(|surface| self.shutdown_owners.stage_surface(&surface)) + .collect::>(); let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; - self.terminate_staged_shutdown_owners_until(deadline); + self.terminate_shutdown_owners_until(owners, deadline); } fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) { let owners = self.shutdown_owners.snapshot(); + self.terminate_shutdown_owners_until(owners, deadline); + } + + fn terminate_shutdown_owners_until( + &self, + owners: Vec<(ShutdownOwnerKey, Arc)>, + deadline: Instant, + ) { if owners.is_empty() { return; } @@ -3945,7 +3967,7 @@ impl Mux { let surfaces = state.surfaces.drain().map(|(_, surface)| surface).collect::>(); for surface in &surfaces { - self.shutdown_owners.stage_surface(surface); + let _ = self.shutdown_owners.stage_surface(surface); } for workspace in &mut state.workspaces { workspace.screens.clear(); @@ -3991,7 +4013,7 @@ impl Mux { incarnation: incarnation.to_string(), }) .expect("hosted owner has a terminal identity"); - self.shutdown_owners.stage(key, owner); + let _ = self.shutdown_owners.stage(key, owner); } self.terminate_staged_shutdown_owners_until(deadline); let retained = self.shutdown_owners.snapshot(); @@ -8265,7 +8287,7 @@ fn take_surface_for_retirement( ) -> Option> { let removed = state.surfaces.remove(&target); if let Some(surface) = &removed { - mux.shutdown_owners.stage_surface(surface); + let _ = mux.shutdown_owners.stage_surface(surface); } removed } diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 907fc3281cf8..74473c544f09 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -103,6 +103,13 @@ pub fn session_is_empty(session: libc::pid_t) -> io::Result { members(session).map(|members| members.is_empty()) } +/// Snapshot process IDs in a non-caller session. Callers must capture exact +/// process birth identities before using the result for later signaling. +pub fn session_member_pids(session: libc::pid_t) -> io::Result> { + validate_session_target(session)?; + members(session) +} + fn signal_members( members: &[libc::pid_t], session: libc::pid_t, diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 6f642850b98c..dffb1450113e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -689,6 +689,7 @@ struct Response { const STREAM_DISCONNECT_POLL: Duration = Duration::from_millis(100); const STREAM_WRITE_TIMEOUT: Duration = Duration::from_secs(2); +const SHUTDOWN_ACK_WRITE_TIMEOUT: Duration = SERVER_SHUTDOWN_TIMEOUT; #[cfg(not(test))] const WEBSOCKET_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(test)] @@ -733,10 +734,7 @@ trait MessageSink: Send + Sync { fn send_initial(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn send_stream(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn send_control(&self, value: &Value) -> std::io::Result<()>; - #[cfg(test)] - fn send_control_confirmed(&self, value: &Value, _timeout: Duration) -> std::io::Result<()> { - self.send_control(value) - } + fn send_control_confirmed(&self, value: &Value, timeout: Duration) -> std::io::Result<()>; fn send_terminal(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()>; fn set_write_timeout(&self, _timeout: Option) -> std::io::Result<()> { Ok(()) @@ -813,6 +811,17 @@ impl MessageWriter { result } + fn send_control_confirmed(&self, value: &Value, timeout: Duration) -> std::io::Result<()> { + if !self.is_open() { + return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, "connection closed")); + } + let result = self.sink.send_control_confirmed(value, timeout); + if result.is_err() { + self.close(); + } + result + } + fn is_open(&self) -> bool { self.open.load(Ordering::Acquire) && self.sink.is_open() } @@ -837,7 +846,7 @@ struct BoundedOutbound { #[derive(Default)] struct BoundedOutboundState { initial: VecDeque, - control: VecDeque, + control: VecDeque, regular: VecDeque, control_bytes: usize, regular_bytes: usize, @@ -849,6 +858,44 @@ struct RegularOutbound { stream: OutboundStream, } +struct ControlOutbound { + text: String, + completion: Option>, +} + +struct OutboundMessage { + text: String, + completion: Option>, +} + +impl OutboundMessage { + fn take_text(&mut self) -> String { + std::mem::take(&mut self.text) + } + + fn finish(mut self, written: bool) { + if let Some(completion) = self.completion.take() { + let _ = completion.send(written); + } + } + + #[cfg(test)] + fn into_test_text(mut self) -> String { + if let Some(completion) = self.completion.take() { + let _ = completion.send(true); + } + std::mem::take(&mut self.text) + } +} + +impl Drop for OutboundMessage { + fn drop(&mut self) { + if let Some(completion) = self.completion.take() { + let _ = completion.send(false); + } + } +} + struct ConnectionPermit(Arc); impl Drop for ConnectionPermit { @@ -934,11 +981,29 @@ impl BoundedOutbound { fn push_control(&self, text: String) -> std::io::Result<()> { let mut state = self.state.lock().unwrap(); - Self::push_control_locked(&mut state, text)?; + Self::push_control_locked(&mut state, text, None)?; self.changed.notify_one(); Ok(()) } + fn push_control_confirmed(&self, text: String, timeout: Duration) -> std::io::Result<()> { + let (completion, written) = std::sync::mpsc::sync_channel(1); + { + let mut state = self.state.lock().unwrap(); + Self::push_control_locked(&mut state, text, Some(completion))?; + self.changed.notify_one(); + } + match written.recv_timeout(timeout) { + Ok(true) => Ok(()), + Ok(false) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, "response write failed")) + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "response write timed out")) + } + } + } + fn push_terminal(&self, text: String, stream: &OutboundStream) -> std::io::Result<()> { let mut state = self.state.lock().unwrap(); stream.close(); @@ -946,7 +1011,7 @@ impl BoundedOutbound { if stream.terminal_enqueued.swap(true, Ordering::AcqRel) { return Ok(()); } - Self::push_control_locked(&mut state, text)?; + Self::push_control_locked(&mut state, text, None)?; self.changed.notify_one(); Ok(()) } @@ -960,7 +1025,8 @@ impl BoundedOutbound { if stream.terminal_enqueued.swap(true, Ordering::AcqRel) { return Ok(()); } - if let Err(error) = Self::push_control_locked(state, stream.overflow_text.to_string()) { + if let Err(error) = Self::push_control_locked(state, stream.overflow_text.to_string(), None) + { state.closed = true; return Err(std::io::Error::new( std::io::ErrorKind::BrokenPipe, @@ -1005,7 +1071,11 @@ impl BoundedOutbound { .map(|(_, _, stream)| stream) } - fn push_control_locked(state: &mut BoundedOutboundState, text: String) -> std::io::Result<()> { + fn push_control_locked( + state: &mut BoundedOutboundState, + text: String, + completion: Option>, + ) -> std::io::Result<()> { if state.closed { return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, "connection closed")); } @@ -1019,21 +1089,21 @@ impl BoundedOutbound { )); } state.control_bytes += bytes; - state.control.push_back(text); + state.control.push_back(ControlOutbound { text, completion }); Ok(()) } #[cfg(test)] fn try_pop(&self) -> Option { let mut state = self.state.lock().unwrap(); - Self::pop_locked(&mut state) + Self::pop_locked(&mut state).map(OutboundMessage::into_test_text) } - fn recv(&self) -> Option { + fn recv(&self) -> Option { let mut state = self.state.lock().unwrap(); loop { - if let Some(text) = Self::pop_locked(&mut state) { - return Some(text); + if let Some(message) = Self::pop_locked(&mut state) { + return Some(message); } if state.closed { return None; @@ -1042,18 +1112,18 @@ impl BoundedOutbound { } } - fn pop_locked(state: &mut BoundedOutboundState) -> Option { + fn pop_locked(state: &mut BoundedOutboundState) -> Option { if let Some(message) = state.initial.pop_front() { state.regular_bytes -= message.text.len(); - return Some(message.text); + return Some(OutboundMessage { text: message.text, completion: None }); } - if let Some(text) = state.control.pop_front() { - state.control_bytes -= text.len(); - return Some(text); + if let Some(message) = state.control.pop_front() { + state.control_bytes -= message.text.len(); + return Some(OutboundMessage { text: message.text, completion: message.completion }); } let message = state.regular.pop_front()?; state.regular_bytes -= message.text.len(); - Some(message.text) + Some(OutboundMessage { text: message.text, completion: None }) } fn is_open(&self) -> bool { @@ -1061,7 +1131,14 @@ impl BoundedOutbound { } fn close(&self) { - self.state.lock().unwrap().closed = true; + let mut state = self.state.lock().unwrap(); + state.closed = true; + for message in &mut state.control { + if let Some(completion) = message.completion.take() { + let _ = completion.send(false); + } + } + drop(state); self.changed.notify_all(); } } @@ -1150,6 +1227,14 @@ impl MessageSink for QueuedSink { self.outbound.push_control(text) } + fn send_control_confirmed(&self, value: &Value, timeout: Duration) -> std::io::Result<()> { + let text = serde_json::to_string(value)?; + if self.control.is_none() { + return self.outbound.push_control(text); + } + self.outbound.push_control_confirmed(text, timeout) + } + fn send_terminal(&self, value: &Value, stream: &OutboundStream) -> std::io::Result<()> { let text = serde_json::to_string(value)?; self.outbound.push_terminal(text, stream) @@ -1699,14 +1784,17 @@ fn handle_connection(mux: Arc, stream: Box) { let writer_outbound = outbound; let Ok(writer_thread) = std::thread::Builder::new().name("mux-line-out".into()).spawn(move || { - while let Some(text) = writer_outbound.recv() { - if write_half.write_all(text.as_bytes()).is_err() - || write_half.write_all(b"\n").is_err() - { + while let Some(message) = writer_outbound.recv() { + let written = write_half + .write_all(message.text.as_bytes()) + .and_then(|()| write_half.write_all(b"\n")); + if written.is_err() { + message.finish(false); writer_outbound.close(); let _ = write_half.shutdown(Shutdown::Both); break; } + message.finish(true); } let _ = write_half.shutdown(Shutdown::Both); }) @@ -1777,11 +1865,13 @@ fn handle_websocket_connection( let Ok(writer_thread) = std::thread::Builder::new().name("mux-ws-out".into()).spawn(move || { let mut websocket = WebSocket::from_raw_socket(writer_stream, Role::Server, None); - while let Some(text) = writer_outbound.recv() { - if websocket.send(Message::Text(text.into())).is_err() { + while let Some(mut message) = writer_outbound.recv() { + if websocket.send(Message::Text(message.take_text().into())).is_err() { + message.finish(false); writer_outbound.close(); break; } + message.finish(true); } let _ = websocket.close(None); let _ = websocket.flush(); @@ -1929,11 +2019,16 @@ fn handle_message(mux: &Arc, client: u64, message: &str, writer: &MessageWr } }; let response_ok = response.ok; - let sent = - serde_json::to_value(&response).is_ok_and(|value| writer.send_control(&value).is_ok()); - // Queue the successful acknowledgement before making the owning loop - // leave. The headless loop polls at a bounded interval, giving the writer - // thread time to flush the response before normal process teardown. + let sent = serde_json::to_value(&response).is_ok_and(|value| { + if shutdown && response_ok { + writer.send_control_confirmed(&value, SHUTDOWN_ACK_WRITE_TIMEOUT).is_ok() + } else { + writer.send_control(&value).is_ok() + } + }); + // Daemon handoff remains reversible until its response is queued. Full + // shutdown uses the confirmed control path above and requests process exit + // only after the writer reports that the ACK reached the socket. if shutdown_daemon && response_ok { if sent { mux.request_daemon_shutdown(); @@ -5075,7 +5170,7 @@ mod tests { outbound.close(); - assert_eq!(drain.join().unwrap(), None); + assert!(drain.join().unwrap().is_none()); } #[test] @@ -5189,7 +5284,6 @@ mod tests { let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); let request = std::thread::spawn({ let mux = mux.clone(); - let writer = writer.clone(); move || { done_tx .send(handle_message(&mux, client, r#"{"id":7,"cmd":"shutdown"}"#, &writer)) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index e822fa44cf72..5b36008dfdc6 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -18,7 +18,10 @@ mod legacy_process; #[cfg(all(unix, test))] use legacy_process::terminate_process_tree; #[cfg(unix)] -use legacy_process::{CapturedProcessTree, ProcessIdentity, capture_process_tree}; +use legacy_process::{ + CapturedProcessSession, CapturedProcessTree, ProcessIdentity, capture_process_session, + capture_process_tree_until, +}; const PROBE_REQUEST_ID: u64 = 0; const SHUTDOWN_REQUEST_ID: u64 = 1; @@ -32,6 +35,12 @@ const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); const SHUTDOWN_RESPONSE_TIMEOUT: Duration = SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); +#[cfg(unix)] +const LEGACY_SHUTDOWN_TIMEOUT: Duration = SHUTDOWN_RESPONSE_TIMEOUT; +#[cfg(unix)] +const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); +#[cfg(unix)] +const LEGACY_HELPER_POLL_INTERVAL: Duration = Duration::from_millis(10); const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; @@ -136,9 +145,14 @@ pub(crate) struct ServerProbe { impl ServerProbe { pub(crate) fn inspect(reader: &mut TransportReader) -> anyhow::Result { + Self::inspect_until(reader, Instant::now() + RESPONSE_TIMEOUT) + } + + fn inspect_until(reader: &mut TransportReader, deadline: Instant) -> anyhow::Result { + set_transport_deadline(reader.get_mut().as_ref(), deadline)?; write_json_line(reader.get_mut(), &json!({"id": PROBE_REQUEST_ID, "cmd": "identify"})) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_response(reader, PROBE_REQUEST_ID)?; + let response = read_response_until(reader, PROBE_REQUEST_ID, deadline)?; if response.get("ok").and_then(Value::as_bool) != Some(true) { anyhow::bail!(crate::localization::catalog().server.identity_failed); } @@ -147,14 +161,19 @@ impl ServerProbe { } pub(crate) fn connect(path: &Path) -> anyhow::Result<(Self, TransportReader, Option)> { + Self::connect_until(path, Instant::now() + RESPONSE_TIMEOUT) + } + + fn connect_until( + path: &Path, + deadline: Instant, + ) -> anyhow::Result<(Self, TransportReader, Option)> { let stream = transport::connect(path) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.connect_failed))?; let peer_process_id = stream.peer_process_id().ok().flatten(); - stream - .set_read_timeout(Some(RESPONSE_TIMEOUT)) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; + set_transport_deadline(stream.as_ref(), deadline)?; let mut reader = BufReader::new(stream); - let probe = Self::inspect(&mut reader)?; + let probe = Self::inspect_until(&mut reader, deadline)?; Ok((probe, reader, peer_process_id)) } @@ -201,6 +220,12 @@ impl ServerLifecycle { Ok(Self { path, probe, reader, peer_process_id }) } + #[cfg(unix)] + fn connect_until(path: PathBuf, deadline: Instant) -> anyhow::Result { + let (probe, reader, peer_process_id) = ServerProbe::connect_until(&path, deadline)?; + Ok(Self { path, probe, reader, peer_process_id }) + } + pub(crate) fn probe(&self) -> &ServerProbe { &self.probe } @@ -247,14 +272,24 @@ impl ServerLifecycle { fn close_legacy_surfaces_until_stable( &mut self, expected: ProcessIdentity, - ) -> anyhow::Result<()> { + deadline: Instant, + ) -> anyhow::Result> { let mut next_request_id = LEGACY_LIST_REQUEST_ID; let mut consecutive_empty_scans = 0; + let mut owners = Vec::::new(); for _ in 0..LEGACY_MAX_SCAN_ROUNDS { - let closed = match self.close_legacy_surface_snapshot(&mut next_request_id) { + if Instant::now() >= deadline { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + let closed = match self.close_legacy_surface_snapshot( + &mut next_request_id, + expected, + &mut owners, + deadline, + ) { Ok(closed) => closed, Err(error) if error.downcast_ref::().is_some() => { - self.reconnect_legacy_server(expected)?; + self.reconnect_legacy_server(expected, deadline)?; consecutive_empty_scans = 0; continue; } @@ -263,7 +298,7 @@ impl ServerLifecycle { if closed == 0 { consecutive_empty_scans += 1; if consecutive_empty_scans == LEGACY_STABLE_EMPTY_SCANS { - return Ok(()); + return Ok(owners); } } else { consecutive_empty_scans = 0; @@ -273,8 +308,12 @@ impl ServerLifecycle { } #[cfg(unix)] - fn reconnect_legacy_server(&mut self, expected: ProcessIdentity) -> anyhow::Result<()> { - let replacement = Self::connect(self.path.clone()).map_err(|_| { + fn reconnect_legacy_server( + &mut self, + expected: ProcessIdentity, + deadline: Instant, + ) -> anyhow::Result<()> { + let replacement = Self::connect_until(self.path.clone(), deadline).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; if replacement.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { @@ -293,28 +332,71 @@ impl ServerLifecycle { fn close_legacy_surface_snapshot( &mut self, next_request_id: &mut u64, + expected: ProcessIdentity, + owners: &mut Vec, + deadline: Instant, ) -> anyhow::Result { let list_request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline) + .map_err(|_| LegacyConnectionInterrupted)?; write_json_line( self.reader.get_mut(), &json!({"id": list_request_id, "cmd": "list-workspaces"}), ) .map_err(|_| LegacyConnectionInterrupted)?; - let response = read_response(&mut self.reader, list_request_id) + let response = read_response_until(&mut self.reader, list_request_id, deadline) .map_err(|_| LegacyConnectionInterrupted)?; let data = response_data(&response)?; - let mut surfaces = legacy_surface_ids(data); - surfaces.sort_unstable(); - surfaces.dedup(); + let surfaces = legacy_surfaces(data)?; + + for surface in &surfaces { + if surface.kind != "pty" { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + let process_request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline) + .map_err(|_| LegacyConnectionInterrupted)?; + write_json_line( + self.reader.get_mut(), + &json!({ + "id": process_request_id, + "cmd": "process-info", + "surface": surface.id, + }), + ) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_response = + read_response_until(&mut self.reader, process_request_id, deadline) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_data = response_data(&process_response)?; + let pid = process_data + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .filter(|pid| *pid > 1) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let owner = capture_process_session(pid, expected) + .map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })? + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + if !owners.iter().any(|captured| captured.id() == owner.id()) { + owners.push(owner); + } - for &surface in &surfaces { let request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline) + .map_err(|_| LegacyConnectionInterrupted)?; write_json_line( self.reader.get_mut(), - &json!({"id": request_id, "cmd": "close-surface", "surface": surface}), + &json!({"id": request_id, "cmd": "close-surface", "surface": surface.id}), ) .map_err(|_| LegacyConnectionInterrupted)?; - let _response = read_response(&mut self.reader, request_id) + let _response = read_response_until(&mut self.reader, request_id, deadline) .map_err(|_| LegacyConnectionInterrupted)?; // Older servers can commit topology removal, then report a late // runtime-cleanup error. Reconcile every close response against @@ -348,9 +430,10 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R .arg(path) .arg(expected.pid().to_string()) .arg(expected.started_at().to_string()) + .arg(LEGACY_SHUTDOWN_TIMEOUT.as_millis().to_string()) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::piped()); + .stderr(Stdio::null()); unsafe { command.pre_exec(|| { if libc::setsid() < 0 { @@ -359,18 +442,42 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R Ok(()) }); } - let output = command.output().map_err(|_| { + let mut helper = command.spawn().map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - if output.status.success() { + let deadline = Instant::now() + LEGACY_SHUTDOWN_TIMEOUT + LEGACY_HELPER_WAIT_MARGIN; + let status = wait_for_child_until(&mut helper, deadline)?; + if status.success() { return Ok(()); } anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) } +#[cfg(unix)] +fn wait_for_child_until( + child: &mut std::process::Child, + deadline: Instant, +) -> anyhow::Result { + loop { + if let Some(status) = child.try_wait().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })? { + return Ok(status); + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(LEGACY_HELPER_POLL_INTERVAL), + ); + } +} + #[cfg(unix)] pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { - let [path, expected_pid, expected_started_at] = args else { + let [path, expected_pid, expected_started_at, timeout_ms] = args else { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); }; let expected_pid = @@ -380,8 +487,17 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { let expected_started_at = expected_started_at .parse::() .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; + let timeout = timeout_ms + .parse::() + .ok() + .map(Duration::from_millis) + .filter(|timeout| !timeout.is_zero() && *timeout <= LEGACY_SHUTDOWN_TIMEOUT) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) + })?; + let deadline = Instant::now() + timeout; let expected = ProcessIdentity::from_parts(expected_pid, expected_started_at); - let mut lifecycle = ServerLifecycle::connect(PathBuf::from(path))?; + let mut lifecycle = ServerLifecycle::connect_until(PathBuf::from(path), deadline)?; if lifecycle.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } @@ -389,13 +505,18 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { if actual != expected { anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - let captured = capture_legacy_process_tree(actual)?; + let captured = capture_legacy_process_tree(actual, deadline)?; - lifecycle.close_legacy_surfaces_until_stable(actual).map_err(|_| { + let owners = lifecycle.close_legacy_surfaces_until_stable(actual, deadline).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - terminate_captured_legacy_process_tree(captured)?; - wait_for_disconnect(&mut lifecycle.reader, &lifecycle.path) + for owner in owners { + owner.terminate_until(deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + } + terminate_captured_legacy_process_tree(captured, deadline)?; + wait_for_disconnect_until(&mut lifecycle.reader, &lifecycle.path, deadline) } #[cfg(unix)] @@ -426,15 +547,21 @@ fn terminate_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result<()> } #[cfg(unix)] -fn capture_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result { - capture_process_tree(process) +fn capture_legacy_process_tree( + process: ProcessIdentity, + deadline: Instant, +) -> anyhow::Result { + capture_process_tree_until(process, deadline) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) } #[cfg(unix)] -fn terminate_captured_legacy_process_tree(process: CapturedProcessTree) -> anyhow::Result<()> { +fn terminate_captured_legacy_process_tree( + process: CapturedProcessTree, + deadline: Instant, +) -> anyhow::Result<()> { process - .terminate() + .terminate_until(deadline) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) } @@ -509,36 +636,52 @@ pub(crate) fn write_json_line(writer: &mut dyn Write, value: &Value) -> std::io: writer.write_all(b"\n") } -fn read_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { - read_matching_response(reader, request_id, false) +fn set_transport_deadline(stream: &dyn transport::Stream, deadline: Instant) -> anyhow::Result<()> { + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; + stream + .set_read_timeout(Some(remaining)) + .and_then(|()| stream.set_write_timeout(Some(remaining))) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed)) +} + +fn read_response_until( + reader: &mut TransportReader, + request_id: u64, + deadline: Instant, +) -> anyhow::Result { + read_matching_response_until(reader, request_id, false, deadline) } fn read_shutdown_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { read_matching_response_with_timeout(reader, request_id, true, SHUTDOWN_RESPONSE_TIMEOUT) } -fn read_matching_response( +fn read_matching_response_with_timeout( reader: &mut TransportReader, request_id: u64, accept_unidentified_error: bool, + timeout: Duration, ) -> anyhow::Result { - read_matching_response_with_timeout( + read_matching_response_until( reader, request_id, accept_unidentified_error, - RESPONSE_TIMEOUT, + Instant::now() + timeout, ) } -fn read_matching_response_with_timeout( +fn read_matching_response_until( reader: &mut TransportReader, request_id: u64, accept_unidentified_error: bool, - timeout: Duration, + deadline: Instant, ) -> anyhow::Result { - let deadline = Instant::now() + timeout; let mut line = String::new(); loop { + set_transport_deadline(reader.get_mut().as_ref(), deadline)?; match reader.read_line(&mut line) { Ok(0) => anyhow::bail!(crate::localization::catalog().server.response_closed), Ok(_) => {} @@ -578,26 +721,64 @@ fn response_data(response: &Value) -> anyhow::Result<&Value> { } #[cfg(unix)] -fn legacy_surface_ids(data: &Value) -> Vec { - let mut surfaces = Vec::new(); +struct LegacySurface { + id: u64, + kind: String, +} + +#[cfg(unix)] +fn legacy_surfaces(data: &Value) -> anyhow::Result> { + let mut surfaces = std::collections::HashMap::::new(); for workspace in data.get("workspaces").and_then(Value::as_array).into_iter().flatten() { for screen in workspace.get("screens").and_then(Value::as_array).into_iter().flatten() { for pane in screen.get("panes").and_then(Value::as_array).into_iter().flatten() { for tab in pane.get("tabs").and_then(Value::as_array).into_iter().flatten() { if let Some(surface) = tab.get("surface").and_then(Value::as_u64) { - surfaces.push(surface); + let kind = tab.get("kind").and_then(Value::as_str).ok_or_else(|| { + anyhow::anyhow!( + crate::localization::catalog().server.legacy_cleanup_failed + ) + })?; + if let Some(previous) = surfaces.insert(surface, kind.to_string()) + && previous != kind + { + anyhow::bail!( + crate::localization::catalog().server.legacy_cleanup_failed + ); + } } } } } } - surfaces + let mut surfaces = + surfaces.into_iter().map(|(id, kind)| LegacySurface { id, kind }).collect::>(); + surfaces.sort_by_key(|surface| surface.id); + Ok(surfaces) +} + +#[cfg(all(unix, test))] +fn legacy_surface_ids(data: &Value) -> Vec { + legacy_surfaces(data).unwrap().into_iter().map(|surface| surface.id).collect() } fn wait_for_disconnect(reader: &mut TransportReader, path: &Path) -> anyhow::Result<()> { - let deadline = Instant::now() + RESPONSE_TIMEOUT; + wait_for_disconnect_until(reader, path, Instant::now() + RESPONSE_TIMEOUT) +} + +fn wait_for_disconnect_until( + reader: &mut TransportReader, + path: &Path, + deadline: Instant, +) -> anyhow::Result<()> { let mut line = String::new(); loop { + if set_transport_deadline(reader.get_mut().as_ref(), deadline).is_err() { + if connection_is_gone(path) { + return Ok(()); + } + anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out); + } match reader.read_line(&mut line) { Ok(0) => return Ok(()), Ok(_) => line.clear(), @@ -770,8 +951,11 @@ mod tests { "workspaces": [{ "screens": [{ "panes": [ - {"tabs": [{"surface": 11}, {"surface": 12}]}, - {"tabs": [{"surface": 13}]}, + {"tabs": [ + {"surface": 11, "kind": "pty"}, + {"surface": 12, "kind": "pty"} + ]}, + {"tabs": [{"surface": 13, "kind": "pty"}]}, ], }], }], diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index f941c8538763..bf4c1132fea1 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -4,6 +4,7 @@ use std::time::{Duration, Instant}; const PROCESS_TREE_MAX_ROUNDS: usize = 64; const PROCESS_TREE_RETRY_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(test)] const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -89,10 +90,17 @@ enum ExactSignalResult { Gone, } +#[cfg(test)] pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { - let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; + terminate_process_tree_until(process, Instant::now() + PROCESS_TREE_RETRY_TIMEOUT) +} + +pub(super) fn terminate_process_tree_until( + process: ProcessIdentity, + deadline: Instant, +) -> io::Result<()> { retry_process_tree_termination(process, deadline, |_| { - let tree = FrozenProcessTree::freeze(process)?; + let tree = FrozenProcessTree::freeze(process, deadline)?; tree.kill() }) } @@ -126,17 +134,22 @@ fn retry_process_tree_termination( } } -pub(super) fn capture_process_tree(process: ProcessIdentity) -> io::Result { - let tree = freeze_process_tree(process)?; +pub(super) fn capture_process_tree_until( + process: ProcessIdentity, + deadline: Instant, +) -> io::Result { + let tree = freeze_process_tree(process, deadline)?; let sessions = tree.captured_sessions()?; drop(tree); Ok(CapturedProcessTree { root: process, sessions }) } -fn freeze_process_tree(process: ProcessIdentity) -> io::Result { - let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; +fn freeze_process_tree( + process: ProcessIdentity, + deadline: Instant, +) -> io::Result { loop { - let error = match FrozenProcessTree::freeze(process) { + let error = match FrozenProcessTree::freeze(process, deadline) { Ok(tree) => return Ok(tree), Err(error) => error, }; @@ -168,8 +181,7 @@ pub(super) struct CapturedProcessTree { } impl CapturedProcessTree { - pub(super) fn terminate(self) -> io::Result<()> { - let deadline = Instant::now() + PROCESS_TREE_RETRY_TIMEOUT; + pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { for session in &self.sessions { if !session.kill_until_empty(deadline)? { return Err(io::Error::other( @@ -177,10 +189,84 @@ impl CapturedProcessTree { )); } } - terminate_process_tree(self.root) + terminate_process_tree_until(self.root, deadline) + } +} + +pub(super) struct CapturedProcessSession(CapturedSession); + +impl CapturedProcessSession { + pub(super) fn id(&self) -> libc::pid_t { + self.0.id + } + + pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { + if self.0.kill_until_empty(deadline)? { + Ok(()) + } else { + Err(io::Error::other( + "captured PTY session did not exit before the legacy shutdown deadline", + )) + } } } +pub(super) fn capture_process_session( + pid: libc::pid_t, + server: ProcessIdentity, +) -> io::Result> { + if pid <= 1 + || pid == server.pid + || pid == libc::pid_t::try_from(std::process::id()).unwrap_or(0) + { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid PTY owner process")); + } + let Some(process) = ProcessIdentity::capture(pid)? else { return Ok(None) }; + if process.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { + return Ok(None); + } + let captured = (|| { + if ProcessIdentity::capture(pid)? != Some(process) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PTY owner process identity changed after fencing", + )); + } + // SAFETY: getsid only queries exact processes revalidated above. + let session = unsafe { libc::getsid(pid) }; + if session <= 1 { + return Err(io::Error::last_os_error()); + } + // SAFETY: the server identity remains validated by the caller. + let server_session = unsafe { libc::getsid(server.pid) }; + // SAFETY: getsid(0) only queries the detached helper. + let helper_session = unsafe { libc::getsid(0) }; + if session == server_session || session == helper_session { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "PTY owner shares a protected process session", + )); + } + let mut members = Vec::new(); + for member in cmux_tui_core::process_session::session_member_pids(session)? { + if let Some(identity) = ProcessIdentity::capture(member)? { + members.push(identity); + } + } + if !members.contains(&process) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PTY owner left its process session while being captured", + )); + } + members.sort_by_key(|member| member.pid); + members.dedup(); + Ok(CapturedProcessSession(CapturedSession { id: session, members })) + })(); + let _ = process.signal(libc::SIGCONT); + captured.map(Some) +} + struct CapturedSession { id: libc::pid_t, members: Vec, @@ -266,7 +352,7 @@ struct FrozenProcessTree { } impl FrozenProcessTree { - fn freeze(root: ProcessIdentity) -> io::Result { + fn freeze(root: ProcessIdentity, deadline: Instant) -> io::Result { let mut tree = Self { root, descendants: Vec::new(), armed: true }; if root.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { return Err(io::Error::new( @@ -285,6 +371,12 @@ impl FrozenProcessTree { .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid helper process id"))?; let mut known = HashSet::from([root.pid, helper_pid]); for _ in 0..PROCESS_TREE_MAX_ROUNDS { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "legacy process tree did not stabilize before the shutdown deadline", + )); + } let parents = std::iter::once(root).chain(tree.descendants.iter().copied()).collect::>(); let mut added = false; diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 3505afca24d9..223261d66091 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -303,8 +303,17 @@ fn legacy_server_process_helper() { | "persistent-close-error" | "browser-surface" ) { - let descendant = - Command::new("yes").stdout(Stdio::null()).stderr(Stdio::null()).spawn().unwrap(); + let mut command = Command::new("yes"); + command.stdout(Stdio::null()).stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let descendant = command.spawn().unwrap(); fs::write( std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), descendant.id().to_string(), @@ -508,8 +517,39 @@ fn legacy_server_process_helper() { .unwrap(); for &expected_surface in surfaces { request.clear(); - reader.read_line(&mut request).unwrap(); - let close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + if reader.read_line(&mut request).unwrap() == 0 { + loop { + std::thread::park(); + } + } + let mut close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + if close_request["cmd"].as_str() == Some("process-info") { + assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + let pid = fs::read_to_string( + std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), + ) + .unwrap() + .trim() + .parse::() + .unwrap(); + writeln!( + stream, + "{}", + serde_json::json!({ + "id": close_request["id"], + "ok": true, + "data": {"pid": pid}, + }) + ) + .unwrap(); + request.clear(); + if reader.read_line(&mut request).unwrap() == 0 { + loop { + std::thread::park(); + } + } + close_request = serde_json::from_str(&request).unwrap(); + } assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); if scenario == "applied-close-disconnect" { From 299af8a9dba16f83f38b22ce29559aeb8ebed609 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:20:39 -0700 Subject: [PATCH 048/521] test(tui): cover scoped shutdown regressions --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 19 +++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 84 +++++++++++++++++++ .../cmux-tui-core/src/process_session.rs | 73 ++++++++++++++-- cmux-tui/crates/cmux-tui/src/cli.rs | 32 ++++++- 4 files changed, 200 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 25cee8789e59..1c237457e1e4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -404,6 +404,11 @@ impl BrowserRuntime { Self::connect_to_endpoint(&web_socket_url, chrome, source) } + #[cfg(test)] + pub(crate) fn connect_external_for_test(web_socket_url: &str) -> anyhow::Result> { + Self::connect_to_endpoint(web_socket_url, None, BrowserSource::External) + } + fn connect_to_endpoint( web_socket_url: &str, chrome: Option, @@ -1248,6 +1253,20 @@ fn emit_browser_failure(mux: &Weak, id: SurfaceId, message: String) { } impl BrowserSurface { + #[cfg(test)] + pub(crate) fn install_shutdown_session_for_test( + &self, + runtime: Arc, + target_id: &str, + session_id: &str, + ) { + *self.session.lock().unwrap() = Some(BrowserSession { + runtime, + target_id: target_id.to_string(), + session_id: session_id.to_string(), + }); + } + pub(crate) fn shutdown_owner(&self) -> Option { if !self.dead.swap(true, Ordering::AcqRel) { self.close_taps(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 01cf55fbd963..7618f017b84d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10190,6 +10190,90 @@ mod tests { ); } + #[test] + fn ordinary_browser_close_terminates_the_owner_staged_during_removal() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (query_tx, query_rx) = std::sync::mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + + let query = read_ws_json(&mut ws); + assert_eq!(query["method"], "Target.getTargets"); + query_tx.send(()).unwrap(); + ws.send(tungstenite::Message::Text( + serde_json::json!({ + "id": query["id"], + "result": {"targetInfos": []} + }) + .to_string() + .into(), + )) + .unwrap(); + }); + let runtime = BrowserRuntime::connect_external_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + let mux = test_mux(); + let options = mux.surface_options.lock().unwrap().clone(); + let surface = browser::new_surface( + 999, + "about:blank".to_string(), + (80, 24), + (8, 16), + &options, + Arc::downgrade(&mux), + ); + surface.as_browser().unwrap().install_shutdown_session_for_test( + runtime.clone(), + "target-1", + "session-1", + ); + insert_surface_checked(&mut mux.state.lock().unwrap(), surface.clone()).unwrap(); + + let removed = + take_surface_for_retirement(&mux, &mut mux.state.lock().unwrap(), surface.id).unwrap(); + mux.retire_surface_runtime(removed); + let attempted_during_close = query_rx.recv_timeout(Duration::from_millis(200)).is_ok(); + + if !attempted_during_close { + mux.terminate_staged_shutdown_owners_until(Instant::now() + Duration::from_secs(1)); + query_rx + .recv_timeout(Duration::from_secs(1)) + .expect("cleanup retry did not reach the staged browser owner"); + } + runtime.shutdown(); + server.join().unwrap(); + + assert!( + attempted_during_close, + "ordinary close lost the consume-once browser owner staged during removal" + ); + assert!(mux.shutdown_owners.is_empty()); + } + #[test] fn server_shutdown_cannot_pass_an_in_flight_surface_retirement() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 74473c544f09..2d34ac720989 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -10,6 +10,11 @@ use std::time::{Duration, Instant}; const SESSION_KILL_POLL: Duration = Duration::from_millis(5); +#[cfg(test)] +thread_local! { + static PROCESS_TABLE_SCAN_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + pub(crate) fn signal(session: libc::pid_t, signal: libc::c_int) -> io::Result<()> { validate_session_target(session)?; for pid in members(session)? { @@ -165,6 +170,8 @@ fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { } fn members(session: libc::pid_t) -> io::Result> { + #[cfg(test)] + PROCESS_TABLE_SCAN_COUNT.set(PROCESS_TABLE_SCAN_COUNT.get() + 1); let mut members = Vec::new(); for pid in all_process_ids()? { if pid <= 1 { @@ -188,6 +195,25 @@ fn members(session: libc::pid_t) -> io::Result> { Ok(members) } +#[cfg(test)] +fn shutdown_batch_members_for_test( + sessions: &[libc::pid_t], + deadline: Instant, +) -> io::Result>> { + sessions + .iter() + .map(|session| { + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "process-session snapshot deadline expired", + )); + } + members(*session) + }) + .collect() +} + #[cfg(all(target_os = "macos", test))] fn macos_session_process_ids(session: libc::pid_t) -> io::Result> { members(session) @@ -259,17 +285,16 @@ fn all_process_ids() -> io::Result> { #[cfg(test)] mod tests { - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] use std::os::unix::process::CommandExt; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] use std::process::{Command, Stdio}; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "linux"))] use super::*; - #[cfg(target_os = "macos")] - #[test] - fn macos_session_query_is_scoped_to_the_owned_session() { + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn spawn_session_child() -> std::process::Child { let mut command = Command::new("sleep"); command.arg("60").stdout(Stdio::null()).stderr(Stdio::null()); unsafe { @@ -280,7 +305,13 @@ mod tests { Ok(()) }); } - let mut child = command.spawn().unwrap(); + command.spawn().unwrap() + } + + #[cfg(target_os = "macos")] + #[test] + fn macos_session_query_is_scoped_to_the_owned_session() { + let mut child = spawn_session_child(); let session = libc::pid_t::try_from(child.id()).unwrap(); let members = macos_session_process_ids(session).unwrap(); @@ -291,4 +322,32 @@ mod tests { child.kill().unwrap(); child.wait().unwrap(); } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn shutdown_batch_reuses_one_process_table_snapshot() { + let mut children = [spawn_session_child(), spawn_session_child()]; + let sessions = children + .iter() + .map(|child| libc::pid_t::try_from(child.id()).unwrap()) + .collect::>(); + PROCESS_TABLE_SCAN_COUNT.set(0); + + let snapshots = + shutdown_batch_members_for_test(&sessions, Instant::now() + Duration::from_secs(1)); + let scan_count = PROCESS_TABLE_SCAN_COUNT.get(); + + for child in &mut children { + child.kill().unwrap(); + child.wait().unwrap(); + } + let snapshots = snapshots.unwrap(); + assert!( + snapshots.iter().zip(&sessions).all(|(members, session)| members.contains(session)) + ); + assert_eq!( + scan_count, 1, + "one shutdown batch scanned the global process table more than once" + ); + } } diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index a4e71219ad71..fcdd7e4ce0d4 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1718,6 +1718,7 @@ fn atom(value: Option<&Value>) -> String { mod tests { use std::collections::VecDeque; use std::net::Shutdown; + use std::sync::{Arc, Mutex}; use super::*; @@ -1725,6 +1726,7 @@ mod tests { reads: VecDeque, io::ErrorKind>>, current: io::Cursor>, writes: Vec, + read_timeouts: Arc>>>, } impl Read for ScriptedStream { @@ -1759,7 +1761,8 @@ mod tests { Err(io::Error::new(io::ErrorKind::Unsupported, "test stream is not cloneable")) } - fn set_read_timeout(&self, _timeout: Option) -> io::Result<()> { + fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { + self.read_timeouts.lock().unwrap().push(timeout); Ok(()) } @@ -1774,6 +1777,7 @@ mod tests { #[test] fn capability_probe_tolerates_polling_timeouts() { + let read_timeouts = Arc::new(Mutex::new(Vec::new())); let stream = ScriptedStream { reads: VecDeque::from([ Err(io::ErrorKind::WouldBlock), @@ -1783,6 +1787,7 @@ mod tests { ]), current: io::Cursor::new(Vec::new()), writes: Vec::new(), + read_timeouts, }; let mut reader = BufReader::new(Box::new(stream) as Box); @@ -1794,6 +1799,7 @@ mod tests { #[test] fn capability_probe_preserves_partial_line_across_timeout() { + let read_timeouts = Arc::new(Mutex::new(Vec::new())); let stream = ScriptedStream { reads: VecDeque::from([ Ok(b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",".to_vec()), @@ -1802,6 +1808,7 @@ mod tests { ]), current: io::Cursor::new(Vec::new()), writes: Vec::new(), + read_timeouts, }; let mut reader = BufReader::new(Box::new(stream) as Box); @@ -1811,6 +1818,29 @@ mod tests { ); } + #[test] + fn streaming_probe_restores_the_polling_read_timeout() { + let read_timeouts = Arc::new(Mutex::new(Vec::new())); + let stream = ScriptedStream { + reads: VecDeque::from([Ok( + b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",\"capabilities\":[]}}\n" + .to_vec(), + )]), + current: io::Cursor::new(Vec::new()), + writes: Vec::new(), + read_timeouts: read_timeouts.clone(), + }; + transport::Stream::set_read_timeout(&stream, Some(Duration::from_millis(250))).unwrap(); + let mut reader = BufReader::new(Box::new(stream) as Box); + + assert!(server_supports_capability(&mut reader, "unused").is_ok()); + assert_eq!( + read_timeouts.lock().unwrap().last().copied().flatten(), + Some(Duration::from_millis(250)), + "identity probing replaced the stream's short shutdown polling timeout" + ); + } + #[test] fn plugin_verb_is_registered_as_local_with_help() { let plugin = verb_by_name("plugin").expect("plugin verb registered"); From 0c950bffad5df01a834fb52918f161758b6d8e4f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:29:57 -0700 Subject: [PATCH 049/521] fix(tui): preserve scoped shutdown guarantees --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 +- cmux-tui/crates/cmux-tui-core/src/platform.rs | 18 + .../cmux-tui-core/src/process_session.rs | 355 +++++++++++++++--- cmux-tui/crates/cmux-tui-core/src/surface.rs | 111 +++++- .../src/terminal_host_runtime.rs | 16 +- cmux-tui/crates/cmux-tui/src/cli.rs | 8 + .../crates/cmux-tui/src/server_lifecycle.rs | 40 +- .../src/server_lifecycle/legacy_process.rs | 21 +- 8 files changed, 499 insertions(+), 90 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7618f017b84d..556e7bd82985 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -67,6 +67,10 @@ struct ShutdownOwnerLedger { } impl ShutdownOwnerLedger { + fn get(&self, key: &ShutdownOwnerKey) -> Option<(ShutdownOwnerKey, Arc)> { + self.owners.lock().unwrap().get(key).cloned().map(|owner| (key.clone(), owner)) + } + fn stage( &self, key: ShutdownOwnerKey, @@ -86,6 +90,10 @@ impl ShutdownOwnerLedger { &self, surface: &Arc, ) -> Option<(ShutdownOwnerKey, Arc)> { + let surface_key = ShutdownOwnerKey::Surface(surface.id); + if let Some(staged) = self.get(&surface_key) { + return Some(staged); + } let owner = surface.shutdown_owner()?; Some(self.stage(shutdown_owner_key(surface.id, &owner), owner)) } @@ -3835,9 +3843,19 @@ impl Mux { if owners.is_empty() { return; } + #[cfg(unix)] + let process_snapshot = crate::process_session::SessionProcessSnapshot::capture( + owners.iter().filter_map(|(_, owner)| owner.local_process_session()), + deadline, + ) + .ok(); let failed = Mutex::new(HashSet::new()); let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { - if !owner.terminate_until(deadline) { + #[cfg(unix)] + let terminated = owner.terminate_until_in_batch(deadline, process_snapshot.as_ref()); + #[cfg(not(unix))] + let terminated = owner.terminate_until(deadline); + if !terminated { failed.lock().unwrap().insert(key.clone()); } }); diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 76e24ec6d6c4..3dd0faf2a911 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -10,7 +10,9 @@ pub mod transport { pub trait Stream: Read + Write + Send + Sync { fn try_clone_box(&self) -> io::Result>; + fn read_timeout(&self) -> io::Result>; fn set_read_timeout(&self, timeout: Option) -> io::Result<()>; + fn write_timeout(&self) -> io::Result>; fn set_write_timeout(&self, timeout: Option) -> io::Result<()>; fn shutdown(&self, how: Shutdown) -> io::Result<()>; fn peer_process_id(&self) -> io::Result> { @@ -69,10 +71,18 @@ pub mod transport { Ok(Box::new(self.try_clone()?)) } + fn read_timeout(&self) -> io::Result> { + UnixStream::read_timeout(self) + } + fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { UnixStream::set_read_timeout(self, timeout) } + fn write_timeout(&self) -> io::Result> { + UnixStream::write_timeout(self) + } + fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { UnixStream::set_write_timeout(self, timeout) } @@ -179,10 +189,18 @@ pub mod transport { Ok(Box::new(self.try_clone()?)) } + fn read_timeout(&self) -> io::Result> { + UnixStream::read_timeout(self) + } + fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { UnixStream::set_read_timeout(self, timeout) } + fn write_timeout(&self) -> io::Result> { + UnixStream::write_timeout(self) + } + fn set_write_timeout(&self, timeout: Option) -> io::Result<()> { UnixStream::set_write_timeout(self, timeout) } diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2d34ac720989..2b8f56fb07ef 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -5,7 +5,9 @@ //! enumerates and signals every session member instead of guessing which //! foreground or background groups still exist. +use std::collections::{HashMap, HashSet}; use std::io; +use std::sync::{Condvar, Mutex}; use std::time::{Duration, Instant}; const SESSION_KILL_POLL: Duration = Duration::from_millis(5); @@ -15,12 +17,159 @@ thread_local! { static PROCESS_TABLE_SCAN_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } -pub(crate) fn signal(session: libc::pid_t, signal: libc::c_int) -> io::Result<()> { - validate_session_target(session)?; - for pid in members(session)? { - signal_if_still_member(pid, session, signal)?; +pub(crate) struct SessionProcessSnapshot { + sessions: HashSet, + state: Mutex, + refreshed: Condvar, + #[cfg(test)] + scan_count: std::sync::atomic::AtomicUsize, +} + +struct SessionProcessSnapshotState { + generation: u64, + members: HashMap>, + refreshing: bool, + failure: Option, +} + +#[derive(Clone)] +struct SessionScanFailure { + kind: io::ErrorKind, + message: String, +} + +impl SessionScanFailure { + fn from_error(error: &io::Error) -> Self { + Self { kind: error.kind(), message: error.to_string() } + } + + fn to_error(&self) -> io::Error { + io::Error::new(self.kind, self.message.clone()) } - Ok(()) +} + +impl SessionProcessSnapshot { + pub(crate) fn capture( + sessions: impl IntoIterator, + deadline: Instant, + ) -> io::Result { + let sessions = sessions.into_iter().collect::>(); + for session in &sessions { + validate_session_target(*session)?; + } + let members = scan_sessions(&sessions, Some(deadline))?; + #[cfg(test)] + let has_sessions = !sessions.is_empty(); + Ok(Self { + sessions, + state: Mutex::new(SessionProcessSnapshotState { + generation: 0, + members, + refreshing: false, + failure: None, + }), + refreshed: Condvar::new(), + #[cfg(test)] + scan_count: std::sync::atomic::AtomicUsize::new(usize::from(has_sessions)), + }) + } + + fn members(&self, session: libc::pid_t) -> io::Result<(u64, Vec)> { + if !self.sessions.contains(&session) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "PTY session is outside the shutdown snapshot", + )); + } + let state = self.state.lock().unwrap(); + if let Some(failure) = &state.failure { + return Err(failure.to_error()); + } + Ok((state.generation, state.members.get(&session).cloned().unwrap_or_default())) + } + + fn refresh_members( + &self, + session: libc::pid_t, + observed_generation: u64, + deadline: Instant, + ) -> io::Result<(u64, Vec)> { + if !self.sessions.contains(&session) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "PTY session is outside the shutdown snapshot", + )); + } + let mut state = self.state.lock().unwrap(); + loop { + if let Some(failure) = &state.failure { + return Err(failure.to_error()); + } + if state.generation != observed_generation { + return Ok(( + state.generation, + state.members.get(&session).cloned().unwrap_or_default(), + )); + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(snapshot_deadline_error()); + }; + if !state.refreshing { + state.refreshing = true; + break; + } + let (next, timeout) = self.refreshed.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.generation == observed_generation { + return Err(snapshot_deadline_error()); + } + } + drop(state); + + #[cfg(test)] + self.scan_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let scanned = scan_sessions(&self.sessions, Some(deadline)); + let mut state = self.state.lock().unwrap(); + state.refreshing = false; + match scanned { + Ok(members) => { + state.generation = state.generation.saturating_add(1); + state.members = members; + } + Err(error) => { + state.failure = Some(SessionScanFailure::from_error(&error)); + } + } + self.refreshed.notify_all(); + if let Some(failure) = &state.failure { + return Err(failure.to_error()); + } + Ok((state.generation, state.members.get(&session).cloned().unwrap_or_default())) + } + + #[cfg(test)] + fn scan_count(&self) -> usize { + self.scan_count.load(std::sync::atomic::Ordering::Relaxed) + } +} + +pub(crate) fn signal_until( + session: libc::pid_t, + signal: libc::c_int, + deadline: Instant, +) -> io::Result<()> { + let snapshot = SessionProcessSnapshot::capture([session], deadline)?; + signal_from_snapshot(&snapshot, session, signal) +} + +pub(crate) fn signal_from_snapshot( + snapshot: &SessionProcessSnapshot, + session: libc::pid_t, + signal: libc::c_int, +) -> io::Result<()> { + validate_session_target(session)?; + let (_, members) = snapshot.members(session)?; + signal_members_with(&members, session, None, signal) } /// Repeatedly SIGKILL every member until the session contains only its @@ -33,32 +182,18 @@ pub(crate) fn kill_until_only_leader( leader_exited: impl Fn() -> bool, ) -> io::Result { validate_session_target(session)?; - let mut known_members = members(session)?; - signal_members(&known_members, session, None)?; - loop { - let known_drained = known_members - .iter() - .copied() - .filter(|pid| *pid != leader) - .map(|pid| still_member(pid, session)) - .collect::>>()? - .into_iter() - .all(|alive| !alive); - if leader_exited() && known_drained { - let current = members(session)?; - if current.iter().all(|pid| *pid == leader) { - return Ok(true); - } - signal_members(¤t, session, None)?; - known_members = current; - } - if Instant::now() >= deadline { - return Ok(false); - } - std::thread::sleep( - deadline.saturating_duration_since(Instant::now()).min(SESSION_KILL_POLL), - ); - } + let snapshot = SessionProcessSnapshot::capture([session], deadline)?; + kill_until_only_leader_from_snapshot(&snapshot, session, leader, deadline, leader_exited) +} + +pub(crate) fn kill_until_only_leader_from_snapshot( + snapshot: &SessionProcessSnapshot, + session: libc::pid_t, + leader: libc::pid_t, + deadline: Instant, + leader_exited: impl Fn() -> bool, +) -> io::Result { + kill_until_only_reserved_from_snapshot(snapshot, session, leader, deadline, leader_exited, None) } /// Drain a captured session while the caller keeps one exact member stopped. @@ -73,8 +208,28 @@ pub fn kill_until_only_reserved( if !still_member(reserved, session)? { return Err(io::Error::new(io::ErrorKind::NotFound, "reserved PTY session process exited")); } - let mut known_members = members(session)?; - signal_members(&known_members, session, Some(reserved))?; + let snapshot = SessionProcessSnapshot::capture([session], deadline)?; + kill_until_only_reserved_from_snapshot( + &snapshot, + session, + reserved, + deadline, + || true, + Some(reserved), + ) +} + +fn kill_until_only_reserved_from_snapshot( + snapshot: &SessionProcessSnapshot, + session: libc::pid_t, + reserved: libc::pid_t, + deadline: Instant, + can_reconcile: impl Fn() -> bool, + signal_exclusion: Option, +) -> io::Result { + validate_session_target(session)?; + let (mut generation, mut known_members) = snapshot.members(session)?; + signal_members(&known_members, session, signal_exclusion)?; loop { let known_drained = known_members .iter() @@ -84,12 +239,14 @@ pub fn kill_until_only_reserved( .collect::>>()? .into_iter() .all(|alive| !alive); - if known_drained { - let current = members(session)?; + if can_reconcile() && known_drained { + let (current_generation, current) = + snapshot.refresh_members(session, generation, deadline)?; if current.iter().all(|pid| *pid == reserved) { return Ok(true); } - signal_members(¤t, session, Some(reserved))?; + signal_members(¤t, session, signal_exclusion)?; + generation = current_generation; known_members = current; } if Instant::now() >= deadline { @@ -103,25 +260,37 @@ pub fn kill_until_only_reserved( /// Read-only proof used after every pre-close process identity in a captured /// legacy PTY session has already disappeared. -pub fn session_is_empty(session: libc::pid_t) -> io::Result { +pub fn session_is_empty_until(session: libc::pid_t, deadline: Instant) -> io::Result { validate_session_target(session)?; - members(session).map(|members| members.is_empty()) + members_until(session, deadline).map(|members| members.is_empty()) } /// Snapshot process IDs in a non-caller session. Callers must capture exact /// process birth identities before using the result for later signaling. -pub fn session_member_pids(session: libc::pid_t) -> io::Result> { +pub fn session_member_pids_until( + session: libc::pid_t, + deadline: Instant, +) -> io::Result> { validate_session_target(session)?; - members(session) + members_until(session, deadline) } fn signal_members( members: &[libc::pid_t], session: libc::pid_t, reserved: Option, +) -> io::Result<()> { + signal_members_with(members, session, reserved, libc::SIGKILL) +} + +fn signal_members_with( + members: &[libc::pid_t], + session: libc::pid_t, + reserved: Option, + signal: libc::c_int, ) -> io::Result<()> { for pid in members.iter().copied().filter(|pid| Some(*pid) != reserved) { - signal_if_still_member(pid, session, libc::SIGKILL)?; + signal_if_still_member(pid, session, signal)?; } Ok(()) } @@ -169,18 +338,37 @@ fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) } } +#[cfg(test)] fn members(session: libc::pid_t) -> io::Result> { + let sessions = HashSet::from([session]); + Ok(scan_sessions(&sessions, None)?.remove(&session).unwrap_or_default()) +} + +fn members_until(session: libc::pid_t, deadline: Instant) -> io::Result> { + let sessions = HashSet::from([session]); + Ok(scan_sessions(&sessions, Some(deadline))?.remove(&session).unwrap_or_default()) +} + +fn scan_sessions( + sessions: &HashSet, + deadline: Option, +) -> io::Result>> { + let mut members = + sessions.iter().copied().map(|session| (session, Vec::new())).collect::>(); + if sessions.is_empty() { + return Ok(members); + } #[cfg(test)] PROCESS_TABLE_SCAN_COUNT.set(PROCESS_TABLE_SCAN_COUNT.get() + 1); - let mut members = Vec::new(); - for pid in all_process_ids()? { + for pid in all_process_ids(deadline)? { + ensure_before_deadline(deadline)?; if pid <= 1 { continue; } // SAFETY: getsid only queries process metadata. let current_session = unsafe { libc::getsid(pid) }; - if current_session == session { - members.push(pid); + if let Some(session_members) = members.get_mut(¤t_session) { + session_members.push(pid); continue; } if current_session < 0 { @@ -190,28 +378,31 @@ fn members(session: libc::pid_t) -> io::Result> { } } } - members.sort_unstable(); - members.dedup(); + for session_members in members.values_mut() { + session_members.sort_unstable(); + session_members.dedup(); + } Ok(members) } +fn ensure_before_deadline(deadline: Option) -> io::Result<()> { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Err(snapshot_deadline_error()); + } + Ok(()) +} + +fn snapshot_deadline_error() -> io::Error { + io::Error::new(io::ErrorKind::TimedOut, "process-session snapshot deadline expired") +} + #[cfg(test)] fn shutdown_batch_members_for_test( sessions: &[libc::pid_t], deadline: Instant, ) -> io::Result>> { - sessions - .iter() - .map(|session| { - if Instant::now() >= deadline { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "process-session snapshot deadline expired", - )); - } - members(*session) - }) - .collect() + let snapshot = SessionProcessSnapshot::capture(sessions.iter().copied(), deadline)?; + sessions.iter().map(|session| snapshot.members(*session).map(|(_, members)| members)).collect() } #[cfg(all(target_os = "macos", test))] @@ -220,12 +411,13 @@ fn macos_session_process_ids(session: libc::pid_t) -> io::Result io::Result> { +fn all_process_ids(deadline: Option) -> io::Result> { use std::ffi::c_void; use std::mem::size_of; // Callers retain this snapshot and poll its members directly. A new full // process-table scan happens only after that generation has drained. + ensure_before_deadline(deadline)?; // SAFETY: a null buffer asks libproc for the current process count. let count = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; if count < 0 { @@ -234,6 +426,7 @@ fn all_process_ids() -> io::Result> { let mut capacity = usize::try_from(count).map_err(|_| io::Error::other("invalid process count"))? + 32; for _ in 0..4 { + ensure_before_deadline(deadline)?; let mut pids = vec![0; capacity]; let bytes = capacity .checked_mul(size_of::()) @@ -258,9 +451,10 @@ fn all_process_ids() -> io::Result> { } #[cfg(target_os = "linux")] -fn all_process_ids() -> io::Result> { +fn all_process_ids(deadline: Option) -> io::Result> { let mut pids = Vec::new(); for entry in std::fs::read_dir("/proc")? { + ensure_before_deadline(deadline)?; let entry = match entry { Ok(entry) => entry, Err(error) if error.kind() == io::ErrorKind::NotFound => continue, @@ -276,7 +470,7 @@ fn all_process_ids() -> io::Result> { } #[cfg(not(any(target_os = "macos", target_os = "linux")))] -fn all_process_ids() -> io::Result> { +fn all_process_ids(_deadline: Option) -> io::Result> { Err(io::Error::new( io::ErrorKind::Unsupported, "PTY session enumeration is unavailable on this platform", @@ -350,4 +544,41 @@ mod tests { "one shutdown batch scanned the global process table more than once" ); } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn concurrent_shutdown_refreshes_share_one_new_snapshot_generation() { + let mut children = [spawn_session_child(), spawn_session_child()]; + let sessions = children + .iter() + .map(|child| libc::pid_t::try_from(child.id()).unwrap()) + .collect::>(); + let deadline = Instant::now() + Duration::from_secs(1); + let snapshot = std::sync::Arc::new( + SessionProcessSnapshot::capture(sessions.clone(), deadline).unwrap(), + ); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(sessions.len())); + + std::thread::scope(|scope| { + for session in &sessions { + let snapshot = snapshot.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + let generation = snapshot.members(*session).unwrap().0; + barrier.wait(); + snapshot.refresh_members(*session, generation, deadline).unwrap(); + }); + } + }); + let scan_count = snapshot.scan_count(); + + for child in &mut children { + child.kill().unwrap(); + child.wait().unwrap(); + } + assert_eq!( + scan_count, 2, + "concurrent refreshes did not coalesce onto one new process-table snapshot" + ); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index bd575d0a725f..a7eff5359508 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -645,6 +645,30 @@ impl LocalProcess { Self::Untracked(killer) => killer.lock().unwrap().kill().is_ok(), } } + + #[cfg(unix)] + fn session_id(&self) -> Option { + match self { + Self::Owned(process) => process.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()), + #[cfg(test)] + Self::Untracked(_) => None, + } + } + + #[cfg(unix)] + fn terminate_and_wait_from_snapshot( + &self, + deadline: Instant, + snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + ) -> bool { + match self { + Self::Owned(process) => snapshot.is_some_and(|snapshot| { + process.terminate_and_wait_from_snapshot(deadline, snapshot) + }), + #[cfg(test)] + Self::Untracked(killer) => killer.lock().unwrap().kill().is_ok(), + } + } } #[cfg(unix)] @@ -681,6 +705,35 @@ impl SurfaceShutdownOwner { } } + #[cfg(unix)] + pub(crate) fn local_process_session(&self) -> Option { + match &self.kind { + SurfaceShutdownOwnerKind::Local(process) => process.session_id(), + SurfaceShutdownOwnerKind::Hosted(_) | SurfaceShutdownOwnerKind::Browser(_) => None, + } + } + + #[cfg(unix)] + pub(crate) fn terminate_until_in_batch( + &self, + deadline: Instant, + process_snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + ) -> bool { + match &self.kind { + SurfaceShutdownOwnerKind::Local(process) => { + process.terminate_and_wait_from_snapshot(deadline, process_snapshot) + } + SurfaceShutdownOwnerKind::Hosted(owner) => { + crate::terminal_host_runtime::terminate_and_confirm_terminal_host_record( + &owner.record, + &owner.record_path, + deadline, + ) + } + SurfaceShutdownOwnerKind::Browser(owner) => owner.terminate_until(deadline), + } + } + pub(crate) fn hosted_identity(&self) -> Option<(&str, &str)> { match &self.kind { #[cfg(unix)] @@ -852,7 +905,12 @@ impl LocalPtyProcess { } #[cfg(unix)] - fn signal_terminal_process_session(&self, signal: libc::c_int) -> std::io::Result<()> { + fn signal_terminal_process_session( + &self, + signal: libc::c_int, + deadline: Instant, + snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + ) -> std::io::Result<()> { let _signal = self.child_signal_lock.lock().unwrap(); if self.child_reaped.load(Ordering::Acquire) { return Ok(()); @@ -860,11 +918,20 @@ impl LocalPtyProcess { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; - crate::process_session::signal(session, signal) + match snapshot { + Some(snapshot) => { + crate::process_session::signal_from_snapshot(snapshot, session, signal) + } + None => crate::process_session::signal_until(session, signal, deadline), + } } #[cfg(unix)] - fn kill_terminal_process_session_until(&self, deadline: Instant) -> std::io::Result { + fn kill_terminal_process_session_until( + &self, + deadline: Instant, + snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + ) -> std::io::Result { let _signal = self.child_signal_lock.lock().unwrap(); if self.child_reaped.load(Ordering::Acquire) { return Ok(true); @@ -872,12 +939,40 @@ impl LocalPtyProcess { let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; - crate::process_session::kill_until_only_leader(leader, leader, deadline, || { - self.child_waitable.load(Ordering::Acquire) - }) + match snapshot { + Some(snapshot) => crate::process_session::kill_until_only_leader_from_snapshot( + snapshot, + leader, + leader, + deadline, + || self.child_waitable.load(Ordering::Acquire), + ), + None => { + crate::process_session::kill_until_only_leader(leader, leader, deadline, || { + self.child_waitable.load(Ordering::Acquire) + }) + } + } } fn terminate_and_wait(&self, deadline: Instant) -> bool { + self.terminate_and_wait_inner(deadline, None) + } + + #[cfg(unix)] + fn terminate_and_wait_from_snapshot( + &self, + deadline: Instant, + snapshot: &crate::process_session::SessionProcessSnapshot, + ) -> bool { + self.terminate_and_wait_inner(deadline, Some(snapshot)) + } + + fn terminate_and_wait_inner( + &self, + deadline: Instant, + #[cfg(unix)] snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + ) -> bool { #[cfg(unix)] { { @@ -887,7 +982,7 @@ impl LocalPtyProcess { } self.termination_started.store(true, Ordering::Release); } - if self.signal_terminal_process_session(libc::SIGHUP).is_err() { + if self.signal_terminal_process_session(libc::SIGHUP, deadline, snapshot).is_err() { return false; } let _ = self.request_hangup(); @@ -898,7 +993,7 @@ impl LocalPtyProcess { // kill every member of the owned PTY session while its leader is // reserved with WNOWAIT, and do not acknowledge shutdown until no // background group remains. - if !matches!(self.kill_terminal_process_session_until(deadline), Ok(true)) { + if !matches!(self.kill_terminal_process_session_until(deadline, snapshot), Ok(true)) { return false; } self.group_escalation_complete.store(true, Ordering::Release); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 92f4754b6720..89218e7948e0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1988,7 +1988,11 @@ mod unix { self.pty_drained.load(Ordering::Acquire) } - fn signal_terminal_process_session(&self, signal: libc::c_int) -> std::io::Result<()> { + fn signal_terminal_process_session( + &self, + signal: libc::c_int, + deadline: Instant, + ) -> std::io::Result<()> { // The wait thread observes exit with WNOWAIT, then takes this lock // before reaping. Holding it reserves the session leader and its // numeric session id across enumeration and signaling. @@ -1999,7 +2003,7 @@ mod unix { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; - crate::process_session::signal(session, signal) + crate::process_session::signal_until(session, signal, deadline) } fn kill_terminal_process_session_until(&self, deadline: Instant) -> std::io::Result { @@ -2069,7 +2073,13 @@ mod unix { // ProcessSignaller only targets the direct child. Start with a // graceful session hangup so foreground and background jobs can // clean up too, then escalate after a strict bound. - if self.signal_terminal_process_session(libc::SIGHUP).is_err() { + if self + .signal_terminal_process_session( + libc::SIGHUP, + Instant::now() + HOST_TERMINATE_GRACE, + ) + .is_err() + { self.termination_started.store(false, Ordering::Release); return; } diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index fcdd7e4ce0d4..5c15be36f465 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1761,11 +1761,19 @@ mod tests { Err(io::Error::new(io::ErrorKind::Unsupported, "test stream is not cloneable")) } + fn read_timeout(&self) -> io::Result> { + Ok(self.read_timeouts.lock().unwrap().last().copied().flatten()) + } + fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { self.read_timeouts.lock().unwrap().push(timeout); Ok(()) } + fn write_timeout(&self) -> io::Result> { + Ok(None) + } + fn set_write_timeout(&self, _timeout: Option) -> io::Result<()> { Ok(()) } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 5b36008dfdc6..b30c456455a2 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -149,15 +149,39 @@ impl ServerProbe { } fn inspect_until(reader: &mut TransportReader, deadline: Instant) -> anyhow::Result { - set_transport_deadline(reader.get_mut().as_ref(), deadline)?; - write_json_line(reader.get_mut(), &json!({"id": PROBE_REQUEST_ID, "cmd": "identify"})) + let previous_read_timeout = reader + .get_ref() + .read_timeout() + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; + let previous_write_timeout = reader + .get_ref() + .write_timeout() .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - let response = read_response_until(reader, PROBE_REQUEST_ID, deadline)?; - if response.get("ok").and_then(Value::as_bool) != Some(true) { - anyhow::bail!(crate::localization::catalog().server.identity_failed); + let result = (|| { + set_transport_deadline(reader.get_mut().as_ref(), deadline)?; + write_json_line(reader.get_mut(), &json!({"id": PROBE_REQUEST_ID, "cmd": "identify"})) + .map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.transport_failed) + })?; + let response = read_response_until(reader, PROBE_REQUEST_ID, deadline)?; + if response.get("ok").and_then(Value::as_bool) != Some(true) { + anyhow::bail!(crate::localization::catalog().server.identity_failed); + } + let data = response.get("data").unwrap_or(&Value::Null); + Ok(Self { identity: ServerIdentity::from_protocol_data(data)? }) + })(); + let restored = reader + .get_ref() + .set_read_timeout(previous_read_timeout) + .and_then(|()| reader.get_ref().set_write_timeout(previous_write_timeout)) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed)); + match result { + Ok(probe) => restored.map(|()| probe), + Err(error) => { + let _ = restored; + Err(error) + } } - let data = response.get("data").unwrap_or(&Value::Null); - Ok(Self { identity: ServerIdentity::from_protocol_data(data)? }) } pub(crate) fn connect(path: &Path) -> anyhow::Result<(Self, TransportReader, Option)> { @@ -377,7 +401,7 @@ impl ServerLifecycle { .ok_or_else(|| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - let owner = capture_process_session(pid, expected) + let owner = capture_process_session(pid, expected, deadline) .map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })? diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index bf4c1132fea1..b41d1ebb4739 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -214,6 +214,7 @@ impl CapturedProcessSession { pub(super) fn capture_process_session( pid: libc::pid_t, server: ProcessIdentity, + deadline: Instant, ) -> io::Result> { if pid <= 1 || pid == server.pid @@ -248,7 +249,8 @@ pub(super) fn capture_process_session( )); } let mut members = Vec::new(); - for member in cmux_tui_core::process_session::session_member_pids(session)? { + for member in cmux_tui_core::process_session::session_member_pids_until(session, deadline)? + { if let Some(identity) = ProcessIdentity::capture(member)? { members.push(identity); } @@ -290,13 +292,16 @@ impl CapturedSession { Err(error) => error, } } - Ok(None) => match cmux_tui_core::process_session::session_is_empty(self.id) { - Ok(true) => return Ok(true), - Ok(false) => io::Error::other( - "captured PTY session lost every exact process before termination", - ), - Err(error) => error, - }, + Ok(None) => { + match cmux_tui_core::process_session::session_is_empty_until(self.id, deadline) + { + Ok(true) => return Ok(true), + Ok(false) => io::Error::other( + "captured PTY session lost every exact process before termination", + ), + Err(error) => error, + } + } Err(error) => error, }; if Instant::now() >= deadline { From 4be87f1c2219859d831052860ba6abe13b8d4876 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:37:54 -0700 Subject: [PATCH 050/521] test(tui): cover closed-peer probe cleanup --- cmux-tui/crates/cmux-tui/src/cli.rs | 32 ++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 5c15be36f465..1e37ecb7f3e1 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1727,6 +1727,7 @@ mod tests { current: io::Cursor>, writes: Vec, read_timeouts: Arc>>>, + fail_read_timeout_call: Option, } impl Read for ScriptedStream { @@ -1766,7 +1767,14 @@ mod tests { } fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { - self.read_timeouts.lock().unwrap().push(timeout); + let call = { + let mut timeouts = self.read_timeouts.lock().unwrap(); + timeouts.push(timeout); + timeouts.len() + }; + if self.fail_read_timeout_call == Some(call) { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } Ok(()) } @@ -1796,6 +1804,7 @@ mod tests { current: io::Cursor::new(Vec::new()), writes: Vec::new(), read_timeouts, + fail_read_timeout_call: None, }; let mut reader = BufReader::new(Box::new(stream) as Box); @@ -1817,6 +1826,7 @@ mod tests { current: io::Cursor::new(Vec::new()), writes: Vec::new(), read_timeouts, + fail_read_timeout_call: None, }; let mut reader = BufReader::new(Box::new(stream) as Box); @@ -1837,6 +1847,7 @@ mod tests { current: io::Cursor::new(Vec::new()), writes: Vec::new(), read_timeouts: read_timeouts.clone(), + fail_read_timeout_call: None, }; transport::Stream::set_read_timeout(&stream, Some(Duration::from_millis(250))).unwrap(); let mut reader = BufReader::new(Box::new(stream) as Box); @@ -1849,6 +1860,25 @@ mod tests { ); } + #[test] + fn completed_probe_survives_closed_peer_timeout_restore() { + let read_timeouts = Arc::new(Mutex::new(Vec::new())); + let stream = ScriptedStream { + reads: VecDeque::from([Ok( + b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",\"capabilities\":[]}}\n" + .to_vec(), + )]), + current: io::Cursor::new(Vec::new()), + writes: Vec::new(), + read_timeouts, + fail_read_timeout_call: Some(4), + }; + transport::Stream::set_read_timeout(&stream, Some(Duration::from_millis(250))).unwrap(); + let mut reader = BufReader::new(Box::new(stream) as Box); + + assert_eq!(server_supports_capability(&mut reader, "unused"), Ok(false)); + } + #[test] fn plugin_verb_is_registered_as_local_with_help() { let plugin = verb_by_name("plugin").expect("plugin verb registered"); From 42047d0bf65db9be82fd58c90caccace3b084553 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:40:35 -0700 Subject: [PATCH 051/521] fix(tui): preserve completed identity probes --- .../crates/cmux-tui/src/server_lifecycle.rs | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index b30c456455a2..f96882bf8c1b 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -173,10 +173,29 @@ impl ServerProbe { let restored = reader .get_ref() .set_read_timeout(previous_read_timeout) - .and_then(|()| reader.get_ref().set_write_timeout(previous_write_timeout)) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed)); + .and_then(|()| reader.get_ref().set_write_timeout(previous_write_timeout)); match result { - Ok(probe) => restored.map(|()| probe), + Ok(probe) => match restored { + Ok(()) => Ok(probe), + // A server may close immediately after its complete identity + // response. macOS then rejects timeout socket options with + // EINVAL even though the response remains authoritative. + Err(error) + if error.raw_os_error() == Some(libc::EINVAL) + || matches!( + error.kind(), + std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + ) => + { + Ok(probe) + } + Err(_) => { + Err(anyhow::anyhow!(crate::localization::catalog().server.transport_failed)) + } + }, Err(error) => { let _ = restored; Err(error) From 9e605f361293d12ffd61963fc985e7b7f5c1fca8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:47:23 -0700 Subject: [PATCH 052/521] test(tui): derive mismatch protocol expectation --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index f96882bf8c1b..4499787e5a52 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -890,7 +890,7 @@ mod tests { let message = incompatible_server_message(&identity, Path::new("/tmp/test socket")); - assert!(message.contains("server: v0.1.0-old protocol 8")); + assert!(message.contains(&format!("server: v0.1.0-old protocol {}", PROTOCOL_VERSION - 1))); assert!(message.contains("client: v")); assert!(message.contains("reason: distribution version differs")); assert!(message.contains("source build differs")); From a0cc01c05cd9c0a747e18423893cc3615a18b55c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:15:35 -0700 Subject: [PATCH 053/521] test(tui): cover lifecycle ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 21 +++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 40 +++++++++ .../src/terminal_host_runtime.rs | 3 + cmux-tui/crates/cmux-tui/tests/cli.rs | 89 +++++++++++++++++-- 4 files changed, 146 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 393ac712a0ec..8b1afabbf1bf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10447,6 +10447,27 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn ordinary_surface_close_reconciles_a_transient_termination_failure() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + let (failing, attempts) = owned.set_recovering_server_shutdown_for_test(); + + assert!(mux.close_surface(surface.id).unwrap()); + assert_eq!(mux.shutdown_owners.len(), 1); + assert_eq!(attempts.load(Ordering::Acquire), 1); + + failing.store(false, Ordering::Release); + let deadline = Instant::now() + Duration::from_secs(2); + while !mux.shutdown_owners.is_empty() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + + assert!(mux.shutdown_owners.is_empty(), "retained owner had no normal-lifecycle retry"); + assert!(attempts.load(Ordering::Acquire) >= 2); + } + #[test] fn ordinary_surface_close_does_not_retry_unrelated_retained_owners() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index a7eff5359508..161eca6e7248 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2008,6 +2008,23 @@ impl Surface { attempts } + #[cfg(test)] + pub(crate) fn set_recovering_server_shutdown_for_test( + &self, + ) -> (Arc, Arc) { + let failing = Arc::new(AtomicBool::new(true)); + let attempts = Arc::new(AtomicUsize::new(0)); + let Self::Pty(pty) = self else { return (failing, attempts) }; + let Some(process) = &pty.local_process else { return (failing, attempts) }; + let process = process.lock().unwrap(); + let LocalProcess::Untracked(killer) = &*process else { return (failing, attempts) }; + *killer.lock().unwrap() = Box::new(TestRecoveringChildKiller { + failing: failing.clone(), + attempts: attempts.clone(), + }); + (failing, attempts) + } + fn as_pty(&self) -> Option<&PtySurface> { match self { Surface::Pty(surface) => Some(surface), @@ -2841,6 +2858,29 @@ impl ChildKiller for TestCountingFailingChildKiller { } } +#[cfg(test)] +#[derive(Debug)] +struct TestRecoveringChildKiller { + failing: Arc, + attempts: Arc, +} + +#[cfg(test)] +impl ChildKiller for TestRecoveringChildKiller { + fn kill(&mut self) -> std::io::Result<()> { + self.attempts.fetch_add(1, Ordering::AcqRel); + if self.failing.load(Ordering::Acquire) { + Err(std::io::Error::other("forced transient shutdown failure")) + } else { + Ok(()) + } + } + + fn clone_killer(&self) -> Box { + Box::new(Self { failing: self.failing.clone(), attempts: self.attempts.clone() }) + } +} + impl PtySurface { /// Snapshot sparse colors and the host-resolved cursor visual without /// touching the shared renderer or consuming its damage. diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 89218e7948e0..808c9161c76a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2359,6 +2359,9 @@ mod unix { incarnation: bootstrapped.incarnation, }; let mut response = Frame::new(MessageKind::Ready, ready.encode()); + if std::env::var("CMUX_TUI_TEST_INVALID_HOST_READY").as_deref() == Ok("1") { + response.kind = MessageKind::Snapshot; + } response.request_id = launch_frame.request_id; // Publication is the ownership handoff. If the launcher dies in the // narrow record-before-Ready window, EPIPE must not tear down the diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b9080015d4e4..1380b2eff571 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -482,6 +482,7 @@ fn legacy_server_process_helper() { | "zombie-child" | "reparent-on-close" | "browser-surface" + | "dead-surface" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { ("zombie-child", _) => &[], @@ -494,6 +495,8 @@ fn legacy_server_process_helper() { .map(|surface| { if scenario == "browser-surface" { serde_json::json!({"surface": surface, "kind": "browser"}) + } else if scenario == "dead-surface" { + serde_json::json!({"surface": surface, "kind": "pty", "dead": true}) } else { serde_json::json!({"surface": surface, "kind": "pty"}) } @@ -527,13 +530,19 @@ fn legacy_server_process_helper() { let mut close_request: serde_json::Value = serde_json::from_str(&request).unwrap(); if close_request["cmd"].as_str() == Some("process-info") { assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); - let pid = fs::read_to_string( - std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), - ) - .unwrap() - .trim() - .parse::() - .unwrap(); + let pid = if scenario == "dead-surface" { + None + } else { + Some( + fs::read_to_string( + std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap(), + ) + .unwrap() + .trim() + .parse::() + .unwrap(), + ) + }; writeln!( stream, "{}", @@ -1317,6 +1326,72 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_closes_exited_legacy_placeholders_without_a_pid() { + let mut server = + LegacyServerProcess::start("legacy-server-dead-placeholder", "dead-surface", None); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); +} + +#[cfg(unix)] +#[test] +fn published_host_launch_errors_reap_the_complete_pty_session() { + let mut server = HeadlessServer::start_with( + "published-host-launch-error", + false, + &[("CMUX_TUI_TEST_INVALID_HOST_READY", "1"), ("CMUX_TUI_TEST_HOST_READY_DELAY_MS", "500")], + ); + let direct_pid_file = server.dir.join("failed-launch-direct.pid"); + let descendant_pid_file = server.dir.join("failed-launch-descendant.pid"); + let command = format!( + "echo $$ > {}; trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", + direct_pid_file.display(), + descendant_pid_file.display() + ); + + let output = cli(&server, &["run", "--new-workspace", "--command", &command]); + assert!(!output.status.success(), "injected invalid launch acknowledgement succeeded"); + let direct_pid = wait_for_pid_file(&direct_pid_file, Duration::from_secs(5)); + let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); + let deadline = Instant::now() + Duration::from_secs(5); + while (process_exists(direct_pid) + || process_group_exists(direct_pid) + || process_exists(descendant_pid) + || !terminal_host_pids(&host_root).is_empty()) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let cleaned = !process_exists(direct_pid) + && !process_group_exists(direct_pid) + && !process_exists(descendant_pid) + && terminal_host_pids(&host_root).is_empty(); + + let stop = cli(&server, &["--json", "server", "stop"]); + assert_success(&stop); + assert!(server.child.wait().unwrap().success()); + for pid in [direct_pid, descendant_pid] { + if let Ok(pid) = libc::pid_t::try_from(pid) { + // SAFETY: both PIDs came from this isolated test fixture. + let _ = unsafe { libc::kill(pid, libc::SIGKILL) }; + } + } + + assert!(cleaned, "post-publication launch error left PTY ownership unreconciled"); +} + #[cfg(unix)] #[test] fn legacy_stop_retains_pty_ownership_captured_before_surface_close() { From 4b34f7e8459387dcc4bde07f3b1822e0cdc4b75d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:19:51 -0700 Subject: [PATCH 054/521] fix(tui): reconcile lifecycle cleanup ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 123 ++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 27 ---- .../src/terminal_host_runtime.rs | 89 +++++++++++-- .../crates/cmux-tui/src/server_lifecycle.rs | 80 ++++++------ 4 files changed, 220 insertions(+), 99 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8b1afabbf1bf..235424df5f08 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -44,6 +44,8 @@ const PROVIDER_WORKSPACE_AUTHORITY_MIN_BYTES: usize = 32; const PROVIDER_WORKSPACE_AUTHORITY_MAX_BYTES: usize = 512; const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); const SHUTDOWN_FANOUT_WORKERS: usize = 32; +const SHUTDOWN_RECONCILE_INITIAL_DELAY: Duration = Duration::from_millis(25); +const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); #[derive(Clone, Debug, PartialEq, Eq, Hash)] enum ShutdownOwnerKey { @@ -126,6 +128,86 @@ impl ShutdownOwnerLedger { } } +#[derive(Default)] +struct ShutdownOwnerReconcilerState { + pending: bool, + stopping: bool, +} + +#[derive(Default)] +struct ShutdownOwnerReconciler { + state: Mutex, + wake: std::sync::Condvar, +} + +impl ShutdownOwnerReconciler { + fn start(self: &Arc, mux: Weak) -> anyhow::Result<()> { + let reconciler = self.clone(); + std::thread::Builder::new() + .name("cmux-shutdown-owner-reconciler".into()) + .spawn(move || reconciler.run(mux)) + .context("start shutdown owner reconciler")?; + Ok(()) + } + + fn schedule(&self) { + let mut state = self.state.lock().unwrap(); + if state.stopping { + return; + } + state.pending = true; + self.wake.notify_one(); + } + + fn stop(&self) { + let mut state = self.state.lock().unwrap(); + state.stopping = true; + self.wake.notify_all(); + } + + fn run(self: Arc, mux: Weak) { + let mut delay = SHUTDOWN_RECONCILE_INITIAL_DELAY; + loop { + let mut state = self.state.lock().unwrap(); + while !state.pending && !state.stopping { + state = self.wake.wait(state).unwrap(); + } + if state.stopping { + return; + } + state.pending = false; + drop(state); + + loop { + let Some(mux) = mux.upgrade() else { return }; + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let pending = match mux.lock_shutdown_coordinator_until(deadline) { + Ok(_coordinator) => mux.terminate_staged_shutdown_owners_until(deadline), + Err(_) => true, + }; + drop(mux); + if !pending { + delay = SHUTDOWN_RECONCILE_INITIAL_DELAY; + break; + } + + let state = self.state.lock().unwrap(); + let mut state = if state.pending { + state + } else { + self.wake.wait_timeout(state, delay).unwrap().0 + }; + if state.stopping { + return; + } + state.pending = false; + drop(state); + delay = delay.saturating_mul(2).min(SHUTDOWN_RECONCILE_MAX_DELAY); + } + } + } +} + /// An opaque per-mux credential provisioned by the external machine /// provider. Debug output is deliberately redacted. #[derive(PartialEq, Eq)] @@ -1001,6 +1083,7 @@ pub struct Mux { async_surface_creations: AsyncSurfaceCreationGate, shutdown_coordinator: Mutex<()>, shutdown_owners: ShutdownOwnerLedger, + shutdown_owner_reconciler: Arc, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -1233,6 +1316,7 @@ impl Mux { async_surface_creations: AsyncSurfaceCreationGate::default(), shutdown_coordinator: Mutex::new(()), shutdown_owners: ShutdownOwnerLedger::default(), + shutdown_owner_reconciler: Arc::new(ShutdownOwnerReconciler::default()), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -1278,6 +1362,7 @@ impl Mux { test_surface_runtime, session, }); + mux.shutdown_owner_reconciler.start(Arc::downgrade(&mux))?; #[cfg(unix)] mux.adopt_terminal_hosts()?; Ok(mux) @@ -3963,22 +4048,27 @@ impl Mux { .filter_map(|surface| self.shutdown_owners.stage_surface(&surface)) .collect::>(); let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; - let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; - self.terminate_shutdown_owners_until(owners, deadline); + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { + self.shutdown_owner_reconciler.schedule(); + return; + }; + if self.terminate_shutdown_owners_until(owners, deadline) { + self.shutdown_owner_reconciler.schedule(); + } } - fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) { + fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) -> bool { let owners = self.shutdown_owners.snapshot(); - self.terminate_shutdown_owners_until(owners, deadline); + self.terminate_shutdown_owners_until(owners, deadline) } fn terminate_shutdown_owners_until( &self, owners: Vec<(ShutdownOwnerKey, Arc)>, deadline: Instant, - ) { + ) -> bool { if owners.is_empty() { - return; + return false; } #[cfg(unix)] let process_snapshot = crate::process_session::SessionProcessSnapshot::capture( @@ -4001,6 +4091,7 @@ impl Mux { let confirmed = owners.into_iter().filter(|(key, _)| !failed.contains(key)).collect::>(); self.shutdown_owners.remove_confirmed(&confirmed); + !failed.is_empty() } pub fn list_agents( @@ -8088,6 +8179,7 @@ fn sidebar_retry_delay(failures: u32) -> Duration { impl Drop for Mux { fn drop(&mut self) { + self.shutdown_owner_reconciler.stop(); if let Ok(state) = self.state.get_mut() { for surface in state.surfaces.values() { surface.disconnect_for_daemon_shutdown(); @@ -10468,25 +10560,6 @@ mod tests { assert!(attempts.load(Ordering::Acquire) >= 2); } - #[test] - fn ordinary_surface_close_does_not_retry_unrelated_retained_owners() { - let mux = test_mux(); - let retained = mux.new_workspace(None, Some((80, 24))).unwrap(); - let retained_owner = mux.surface(retained.id).unwrap(); - let attempts = retained_owner.count_server_shutdown_attempts_for_test(); - - assert!(mux.close_surface(retained.id).unwrap()); - assert_eq!(attempts.load(Ordering::Acquire), 1); - - let current = mux.new_workspace(None, Some((80, 24))).unwrap(); - assert!(mux.close_surface(current.id).unwrap()); - assert_eq!( - attempts.load(Ordering::Acquire), - 1, - "an unrelated close retried retained shutdown ownership" - ); - } - #[test] fn ordinary_browser_close_terminates_the_owner_staged_during_removal() { fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 161eca6e7248..9b0991af20a2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1997,17 +1997,6 @@ impl Surface { *killer.lock().unwrap() = Box::new(TestSlowFailingChildKiller(delay)); } - #[cfg(test)] - pub(crate) fn count_server_shutdown_attempts_for_test(&self) -> Arc { - let attempts = Arc::new(AtomicUsize::new(0)); - let Self::Pty(pty) = self else { return attempts }; - let Some(process) = &pty.local_process else { return attempts }; - let process = process.lock().unwrap(); - let LocalProcess::Untracked(killer) = &*process else { return attempts }; - *killer.lock().unwrap() = Box::new(TestCountingFailingChildKiller(attempts.clone())); - attempts - } - #[cfg(test)] pub(crate) fn set_recovering_server_shutdown_for_test( &self, @@ -2842,22 +2831,6 @@ impl ChildKiller for TestSlowFailingChildKiller { } } -#[cfg(test)] -#[derive(Debug)] -struct TestCountingFailingChildKiller(Arc); - -#[cfg(test)] -impl ChildKiller for TestCountingFailingChildKiller { - fn kill(&mut self) -> std::io::Result<()> { - self.0.fetch_add(1, Ordering::AcqRel); - Err(std::io::Error::other("forced counted shutdown failure")) - } - - fn clone_killer(&self) -> Box { - Box::new(Self(self.0.clone())) - } -} - #[cfg(test)] #[derive(Debug)] struct TestRecoveringChildKiller { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 808c9161c76a..96ed4136ba8c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -350,6 +350,75 @@ mod unix { } } + struct PublishedHostRecovery { + process: Option, + record_path: PathBuf, + terminal_id: String, + incarnation: String, + owner_token: String, + host_pid: u32, + } + + impl PublishedHostRecovery { + fn new( + process: SpawnedHostProcess, + record_path: PathBuf, + terminal_id: String, + incarnation: HostIncarnation, + owner_token: CapabilityToken, + host_pid: u32, + ) -> Self { + Self { + process: Some(process), + record_path, + terminal_id, + incarnation: incarnation.to_hex(), + owner_token: encode_hex(owner_token.as_bytes()), + host_pid, + } + } + + fn commit(mut self) -> SpawnedHostProcess { + self.process.take().expect("published terminal-host process is present") + } + + fn matching_record(&self) -> Option { + let record: TerminalHostRecord = + serde_json::from_slice(&fs::read(&self.record_path).ok()?).ok()?; + validate_terminal_host_record(&self.record_path, &record).ok()?; + (record.terminal_id == self.terminal_id + && record.incarnation == self.incarnation + && record.owner_token == self.owner_token + && record.host_pid == self.host_pid) + .then_some(record) + } + + fn reconcile(&mut self) { + let Some(process) = self.process.take() else { return }; + let Some(record) = self.matching_record() else { + // Publication was not proven, so the exact spawned process + // remains the only safe cleanup target. + drop(process); + return; + }; + + // The authenticated durable record now owns PTY cleanup. Never + // exact-kill the host after this handoff because its child runs + // in a separate session and may ignore terminal hangup. + process.detach_reaper(); + if let Ok(attachment) = connect_record(record, self.record_path.clone()) { + let _ = attachment.terminate(); + attachment.disconnect(); + } + } + } + + impl Drop for PublishedHostRecovery { + fn drop(&mut self) { + self.reconcile(); + } + } + #[derive(Debug)] struct HostLaunch { endpoint: String, @@ -965,18 +1034,18 @@ mod unix { anyhow::bail!("terminal host changed terminal identity during bootstrap"); } + let launch_recovery = PublishedHostRecovery::new( + process, + record_path.clone(), + terminal_hex.clone(), + ready.incarnation, + owner_token, + host_pid, + ); let mut launch_frame = Frame::new(MessageKind::Launch, launch.encode()?); launch_frame.request_id = 2; write_frame(&mut stdin, &launch_frame)?; - let launched_frame = match read_required_frame(&mut stdout, "launch ready") { - Ok(frame) => frame, - Err(error) => { - if cancelled() && record_path.exists() { - process.detach_reaper(); - } - return Err(error); - } - }; + let launched_frame = read_required_frame(&mut stdout, "launch ready")?; if launched_frame.kind != MessageKind::Ready || launched_frame.request_id != 2 { anyhow::bail!("terminal host did not acknowledge launch"); } @@ -1003,7 +1072,7 @@ mod unix { // would leave a live published host while the mux marks its registry // row Exited. let mut attachment = connect_record(record, record_path)?; - attachment.launch_process = Some(process); + attachment.launch_process = Some(launch_recovery.commit()); if cancelled() { drop(attachment); anyhow::bail!("terminal host launch cancelled because the server is shutting down"); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 4499787e5a52..d135810f41d3 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -396,39 +396,41 @@ impl ServerLifecycle { if surface.kind != "pty" { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } - let process_request_id = take_legacy_request_id(next_request_id)?; - set_transport_deadline(self.reader.get_mut().as_ref(), deadline) - .map_err(|_| LegacyConnectionInterrupted)?; - write_json_line( - self.reader.get_mut(), - &json!({ - "id": process_request_id, - "cmd": "process-info", - "surface": surface.id, - }), - ) - .map_err(|_| LegacyConnectionInterrupted)?; - let process_response = - read_response_until(&mut self.reader, process_request_id, deadline) + if !surface.dead { + let process_request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline) .map_err(|_| LegacyConnectionInterrupted)?; - let process_data = response_data(&process_response)?; - let pid = process_data - .get("pid") - .and_then(Value::as_u64) - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .filter(|pid| *pid > 1) - .ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - let owner = capture_process_session(pid, expected, deadline) - .map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })? - .ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - if !owners.iter().any(|captured| captured.id() == owner.id()) { - owners.push(owner); + write_json_line( + self.reader.get_mut(), + &json!({ + "id": process_request_id, + "cmd": "process-info", + "surface": surface.id, + }), + ) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_response = + read_response_until(&mut self.reader, process_request_id, deadline) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_data = response_data(&process_response)?; + let pid = process_data + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .filter(|pid| *pid > 1) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let owner = capture_process_session(pid, expected, deadline) + .map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })? + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + if !owners.iter().any(|captured| captured.id() == owner.id()) { + owners.push(owner); + } } let request_id = take_legacy_request_id(next_request_id)?; @@ -767,11 +769,12 @@ fn response_data(response: &Value) -> anyhow::Result<&Value> { struct LegacySurface { id: u64, kind: String, + dead: bool, } #[cfg(unix)] fn legacy_surfaces(data: &Value) -> anyhow::Result> { - let mut surfaces = std::collections::HashMap::::new(); + let mut surfaces = std::collections::HashMap::::new(); for workspace in data.get("workspaces").and_then(Value::as_array).into_iter().flatten() { for screen in workspace.get("screens").and_then(Value::as_array).into_iter().flatten() { for pane in screen.get("panes").and_then(Value::as_array).into_iter().flatten() { @@ -782,8 +785,9 @@ fn legacy_surfaces(data: &Value) -> anyhow::Result> { crate::localization::catalog().server.legacy_cleanup_failed ) })?; - if let Some(previous) = surfaces.insert(surface, kind.to_string()) - && previous != kind + let dead = tab.get("dead").and_then(Value::as_bool).unwrap_or(false); + if let Some(previous) = surfaces.insert(surface, (kind.to_string(), dead)) + && previous != (kind.to_string(), dead) { anyhow::bail!( crate::localization::catalog().server.legacy_cleanup_failed @@ -794,8 +798,10 @@ fn legacy_surfaces(data: &Value) -> anyhow::Result> { } } } - let mut surfaces = - surfaces.into_iter().map(|(id, kind)| LegacySurface { id, kind }).collect::>(); + let mut surfaces = surfaces + .into_iter() + .map(|(id, (kind, dead))| LegacySurface { id, kind, dead }) + .collect::>(); surfaces.sort_by_key(|surface| surface.id); Ok(surfaces) } From a097e0c16983da935ff649ed1d12baff5fa2d1c9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:31:30 -0700 Subject: [PATCH 055/521] test(tui): cover blocked interactive shutdown --- cmux-tui/crates/cmux-tui/src/main.rs | 7 ++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 33 +++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 08eb7f43e2b3..d55221df4531 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1027,6 +1027,13 @@ fn run_server( } cmux_tui_core::server::serve(mux.clone(), Some(socket_path.clone()))?; + #[cfg(debug_assertions)] + if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { + loop { + std::thread::park(); + } + } + let machine_runtime = (config.machine_sidebar.enabled || !config.machines.is_empty()) .then(|| MachineRuntime::new(socket_path.clone(), config.machines.clone())); let result = if args.headless { diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 1380b2eff571..33a0c7e2a956 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1059,6 +1059,39 @@ fn server_status_and_stop_control_a_compatible_headless_session() { assert!(!server.socket.exists()); } +#[cfg(unix)] +#[test] +fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { + let dir = unique_temp_dir("server-stop-blocked-interactive-driver"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let state = dir.join("state"); + let mut server = PtyChild::start_with_env( + &["--socket", socket.to_str().unwrap(), "--state", state.to_str().unwrap()], + &[ + ("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER", std::ffi::OsStr::new("1")), + ("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS", std::ffi::OsStr::new("100")), + ], + ); + wait_for_socket_path(&socket); + + let response = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "shutdown"})); + assert_eq!(response, serde_json::json!({})); + + let deadline = Instant::now() + Duration::from_secs(3); + let status = loop { + if let Some(status) = server.child.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "server remained alive after acknowledging shutdown"); + std::thread::sleep(Duration::from_millis(10)); + }; + assert!(status.success(), "server exited with {status}"); + assert!(transport::connect(&socket).is_err()); + drop(server); + fs::remove_dir_all(dir).unwrap(); +} + #[cfg(unix)] #[test] fn server_stop_cancels_a_blocked_terminal_host_launch() { From e7fa83805a7557e5ec8f6648f96c8b09ad3ffcb4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:33:36 -0700 Subject: [PATCH 056/521] fix(tui): guarantee acknowledged server exit --- cmux-tui/crates/cmux-tui/src/main.rs | 87 +++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index d55221df4531..2e7805cdaf1d 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -55,6 +55,8 @@ static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); #[cfg(unix)] const MACHINE_PROVIDER_TOKEN_ENV: &str = "CMUX_MACHINE_PROVIDER_TOKEN"; const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORITY"; +const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(5); +const SERVER_SHUTDOWN_WATCH_POLL: std::time::Duration = std::time::Duration::from_millis(25); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -904,6 +906,82 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result>, + worker: Option>, +} + +impl ServerProcessShutdownGuard { + fn start(mux: &Arc, socket_path: PathBuf) -> io::Result { + let weak_mux = Arc::downgrade(mux); + let worker_socket_path = socket_path.clone(); + let (completion, completed) = std::sync::mpsc::channel(); + let worker = std::thread::Builder::new().name("cmux-server-exit-watchdog".into()).spawn( + move || { + loop { + match completed.recv_timeout(SERVER_SHUTDOWN_WATCH_POLL) { + Ok(()) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + } + let Some(mux) = weak_mux.upgrade() else { + return; + }; + if mux.shutdown_requested() { + break; + } + } + + if matches!( + completed.recv_timeout(server_shutdown_exit_grace()), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ) { + // A successful shutdown request drains every surface and + // attempts its confirmed response before setting + // shutdown_requested. If the interactive driver cannot + // return after that boundary, it owns no remaining work + // that can justify retaining the process or control socket. + cmux_tui_core::server::cleanup(&worker_socket_path); + std::process::exit(0); + } + }, + )?; + Ok(Self { socket_path, completion: Some(completion), worker: Some(worker) }) + } + + fn complete(mut self) { + self.finish(); + } + + fn finish(&mut self) { + cmux_tui_core::server::cleanup(&self.socket_path); + if let Some(completion) = self.completion.take() { + let _ = completion.send(()); + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +impl Drop for ServerProcessShutdownGuard { + fn drop(&mut self) { + self.finish(); + } +} + +fn server_shutdown_exit_grace() -> std::time::Duration { + #[cfg(debug_assertions)] + if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|milliseconds| (1..=5_000).contains(milliseconds)) + { + return std::time::Duration::from_millis(milliseconds); + } + SERVER_SHUTDOWN_EXIT_GRACE +} + fn run_server( args: Args, provider_workspace_authority: Option, @@ -1026,6 +1104,13 @@ fn run_server( eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } cmux_tui_core::server::serve(mux.clone(), Some(socket_path.clone()))?; + let server_process = match ServerProcessShutdownGuard::start(&mux, socket_path.clone()) { + Ok(server_process) => server_process, + Err(error) => { + cmux_tui_core::server::cleanup(&socket_path); + return Err(error.into()); + } + }; #[cfg(debug_assertions)] if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { @@ -1045,7 +1130,7 @@ fn run_server( }; drop(websocket_server); mux.shutdown(); - cmux_tui_core::server::cleanup(&socket_path); + server_process.complete(); result } From e19a74befafb8fc2ece306f8f44fd50a2ab15120 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:35:05 -0700 Subject: [PATCH 057/521] test(tui): accept background shutdown reconciliation --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 235424df5f08..19cf2f326587 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10524,7 +10524,7 @@ mod tests { } #[test] - fn ordinary_surface_close_retains_failed_process_ownership_for_shutdown() { + fn ordinary_surface_close_retains_failed_process_ownership_until_reconciled() { let mux = test_mux(); let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let owned = mux.surface(surface.id).unwrap(); @@ -10535,7 +10535,7 @@ mod tests { assert_eq!(mux.shutdown_owners.len(), 1); owned.set_server_shutdown_failure_for_test(false); - assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + mux.close_all_surfaces_for_shutdown().unwrap(); assert!(mux.shutdown_owners.is_empty()); } From 5ca4b1e6caedbc633e0044bacb206faf52d7ecde Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:37:28 -0700 Subject: [PATCH 058/521] test(tui): cover idle reconciler allocation --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 19cf2f326587..fff3dc289d9e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -138,6 +138,8 @@ struct ShutdownOwnerReconcilerState { struct ShutdownOwnerReconciler { state: Mutex, wake: std::sync::Condvar, + #[cfg(test)] + worker_started: AtomicBool, } impl ShutdownOwnerReconciler { @@ -147,9 +149,16 @@ impl ShutdownOwnerReconciler { .name("cmux-shutdown-owner-reconciler".into()) .spawn(move || reconciler.run(mux)) .context("start shutdown owner reconciler")?; + #[cfg(test)] + self.worker_started.store(true, Ordering::Release); Ok(()) } + #[cfg(test)] + fn worker_started(&self) -> bool { + self.worker_started.load(Ordering::Acquire) + } + fn schedule(&self) { let mut state = self.state.lock().unwrap(); if state.stopping { @@ -10523,6 +10532,17 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn ordinary_mux_lifecycle_does_not_start_a_reconciler_worker() { + let mux = test_mux(); + assert!(!mux.shutdown_owner_reconciler.worker_started()); + + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + assert!(mux.close_surface(surface.id).unwrap()); + + assert!(!mux.shutdown_owner_reconciler.worker_started()); + } + #[test] fn ordinary_surface_close_retains_failed_process_ownership_until_reconciled() { let mux = test_mux(); From 5854411efa8e4c8868a9c81ce04b718c6640865b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:38:43 -0700 Subject: [PATCH 059/521] fix(tui): start cleanup reconciliation lazily --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 41 +++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fff3dc289d9e..0faf0151e226 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -132,40 +132,51 @@ impl ShutdownOwnerLedger { struct ShutdownOwnerReconcilerState { pending: bool, stopping: bool, + worker_started: bool, } #[derive(Default)] struct ShutdownOwnerReconciler { state: Mutex, wake: std::sync::Condvar, - #[cfg(test)] - worker_started: AtomicBool, + mux: std::sync::OnceLock>, } impl ShutdownOwnerReconciler { - fn start(self: &Arc, mux: Weak) -> anyhow::Result<()> { - let reconciler = self.clone(); - std::thread::Builder::new() - .name("cmux-shutdown-owner-reconciler".into()) - .spawn(move || reconciler.run(mux)) - .context("start shutdown owner reconciler")?; - #[cfg(test)] - self.worker_started.store(true, Ordering::Release); - Ok(()) + fn bind(&self, mux: Weak) { + assert!(self.mux.set(mux).is_ok(), "shutdown owner reconciler was bound twice"); } #[cfg(test)] fn worker_started(&self) -> bool { - self.worker_started.load(Ordering::Acquire) + self.state.lock().unwrap().worker_started } - fn schedule(&self) { + fn schedule(self: &Arc) { let mut state = self.state.lock().unwrap(); if state.stopping { return; } state.pending = true; - self.wake.notify_one(); + if state.worker_started { + self.wake.notify_one(); + return; + } + state.worker_started = true; + drop(state); + + let mux = + self.mux.get().cloned().expect("shutdown owner reconciler must bind before scheduling"); + let reconciler = self.clone(); + if std::thread::Builder::new() + .name("cmux-shutdown-owner-reconciler".into()) + .spawn(move || reconciler.run(mux)) + .is_err() + { + // Ownership remains in the ledger for full shutdown. A later + // retirement will retry worker creation if process pressure eases. + self.state.lock().unwrap().worker_started = false; + } } fn stop(&self) { @@ -1371,7 +1382,7 @@ impl Mux { test_surface_runtime, session, }); - mux.shutdown_owner_reconciler.start(Arc::downgrade(&mux))?; + mux.shutdown_owner_reconciler.bind(Arc::downgrade(&mux)); #[cfg(unix)] mux.adopt_terminal_hosts()?; Ok(mux) From 7fd66208eada7eb8cde9a2556b17aeefc22813dd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:19:54 -0700 Subject: [PATCH 060/521] test(tui): cover bounded shutdown cleanup --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 50 +++++++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 49 +++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 10 +++ .../crates/cmux-tui/src/server_lifecycle.rs | 83 +++++++++++++++++++ 4 files changed, 192 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 1c237457e1e4..a35dc313d447 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -248,6 +248,8 @@ pub struct BrowserRuntime { stealth_user_agent: Option, routes: Mutex, closed: AtomicBool, + #[cfg(test)] + shutdown_reconnect_before_resolve: Mutex>>, } #[derive(Default)] @@ -429,6 +431,8 @@ impl BrowserRuntime { stealth_user_agent, routes: Mutex::new(Routes::default()), closed: AtomicBool::new(false), + #[cfg(test)] + shutdown_reconnect_before_resolve: Mutex::new(None), }); start_router(Arc::downgrade(&runtime), event_rx)?; runtime.client.set_discover_targets(true)?; @@ -570,6 +574,10 @@ impl BrowserRuntime { let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false; }; + #[cfg(test)] + if let Some(hook) = self.shutdown_reconnect_before_resolve.lock().unwrap().clone() { + hook(); + } let (event_tx, _event_rx) = sync_channel(CDP_EVENT_QUEUE_CAPACITY); let Ok(client) = CdpClient::connect_with_timeout(&self.web_socket_url, event_tx, remaining) else { @@ -3180,6 +3188,48 @@ mod tests { assert!(runtime_released, "closed external runtime remained permanently retained"); } + #[test] + fn server_shutdown_reconnect_does_not_resolve_endpoint_after_deadline_starts() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + ws.close(None).unwrap(); + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::External, + ) + .unwrap(); + server.join().unwrap(); + let closed_deadline = Instant::now() + Duration::from_secs(1); + while !runtime.is_closed() && Instant::now() < closed_deadline { + thread::yield_now(); + } + assert!(runtime.is_closed(), "initial CDP transport did not close"); + *runtime.shutdown_reconnect_before_resolve.lock().unwrap() = + Some(Arc::new(|| thread::sleep(Duration::from_millis(250)))); + let started = Instant::now(); + + let terminated = runtime.close_surface_for_shutdown( + "target-1", + "session-1", + Instant::now() + Duration::from_millis(50), + ); + + assert!(!terminated); + assert!( + started.elapsed() < Duration::from_millis(150), + "hostname resolution escaped the shutdown deadline: {:?}", + started.elapsed() + ); + } + #[test] fn server_shutdown_treats_a_missing_target_as_already_closed() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0faf0151e226..cd9fb5a8093d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1133,6 +1133,8 @@ pub struct Mux { terminal_create_after_workspace_reservation: Mutex>>, #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, + #[cfg(test)] + shutdown_owner_capacity: AtomicUsize, browser_runtime: Mutex>>, cell_pixels: Mutex<(u16, u16)>, default_colors: Mutex, @@ -1366,6 +1368,8 @@ impl Mux { terminal_create_after_workspace_reservation: Mutex::new(None), #[cfg(test)] browser_bootstrap_before_runtime: Mutex::new(None), + #[cfg(test)] + shutdown_owner_capacity: AtomicUsize::new(usize::MAX), browser_runtime: Mutex::new(None), cell_pixels: Mutex::new((8, 16)), default_colors: Mutex::new(DefaultColors::default()), @@ -3544,6 +3548,11 @@ impl Mux { *self.terminal_move_before_projection.lock().unwrap() = hook; } + #[cfg(test)] + fn set_shutdown_owner_capacity_for_test(&self, capacity: usize) { + self.shutdown_owner_capacity.store(capacity, Ordering::Release); + } + #[cfg(all(test, unix))] pub(crate) fn seed_running_terminal_for_test( self: &Arc, @@ -10591,6 +10600,46 @@ mod tests { assert!(attempts.load(Ordering::Acquire) >= 2); } + #[test] + fn retained_shutdown_owners_bound_future_surface_admission() { + let mux = test_mux(); + mux.set_shutdown_owner_capacity_for_test(1); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + owned.set_server_shutdown_failure_for_test(true); + + assert!(mux.close_surface(surface.id).unwrap()); + assert_eq!(mux.shutdown_owners.len(), 1); + + let error = mux.new_workspace(None, Some((80, 24))).unwrap_err(); + assert_eq!(error.to_string(), "surface_owner_capacity_exhausted"); + } + + #[test] + fn permanent_shutdown_owner_failure_stops_automatic_retry_sweeps() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + let (_failing, attempts) = owned.set_recovering_server_shutdown_for_test(); + + assert!(mux.close_surface(surface.id).unwrap()); + let first_deadline = Instant::now() + Duration::from_secs(1); + while attempts.load(Ordering::Acquire) < 4 && Instant::now() < first_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let settled = attempts.load(Ordering::Acquire); + assert!(settled >= 4, "reconciler did not exercise its retry budget"); + + std::thread::sleep(Duration::from_millis(250)); + + assert_eq!( + attempts.load(Ordering::Acquire), + settled, + "permanent cleanup failure kept an unbounded retry sweep alive" + ); + assert_eq!(mux.shutdown_owners.len(), 1); + } + #[test] fn ordinary_browser_close_terminates_the_owner_staged_during_removal() { fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index a5d45740a964..fc3b7f2cf73d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -5266,6 +5266,16 @@ mod tests { assert_eq!(PROTOCOL_VERSION, 10); } + #[test] + fn identify_exposes_shutdown_cleanup_health() { + let mux = test_mux(); + let identity = handle_command(&mux, 0, Command::Identify, &test_writer()).unwrap(); + + assert_eq!(identity["shutdown_cleanup"]["pending"].as_u64(), Some(0)); + assert_eq!(identity["shutdown_cleanup"]["retrying"].as_bool(), Some(false)); + assert_eq!(identity["shutdown_cleanup"]["degraded"].as_bool(), Some(false)); + } + #[test] fn local_shutdown_responds_before_requesting_mux_exit() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index d135810f41d3..bb18bb8493e6 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -973,6 +973,89 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn modern_shutdown_does_not_expose_unlocalized_server_details() { + let path = PathBuf::from("/tmp").join(format!( + "cmux-tui-shutdown-localization-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + let identify: Value = serde_json::from_str(&line).unwrap(); + write_json_line( + &mut stream, + &json!({ + "id": identify["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "version": "test", + "protocol": PROTOCOL_VERSION, + "pid": std::process::id(), + "capabilities": [SERVER_SHUTDOWN_CAPABILITY], + }, + }), + ) + .unwrap(); + line.clear(); + reader.read_line(&mut line).unwrap(); + let shutdown: Value = serde_json::from_str(&line).unwrap(); + write_json_line( + &mut stream, + &json!({ + "id": shutdown["id"], + "ok": false, + "error": "raw internal shutdown detail", + }), + ) + .unwrap(); + }); + + let error = ServerLifecycle::connect(path.clone()).unwrap().stop().unwrap_err(); + + server.join().unwrap(); + std::fs::remove_file(path).unwrap(); + assert_eq!(error.to_string(), crate::localization::catalog().server.shutdown_failed); + } + + #[cfg(unix)] + #[test] + fn legacy_helper_timeout_is_bounded_without_force_killing_the_helper() { + use std::io::BufRead as _; + + let mut helper = Command::new("/bin/sh") + .arg("-c") + .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut ready = String::new(); + BufReader::new(helper.stdout.take().unwrap()).read_line(&mut ready).unwrap(); + assert_eq!(ready, "ready\n"); + let started = Instant::now(); + + let error = wait_for_child_until(&mut helper, Instant::now() + Duration::from_millis(50)) + .unwrap_err(); + let elapsed = started.elapsed(); + let still_running = helper.try_wait().unwrap().is_none(); + let _ = helper.kill(); + let _ = helper.wait(); + + assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_cleanup_failed); + assert!( + elapsed < Duration::from_millis(250), + "helper wait exceeded its bound: {elapsed:?}" + ); + assert!(still_running, "parent force-killed the helper after its deadline"); + } + #[cfg(unix)] #[test] fn legacy_stop_terminates_only_the_identified_process() { From 4507f5530566dcc2855c19b99e4c78756bf1cefa Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:27:52 -0700 Subject: [PATCH 061/521] fix(tui): bound shutdown recovery lifecycle --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 33 +++- cmux-tui/crates/cmux-tui-core/src/browser.rs | 9 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 165 ++++++++++++------ cmux-tui/crates/cmux-tui-core/src/server.rs | 23 ++- cmux-tui/crates/cmux-tui/src/cli.rs | 19 ++ cmux-tui/crates/cmux-tui/src/localization.rs | 15 ++ .../crates/cmux-tui/src/server_lifecycle.rs | 119 ++++++++++++- .../src/server_lifecycle/legacy_process.rs | 19 ++ 8 files changed, 335 insertions(+), 67 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index af81db4fd03e..f6168f455c64 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::io::{Read, Write}; use std::mem::size_of; -use std::net::{TcpStream, ToSocketAddrs}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel}; use std::sync::{Arc, Mutex, Weak}; @@ -102,6 +102,8 @@ struct Inner { next_id: AtomicU64, closed: AtomicBool, timeout: Duration, + web_socket_url: Arc, + peer_addr: SocketAddr, #[cfg(test)] reader_stopped: Arc, } @@ -291,6 +293,31 @@ impl CdpClient { let addr = addrs.next().ok_or_else(|| { anyhow::anyhow!("no socket address for {}:{}", endpoint.host, endpoint.port) })?; + Self::connect_to_addr_with_timeout(web_socket_url, addr, events, timeout) + } + + pub fn reconnect_with_timeout( + &self, + events: SyncSender, + timeout: Duration, + ) -> anyhow::Result { + Self::connect_to_addr_with_timeout( + &self.inner.web_socket_url, + self.inner.peer_addr, + events, + timeout, + ) + } + + fn connect_to_addr_with_timeout( + web_socket_url: &str, + addr: SocketAddr, + events: SyncSender, + timeout: Duration, + ) -> anyhow::Result { + if timeout.is_zero() { + anyhow::bail!("CDP connection deadline expired"); + } let stream = TcpStream::connect_timeout(&addr, timeout)?; stream.set_nodelay(true)?; stream.set_read_timeout(Some(timeout))?; @@ -313,6 +340,8 @@ impl CdpClient { next_id: AtomicU64::new(1), closed: AtomicBool::new(false), timeout: Duration::from_secs(30), + web_socket_url: Arc::from(web_socket_url), + peer_addr: addr, #[cfg(test)] reader_stopped: Arc::new(AtomicBool::new(false)), }), @@ -1188,6 +1217,8 @@ mod tests { next_id: AtomicU64::new(1), closed: AtomicBool::new(false), timeout: Duration::from_secs(1), + web_socket_url: Arc::from("ws://127.0.0.1:1/devtools/browser/test"), + peer_addr: "127.0.0.1:1".parse().unwrap(), reader_stopped: Arc::new(AtomicBool::new(false)), }); handle_text( diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index a35dc313d447..7b9167b18cd2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -242,7 +242,6 @@ struct BrowserWorkerErrorState { pub struct BrowserRuntime { client: CdpClient, - web_socket_url: String, chrome: Option, source: BrowserSource, stealth_user_agent: Option, @@ -425,7 +424,6 @@ impl BrowserRuntime { }; let runtime = Arc::new(BrowserRuntime { client, - web_socket_url: web_socket_url.to_string(), chrome, source, stealth_user_agent, @@ -574,13 +572,8 @@ impl BrowserRuntime { let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false; }; - #[cfg(test)] - if let Some(hook) = self.shutdown_reconnect_before_resolve.lock().unwrap().clone() { - hook(); - } let (event_tx, _event_rx) = sync_channel(CDP_EVENT_QUEUE_CAPACITY); - let Ok(client) = CdpClient::connect_with_timeout(&self.web_socket_url, event_tx, remaining) - else { + let Ok(client) = self.client.reconnect_with_timeout(event_tx, remaining) else { return false; }; matches!( diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index cd9fb5a8093d..fd33fa81a1fa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -46,6 +46,11 @@ const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); const SHUTDOWN_FANOUT_WORKERS: usize = 32; const SHUTDOWN_RECONCILE_INITIAL_DELAY: Duration = Duration::from_millis(25); const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); +const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; +#[cfg(not(test))] +const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 8; +#[cfg(test)] +const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 3; #[derive(Clone, Debug, PartialEq, Eq, Hash)] enum ShutdownOwnerKey { @@ -133,6 +138,7 @@ struct ShutdownOwnerReconcilerState { pending: bool, stopping: bool, worker_started: bool, + degraded: bool, } #[derive(Default)] @@ -158,6 +164,7 @@ impl ShutdownOwnerReconciler { return; } state.pending = true; + state.degraded = false; if state.worker_started { self.wake.notify_one(); return; @@ -175,7 +182,9 @@ impl ShutdownOwnerReconciler { { // Ownership remains in the ledger for full shutdown. A later // retirement will retry worker creation if process pressure eases. - self.state.lock().unwrap().worker_started = false; + let mut state = self.state.lock().unwrap(); + state.worker_started = false; + state.degraded = true; } } @@ -198,6 +207,7 @@ impl ShutdownOwnerReconciler { state.pending = false; drop(state); + let mut attempts = 0; loop { let Some(mux) = mux.upgrade() else { return }; let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; @@ -210,6 +220,14 @@ impl ShutdownOwnerReconciler { delay = SHUTDOWN_RECONCILE_INITIAL_DELAY; break; } + attempts += 1; + if attempts >= SHUTDOWN_RECONCILE_MAX_ATTEMPTS { + let mut state = self.state.lock().unwrap(); + state.pending = false; + state.worker_started = false; + state.degraded = true; + return; + } let state = self.state.lock().unwrap(); let mut state = if state.pending { @@ -743,6 +761,13 @@ pub struct SidebarPluginStatus { pub retry_after: Option, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ShutdownCleanupHealth { + pub(crate) pending: usize, + pub(crate) retrying: bool, + pub(crate) degraded: bool, +} + #[derive(Debug, Default)] struct SidebarPluginRuntime { options: Option, @@ -754,6 +779,7 @@ struct SidebarPluginRuntime { enum BrowserSurfaceAttach { MissingPane, + Rejected, Attached(Option), } @@ -1672,7 +1698,7 @@ impl Mux { }), )?; if existing.is_none() { - insert_surface_checked(&mut state, placeholder.clone())?; + insert_surface_checked(self, &mut state, placeholder.clone())?; } match self.project_terminal_to_workspace_in_state( &mut state, @@ -1781,7 +1807,7 @@ impl Mux { } anyhow::bail!("terminal workspace disappeared during adoption"); }; - if let Err(error) = insert_surface_checked(&mut state, surface) { + if let Err(error) = insert_surface_checked(self, &mut state, surface) { drop(state); if let Ok((_, revision)) = commit_terminal_lifecycle( &mut registry, @@ -2024,6 +2050,16 @@ impl Mux { self.surface_creations.begin() } + #[cfg(not(test))] + fn shutdown_owner_capacity(&self) -> usize { + SHUTDOWN_OWNER_CAPACITY + } + + #[cfg(test)] + fn shutdown_owner_capacity(&self) -> usize { + self.shutdown_owner_capacity.load(Ordering::Acquire).min(SHUTDOWN_OWNER_CAPACITY) + } + fn next_active_at(&self) -> u64 { self.next_active_at.fetch_add(1, Ordering::Relaxed) } @@ -2769,7 +2805,7 @@ impl Mux { }; self.emit_terminal_registry_changed(®istry, ready_revision); if let Err(error) = - insert_surface_checked(&mut self.state.lock().unwrap(), surface.clone()) + insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone()) { if let Ok((_, revision)) = commit_terminal_lifecycle( &mut registry, @@ -2808,7 +2844,8 @@ impl Mux { return Err(error); } }; - if let Err(error) = insert_surface_checked(&mut self.state.lock().unwrap(), surface.clone()) + if let Err(error) = + insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone()) { self.pending_workspace_surfaces.lock().unwrap().remove(&id); surface.kill(); @@ -2840,7 +2877,7 @@ impl Mux { }; #[cfg(not(test))] let surface = Surface::spawn(id, opts, Arc::downgrade(self))?; - insert_surface_checked(&mut self.state.lock().unwrap(), surface.clone())?; + insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone())?; Ok(surface) } @@ -2849,7 +2886,7 @@ impl Mux { url: String, size: Option<(u16, u16)>, pending_workspace: Option, - ) -> Arc { + ) -> anyhow::Result> { let id = self.next_id(); if let Some(workspace) = pending_workspace { self.pending_workspace_surfaces.lock().unwrap().insert(id, workspace); @@ -2859,9 +2896,15 @@ impl Mux { let cell_pixels = *self.cell_pixels.lock().unwrap(); let surface = browser::new_surface(id, url.clone(), size, cell_pixels, &opts, Arc::downgrade(self)); - self.state.lock().unwrap().surfaces.insert(id, surface.clone()); + if let Err(error) = + insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone()) + { + self.pending_workspace_surfaces.lock().unwrap().remove(&id); + surface.kill(); + return Err(error); + } self.start_browser_bootstrap(surface.clone(), BrowserBootstrap::Create { url }, None); - surface + Ok(surface) } fn resolve_client_size( @@ -3594,7 +3637,7 @@ impl Mux { }, )?; let mut state = self.state.lock().unwrap(); - insert_surface_checked(&mut state, surface.clone())?; + insert_surface_checked(self, &mut state, surface.clone())?; let (placement, changed) = self.project_terminal_to_workspace_in_state(&mut state, terminal_id, workspace_key)?; anyhow::ensure!(changed, "seeded terminal did not change topology"); @@ -4161,6 +4204,16 @@ impl Mux { self.shutdown_requested.load(Ordering::Acquire) } + pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { + let pending = self.shutdown_owners.len(); + let state = self.shutdown_owner_reconciler.state.lock().unwrap(); + ShutdownCleanupHealth { + pending, + retrying: pending != 0 && state.worker_started && !state.degraded, + degraded: pending != 0 && (state.degraded || !state.worker_started), + } + } + /// Permanently fence new surface creation, tombstone every durable /// terminal in one transaction, detach the complete topology in one /// state mutation, and terminate each runtime before acknowledging the @@ -5472,7 +5525,7 @@ impl Mux { return self.create_browser_surface_in_workspace(workspace, url, size); } let workspace_key = Self::new_workspace_key()?; - let surface = self.spawn_browser_surface(url, size, None); + let surface = self.spawn_browser_surface(url, size, None)?; let (pane_id, pane) = self.make_pane(surface.id); let screen_id = self.next_id(); let ws_id = self.next_id(); @@ -5603,7 +5656,7 @@ impl Mux { return Ok(surface); }; - let surface = self.spawn_browser_surface(url, size, None); + let surface = self.spawn_browser_surface(url, size, None)?; let active_at = self.next_active_at(); let notifications = self.surface_notifications(); let attached = { @@ -5662,7 +5715,7 @@ impl Mux { if self.state.lock().unwrap().workspace_by_id(workspace).is_none() { anyhow::bail!("unknown workspace {workspace}"); } - let surface = self.spawn_browser_surface(url, size, Some(workspace)); + let surface = self.spawn_browser_surface(url, size, Some(workspace))?; let pending_surface = self.pending_workspace_surface(surface.id); let notifications = self.surface_notifications(); let active_at = self.next_active_at(); @@ -5779,7 +5832,7 @@ impl Mux { browser::new_surface(id, url.clone(), size, cell_pixels, &opts, Arc::downgrade(self)); let active_at = self.next_active_at(); match self.attach_browser_surface_to_pane_or_kill(pane_id, &surface, active_at) { - BrowserSurfaceAttach::MissingPane => return false, + BrowserSurfaceAttach::MissingPane | BrowserSurfaceAttach::Rejected => return false, BrowserSurfaceAttach::Attached(Some(delta)) => self.emit_tree_delta(delta, true), BrowserSurfaceAttach::Attached(None) => self.emit(MuxEvent::TreeChanged), } @@ -5800,39 +5853,40 @@ impl Mux { let notifications = self.surface_notifications(); let attached = { let mut state = self.state.lock().unwrap(); - match state.panes.get_mut(&pane_id) { - Some(pane) => { - pane.tabs.push(surface.id); - pane.active_tab = pane.tabs.len() - 1; - pane.active_at = active_at; - state.surfaces.insert(surface.id, surface.clone()); - let delta = (|| { - let (wi, si) = state.screen_of(pane_id)?; - let pane = state.panes.get(&pane_id)?; - let index = pane.tabs.iter().position(|id| *id == surface.id)?; - let entity = crate::server::tree_entity_json( - &state, - ¬ifications, - TreeDeltaKind::TabAdded, - surface.id, - )?; - Some(TreeDelta { - kind: TreeDeltaKind::TabAdded, - workspace: state.workspaces[wi].id, - screen: Some(state.workspaces[wi].screens[si].id), - pane: Some(pane_id), - surface: Some(surface.id), - index: Some(index), - entity, - workspace_revision: None, - }) - })(); - BrowserSurfaceAttach::Attached(delta) - } - None => BrowserSurfaceAttach::MissingPane, + if !state.panes.contains_key(&pane_id) { + BrowserSurfaceAttach::MissingPane + } else if insert_surface_checked(self, &mut state, surface.clone()).is_err() { + BrowserSurfaceAttach::Rejected + } else { + let pane = state.panes.get_mut(&pane_id).expect("pane existence was checked"); + pane.tabs.push(surface.id); + pane.active_tab = pane.tabs.len() - 1; + pane.active_at = active_at; + let delta = (|| { + let (wi, si) = state.screen_of(pane_id)?; + let pane = state.panes.get(&pane_id)?; + let index = pane.tabs.iter().position(|id| *id == surface.id)?; + let entity = crate::server::tree_entity_json( + &state, + ¬ifications, + TreeDeltaKind::TabAdded, + surface.id, + )?; + Some(TreeDelta { + kind: TreeDeltaKind::TabAdded, + workspace: state.workspaces[wi].id, + screen: Some(state.workspaces[wi].screens[si].id), + pane: Some(pane_id), + surface: Some(surface.id), + index: Some(index), + entity, + workspace_revision: None, + }) + })(); + BrowserSurfaceAttach::Attached(delta) } }; - if matches!(attached, BrowserSurfaceAttach::MissingPane) { + if matches!(attached, BrowserSurfaceAttach::MissingPane | BrowserSurfaceAttach::Rejected) { surface.kill(); } attached @@ -8151,7 +8205,11 @@ fn bounded_shutdown_fanout( next.load(Ordering::Relaxed).min(items.len()) } -fn insert_surface_checked(state: &mut State, surface: Arc) -> anyhow::Result<()> { +fn insert_surface_checked( + mux: &Mux, + state: &mut State, + surface: Arc, +) -> anyhow::Result<()> { if state.surfaces.contains_key(&surface.id) { anyhow::bail!("duplicate_surface_id"); } @@ -8166,6 +8224,11 @@ fn insert_surface_checked(state: &mut State, surface: Arc) -> anyhow::R { anyhow::bail!("duplicate_terminal_id"); } + if state.surfaces.len().saturating_add(mux.shutdown_owners.len()) + >= mux.shutdown_owner_capacity() + { + anyhow::bail!("surface_owner_capacity_exhausted"); + } state.surfaces.insert(surface.id, surface); Ok(()) } @@ -8711,7 +8774,7 @@ mod tests { .unwrap(); let _registry = mux.workspace_registry.lock().unwrap(); let mut state = mux.state.lock().unwrap(); - insert_surface_checked(&mut state, surface.clone()).unwrap(); + insert_surface_checked(mux, &mut state, surface.clone()).unwrap(); let (placement, changed) = mux .project_terminal_to_workspace_in_state(&mut state, terminal_id, workspace_key) .unwrap(); @@ -10638,6 +10701,10 @@ mod tests { "permanent cleanup failure kept an unbounded retry sweep alive" ); assert_eq!(mux.shutdown_owners.len(), 1); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 1, retrying: false, degraded: true } + ); } #[test] @@ -10701,7 +10768,7 @@ mod tests { "target-1", "session-1", ); - insert_surface_checked(&mut mux.state.lock().unwrap(), surface.clone()).unwrap(); + insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface.clone()).unwrap(); let removed = take_surface_for_retirement(&mux, &mut mux.state.lock().unwrap(), surface.id).unwrap(); @@ -12059,7 +12126,7 @@ mod tests { TerminalHostIdentity { terminal_id: TERMINAL.into(), incarnation: INCARNATION.into() }, ) .unwrap(); - insert_surface_checked(&mut mux.state.lock().unwrap(), surface.clone()).unwrap(); + insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface.clone()).unwrap(); let (placement, canonical_workspace, changed) = mux.bind_running_terminal_to_canonical_workspace(&surface).unwrap(); assert!(changed); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index fc3b7f2cf73d..37fc64b1601a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -58,6 +58,7 @@ use crate::{ const ATTACH_INITIAL_SIZE_CAPABILITY: &str = "attach-initial-size"; const WORKSPACE_REGISTRY_CAPABILITY: &str = "workspace-registry-v1"; pub const SERVER_SHUTDOWN_CAPABILITY: &str = "server-shutdown-v1"; +pub const SERVER_SHUTDOWN_INCOMPLETE_ERROR: &str = "shutdown_cleanup_incomplete"; /// Maximum wall-clock time the server spends draining a shutdown request. /// Clients add their own transport margin to this server-owned bound. pub const SERVER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); @@ -3109,6 +3110,7 @@ fn handle_command( Command::Identify => { let release = ReleaseIdentity::current(PROTOCOL_VERSION); let (registry_id, generation) = mux.registry_identity(); + let shutdown_cleanup = mux.shutdown_cleanup_health(); Ok(json!({ "app": "cmux-tui", "version": release.version, @@ -3123,6 +3125,11 @@ fn handle_command( "workspace_revision": mux.with_state(|state| state.workspace_revision), "terminal_revision": mux.terminal_registry_snapshot()?.revision, "daemon_handoff": 1, + "shutdown_cleanup": { + "pending": shutdown_cleanup.pending, + "retrying": shutdown_cleanup.retrying, + "degraded": shutdown_cleanup.degraded, + }, })) } Command::Ping => { @@ -3139,7 +3146,8 @@ fn handle_command( if !mux.control_clients.is_unix(client) { anyhow::bail!("shutdown is only available over the local session socket"); } - mux.close_all_surfaces_for_shutdown()?; + mux.close_all_surfaces_for_shutdown() + .map_err(|_| anyhow::anyhow!(SERVER_SHUTDOWN_INCOMPLETE_ERROR))?; Ok(json!({})) } Command::ShutdownDaemon { pid, generation } => { @@ -5398,6 +5406,19 @@ mod tests { assert!(!mux.shutdown_requested()); } + #[test] + fn local_shutdown_returns_a_stable_cleanup_error_code() { + let mux = test_mux(); + let surface = mux.new_workspace(None, None).unwrap(); + surface.set_server_shutdown_failure_for_test(true); + let writer = test_writer(); + let client = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + + let error = handle_command(&mux, client, Command::Shutdown, &writer).unwrap_err(); + + assert_eq!(error.to_string(), SERVER_SHUTDOWN_INCOMPLETE_ERROR); + } + #[test] fn split_ids_serialize_stably_and_both_ratio_commands_work() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 54405f9cbe92..d5a85cde02a3 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -739,6 +739,11 @@ fn print_server_status(json_output: bool, probe: &crate::server_lifecycle::Serve "server": { "version": server.release.version, "protocol": server.release.protocol, + "shutdown_cleanup": { + "pending": server.shutdown_cleanup.pending, + "retrying": server.shutdown_cleanup.retrying, + "degraded": server.shutdown_cleanup.degraded, + }, }, "client": { "version": client.version, @@ -781,6 +786,20 @@ fn print_server_status(json_output: bool, probe: &crate::server_lifecycle::Serve .join(messages.reason_separator); println!("{}: {}", messages.reason_label, reasons); } + if server.shutdown_cleanup.pending != 0 { + let state = if server.shutdown_cleanup.degraded { + messages.cleanup_degraded + } else { + messages.cleanup_retrying + }; + println!( + "{}: {} ({}: {})", + messages.cleanup_label, + state, + messages.cleanup_pending_label, + server.shutdown_cleanup.pending, + ); + } 0 } } diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 253946ddca28..f65a6731f02d 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -66,8 +66,13 @@ pub(crate) struct ServerMessages { #[cfg(unix)] pub legacy_signal_failed: &'static str, pub shutdown_failed: &'static str, + pub shutdown_cleanup_incomplete: &'static str, pub shutdown_unsupported: &'static str, pub shutdown_timed_out: &'static str, + pub cleanup_label: &'static str, + pub cleanup_pending_label: &'static str, + pub cleanup_retrying: &'static str, + pub cleanup_degraded: &'static str, } #[derive(Debug, PartialEq, Eq)] @@ -295,8 +300,13 @@ static ENGLISH: Catalog = Catalog { #[cfg(unix)] legacy_signal_failed: "The verified older server process could not be stopped.", shutdown_failed: "the server could not stop cleanly", + shutdown_cleanup_incomplete: "the server could not finish pane cleanup before the shutdown deadline", shutdown_unsupported: "this server cannot be stopped by this client", shutdown_timed_out: "the server did not stop within 10 seconds", + cleanup_label: "cleanup", + cleanup_pending_label: "pending panes", + cleanup_retrying: "retrying", + cleanup_degraded: "run server stop again", }, menu: MenuMessages { maximize_pane: "Maximize pane", @@ -446,8 +456,13 @@ static JAPANESE: Catalog = Catalog { #[cfg(unix)] legacy_signal_failed: "検証済みの古いサーバープロセスを停止できませんでした。", shutdown_failed: "サーバーを正常に停止できませんでした", + shutdown_cleanup_incomplete: "シャットダウン期限までにペインのクリーンアップを完了できませんでした", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", + cleanup_label: "クリーンアップ", + cleanup_pending_label: "保留中のペイン", + cleanup_retrying: "再試行中", + cleanup_degraded: "server stop をもう一度実行してください", }, menu: MenuMessages { maximize_pane: "ペインを最大化", diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index bb18bb8493e6..de55f241e39f 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -4,12 +4,15 @@ use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; #[cfg(unix)] use std::process::{Command, Stdio}; +#[cfg(unix)] +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; use cmux_tui_core::release::ReleaseIdentity; use cmux_tui_core::server::{ - PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, SERVER_SHUTDOWN_TIMEOUT, + PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, SERVER_SHUTDOWN_INCOMPLETE_ERROR, + SERVER_SHUTDOWN_TIMEOUT, }; use serde_json::{Value, json}; @@ -41,11 +44,41 @@ const LEGACY_SHUTDOWN_TIMEOUT: Duration = SHUTDOWN_RESPONSE_TIMEOUT; const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); #[cfg(unix)] const LEGACY_HELPER_POLL_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(unix)] +const LEGACY_HELPER_CANCEL_MARGIN: Duration = Duration::from_millis(100); const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; type TransportReader = BufReader>; +#[cfg(unix)] +static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); + +#[cfg(unix)] +extern "C" fn handle_legacy_helper_signal(_: libc::c_int) { + LEGACY_HELPER_CANCELLED.store(true, Ordering::Release); +} + +#[cfg(unix)] +fn install_legacy_helper_signal_handler() { + unsafe { + libc::signal(libc::SIGTERM, handle_legacy_helper_signal as *const () as libc::sighandler_t); + } +} + +#[cfg(unix)] +fn legacy_helper_cancelled() -> bool { + LEGACY_HELPER_CANCELLED.load(Ordering::Acquire) +} + +#[cfg(unix)] +fn ensure_legacy_helper_active() -> anyhow::Result<()> { + if legacy_helper_cancelled() { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + Ok(()) +} + #[cfg(unix)] #[derive(Debug)] struct LegacyConnectionInterrupted; @@ -110,6 +143,14 @@ pub(crate) struct ServerIdentity { pub release: ReleaseIdentity, pub pid: u32, capabilities: HashSet, + pub(crate) shutdown_cleanup: ShutdownCleanupStatus, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct ShutdownCleanupStatus { + pub(crate) pending: u64, + pub(crate) retrying: bool, + pub(crate) degraded: bool, } impl ServerIdentity { @@ -130,7 +171,27 @@ impl ServerIdentity { .filter_map(Value::as_str) .map(str::to_string) .collect(); - Ok(Self { release: ReleaseIdentity::from_protocol_data(data), pid, capabilities }) + let cleanup = data.get("shutdown_cleanup"); + let shutdown_cleanup = ShutdownCleanupStatus { + pending: cleanup + .and_then(|cleanup| cleanup.get("pending")) + .and_then(Value::as_u64) + .unwrap_or(0), + retrying: cleanup + .and_then(|cleanup| cleanup.get("retrying")) + .and_then(Value::as_bool) + .unwrap_or(false), + degraded: cleanup + .and_then(|cleanup| cleanup.get("degraded")) + .and_then(Value::as_bool) + .unwrap_or(false), + }; + Ok(Self { + release: ReleaseIdentity::from_protocol_data(data), + pid, + capabilities, + shutdown_cleanup, + }) } pub(crate) fn supports(&self, capability: &str) -> bool { @@ -292,10 +353,7 @@ impl ServerLifecycle { if response.get("ok").and_then(Value::as_bool) == Some(true) { return wait_for_disconnect(&mut self.reader, &self.path); } - if let Some(error) = response.get("error").and_then(Value::as_str) { - anyhow::bail!("{}: {error}", crate::localization::catalog().server.shutdown_failed); - } - anyhow::bail!(crate::localization::catalog().server.shutdown_failed) + anyhow::bail!(localized_shutdown_error(response.get("error").and_then(Value::as_str))) } #[cfg(unix)] @@ -321,6 +379,7 @@ impl ServerLifecycle { let mut consecutive_empty_scans = 0; let mut owners = Vec::::new(); for _ in 0..LEGACY_MAX_SCAN_ROUNDS { + ensure_legacy_helper_active()?; if Instant::now() >= deadline { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } @@ -393,6 +452,7 @@ impl ServerLifecycle { let surfaces = legacy_surfaces(data)?; for surface in &surfaces { + ensure_legacy_helper_active()?; if surface.kind != "pty" { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } @@ -453,6 +513,14 @@ impl ServerLifecycle { } } +fn localized_shutdown_error(error: Option<&str>) -> &'static str { + let messages = &crate::localization::catalog().server; + match error { + Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR) => messages.shutdown_cleanup_incomplete, + Some(_) | None => messages.shutdown_failed, + } +} + #[cfg(unix)] fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { let request_id = *next_request_id; @@ -510,8 +578,29 @@ fn wait_for_child_until( return Ok(status); } if Instant::now() >= deadline { - let _ = child.kill(); - let _ = child.wait(); + let pid = libc::pid_t::try_from(child.id()).ok(); + if let Some(pid) = pid { + // The helper installs a cooperative handler so Rust cleanup + // can thaw any exact process identities currently fenced. + let _ = unsafe { libc::kill(pid, libc::SIGTERM) }; + } + let cancel_deadline = Instant::now() + LEGACY_HELPER_CANCEL_MARGIN; + while Instant::now() < cancel_deadline { + if child + .try_wait() + .map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })? + .is_some() + { + break; + } + std::thread::sleep( + cancel_deadline + .saturating_duration_since(Instant::now()) + .min(LEGACY_HELPER_POLL_INTERVAL), + ); + } anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } std::thread::sleep( @@ -522,6 +611,7 @@ fn wait_for_child_until( #[cfg(unix)] pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { + install_legacy_helper_signal_handler(); let [path, expected_pid, expected_started_at, timeout_ms] = args else { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); }; @@ -541,6 +631,7 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) })?; let deadline = Instant::now() + timeout; + ensure_legacy_helper_active()?; let expected = ProcessIdentity::from_parts(expected_pid, expected_started_at); let mut lifecycle = ServerLifecycle::connect_until(PathBuf::from(path), deadline)?; if lifecycle.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { @@ -552,14 +643,17 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { } let captured = capture_legacy_process_tree(actual, deadline)?; + ensure_legacy_helper_active()?; let owners = lifecycle.close_legacy_surfaces_until_stable(actual, deadline).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; for owner in owners { + ensure_legacy_helper_active()?; owner.terminate_until(deadline).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; } + ensure_legacy_helper_active()?; terminate_captured_legacy_process_tree(captured, deadline)?; wait_for_disconnect_until(&mut lifecycle.reader, &lifecycle.path, deadline) } @@ -892,6 +986,7 @@ mod tests { }, pid: 42, capabilities: HashSet::new(), + shutdown_cleanup: ShutdownCleanupStatus::default(), }; let message = incompatible_server_message(&identity, Path::new("/tmp/test socket")); @@ -1024,6 +1119,14 @@ mod tests { assert_eq!(error.to_string(), crate::localization::catalog().server.shutdown_failed); } + #[test] + fn modern_shutdown_localizes_stable_cleanup_error_code() { + assert_eq!( + localized_shutdown_error(Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR)), + crate::localization::catalog().server.shutdown_cleanup_incomplete + ); + } + #[cfg(unix)] #[test] fn legacy_helper_timeout_is_bounded_without_force_killing_the_helper() { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index b41d1ebb4739..f719992a496a 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -7,6 +7,16 @@ const PROCESS_TREE_RETRY_INTERVAL: Duration = Duration::from_millis(10); #[cfg(test)] const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); +fn ensure_helper_active() -> io::Result<()> { + if super::legacy_helper_cancelled() { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "legacy cleanup helper was cancelled", + )); + } + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct ProcessIdentity { pid: libc::pid_t, @@ -111,6 +121,7 @@ fn retry_process_tree_termination( mut attempt: impl FnMut(Instant) -> io::Result<()>, ) -> io::Result<()> { loop { + ensure_helper_active()?; let error = match attempt(deadline) { Ok(()) => return Ok(()), Err(error) => error, @@ -149,6 +160,7 @@ fn freeze_process_tree( deadline: Instant, ) -> io::Result { loop { + ensure_helper_active()?; let error = match FrozenProcessTree::freeze(process, deadline) { Ok(tree) => return Ok(tree), Err(error) => error, @@ -227,6 +239,7 @@ pub(super) fn capture_process_session( return Ok(None); } let captured = (|| { + ensure_helper_active()?; if ProcessIdentity::capture(pid)? != Some(process) { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -277,6 +290,7 @@ struct CapturedSession { impl CapturedSession { fn kill_until_empty(&self, deadline: Instant) -> io::Result { loop { + ensure_helper_active()?; let error = match SessionProbe::acquire(self) { Ok(Some(mut probe)) => { match cmux_tui_core::process_session::kill_until_only_reserved( @@ -323,6 +337,7 @@ struct SessionProbe { impl SessionProbe { fn acquire(session: &CapturedSession) -> io::Result> { for process in &session.members { + ensure_helper_active()?; if ProcessIdentity::capture(process.pid)? != Some(*process) { continue; } @@ -358,6 +373,7 @@ struct FrozenProcessTree { impl FrozenProcessTree { fn freeze(root: ProcessIdentity, deadline: Instant) -> io::Result { + ensure_helper_active()?; let mut tree = Self { root, descendants: Vec::new(), armed: true }; if root.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { return Err(io::Error::new( @@ -376,6 +392,7 @@ impl FrozenProcessTree { .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid helper process id"))?; let mut known = HashSet::from([root.pid, helper_pid]); for _ in 0..PROCESS_TREE_MAX_ROUNDS { + ensure_helper_active()?; if Instant::now() >= deadline { return Err(io::Error::new( io::ErrorKind::TimedOut, @@ -386,7 +403,9 @@ impl FrozenProcessTree { std::iter::once(root).chain(tree.descendants.iter().copied()).collect::>(); let mut added = false; for parent in parents { + ensure_helper_active()?; for pid in direct_child_pids(parent.pid)? { + ensure_helper_active()?; if known.contains(&pid) { continue; } From 36ffb98dc44e38de410b62407076b21d2f4c6619 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:15:17 -0700 Subject: [PATCH 062/521] test(tui): cover shutdown ownership races --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 45 ++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 224 ++++++++++++++++++++- 2 files changed, 266 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index f6168f455c64..919fbfedde76 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1099,7 +1099,7 @@ impl WsEndpoint { #[cfg(test)] mod tests { - use std::io::Write; + use std::io::{Read, Write}; use std::net::TcpListener; use std::sync::mpsc::sync_channel; use std::sync::{Arc, Barrier}; @@ -1301,6 +1301,49 @@ mod tests { server.join().unwrap(); } + #[test] + fn websocket_handshake_uses_one_absolute_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + if stream.read(&mut byte).unwrap_or(0) == 0 { + return; + } + request.push(byte[0]); + } + for byte in b"HTTP/1.1 400 Bad Request\r\n\r\n" { + if stream.write_all(&[*byte]).is_err() { + break; + } + thread::sleep(Duration::from_millis(10)); + } + }); + let (events, _receiver) = sync_channel(1); + + let started = Instant::now(); + let error = match CdpClient::connect_to_addr_with_timeout( + &format!("ws://{addr}/devtools/browser/fake"), + addr, + events, + Duration::from_millis(50), + ) { + Ok(_) => panic!("invalid WebSocket handshake unexpectedly succeeded"), + Err(error) => error, + }; + let elapsed = started.elapsed(); + server.join().unwrap(); + + assert!(!error.to_string().is_empty()); + assert!( + elapsed < Duration::from_millis(200), + "WebSocket handshake exceeded its absolute deadline: {elapsed:?}" + ); + } + #[test] fn concurrent_calls_complete_while_reader_receives_events() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fd33fa81a1fa..e79e53b3539f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -146,6 +146,8 @@ struct ShutdownOwnerReconciler { state: Mutex, wake: std::sync::Condvar, mux: std::sync::OnceLock>, + #[cfg(test)] + after_attempt: Mutex>>, } impl ShutdownOwnerReconciler { @@ -221,6 +223,10 @@ impl ShutdownOwnerReconciler { break; } attempts += 1; + #[cfg(test)] + if let Some(hook) = self.after_attempt.lock().unwrap().clone() { + hook(attempts); + } if attempts >= SHUTDOWN_RECONCILE_MAX_ATTEMPTS { let mut state = self.state.lock().unwrap(); state.pending = false; @@ -1158,6 +1164,11 @@ pub struct Mux { #[cfg(test)] terminal_create_after_workspace_reservation: Mutex>>, #[cfg(test)] + terminal_adoption_after_attach: Mutex>>, + #[cfg(all(test, unix))] + terminal_adoption_surface_factory: + Mutex anyhow::Result> + Send + Sync>>>, + #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, #[cfg(test)] shutdown_owner_capacity: AtomicUsize, @@ -1393,6 +1404,10 @@ impl Mux { #[cfg(test)] terminal_create_after_workspace_reservation: Mutex::new(None), #[cfg(test)] + terminal_adoption_after_attach: Mutex::new(None), + #[cfg(all(test, unix))] + terminal_adoption_surface_factory: Mutex::new(None), + #[cfg(test)] browser_bootstrap_before_runtime: Mutex::new(None), #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), @@ -1952,13 +1967,35 @@ impl Mux { break; } let id = mux.next_id(); - if let Ok(surface) = Surface::adopt_hosted( + #[cfg(test)] + let adopted = if let Some(factory) = + mux.terminal_adoption_surface_factory.lock().unwrap().clone() + { + factory(id) + } else { + Surface::adopt_hosted( + id, + options.clone(), + Arc::downgrade(&mux), + record.clone(), + record_path.clone(), + ) + }; + #[cfg(not(test))] + let adopted = Surface::adopt_hosted( id, options.clone(), Arc::downgrade(&mux), record.clone(), record_path.clone(), - ) { + ); + if let Ok(surface) = adopted { + #[cfg(test)] + if let Some(hook) = + mux.terminal_adoption_after_attach.lock().unwrap().clone() + { + hook(); + } if mux .finish_terminal_adoption( &terminal_id, @@ -10707,6 +10744,189 @@ mod tests { ); } + #[test] + fn cleanup_scheduled_during_the_final_retry_gets_a_fresh_retry_budget() { + let mux = test_mux(); + let first = mux.new_workspace(None, Some((80, 24))).unwrap(); + let first = mux.surface(first.id).unwrap(); + let (first_failing, _) = first.set_recovering_server_shutdown_for_test(); + let second = Surface::spawn_for_test( + mux.next_id(), + mux.surface_options.lock().unwrap().clone(), + Arc::downgrade(&mux), + ) + .unwrap(); + let (second_failing, _) = second.set_recovering_server_shutdown_for_test(); + let injected = Arc::new(AtomicBool::new(false)); + *mux.shutdown_owner_reconciler.after_attempt.lock().unwrap() = Some(Arc::new({ + let mux = mux.clone(); + let second = second.clone(); + let first_failing = first_failing.clone(); + let second_failing = second_failing.clone(); + let injected = injected.clone(); + move |attempt| { + if attempt != SHUTDOWN_RECONCILE_MAX_ATTEMPTS + || injected.swap(true, Ordering::AcqRel) + { + return; + } + mux.retire_surface_runtime(second.clone()); + first_failing.store(false, Ordering::Release); + second_failing.store(false, Ordering::Release); + } + })); + + assert!(mux.close_surface(first.id).unwrap()); + let deadline = Instant::now() + Duration::from_secs(2); + while (!injected.load(Ordering::Acquire) || !mux.shutdown_owners.is_empty()) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let reconciled = injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty(); + if !reconciled { + mux.close_all_surfaces_for_shutdown().unwrap(); + } + + assert!( + reconciled, + "cleanup staged during the final attempt was left without a retry worker" + ); + } + + #[test] + fn rejected_sidebar_admission_does_not_launch_a_process() { + let root = std::env::temp_dir() + .join(format!("cmux-sidebar-admission-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let pid_path = root.join("sidebar.pid"); + let mux = Mux::new("sidebar-admission", SurfaceOptions::default()); + mux.set_shutdown_owner_capacity_for_test(0); + let options = SidebarPluginOptions { + command: vec![ + "/bin/sh".into(), + "-c".into(), + format!("echo $$ > {}; exec sleep 30", pid_path.display()), + ], + cwd: None, + }; + + let error = mux.spawn_sidebar_plugin_surface(&options, (80, 24)).unwrap_err(); + let deadline = Instant::now() + Duration::from_secs(1); + while !pid_path.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let spawned_pid = std::fs::read_to_string(&pid_path) + .ok() + .and_then(|pid| pid.trim().parse::().ok()); + if let Some(pid) = spawned_pid { + // SAFETY: this PID was written by the test-owned sidebar command. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + } + drop(mux); + let _ = std::fs::remove_dir_all(root); + + assert_eq!(error.to_string(), "surface_owner_capacity_exhausted"); + assert!( + spawned_pid.is_none(), + "capacity rejection launched sidebar process {spawned_pid:?}" + ); + } + + #[cfg(unix)] + #[test] + fn daemon_handoff_does_not_terminate_an_in_flight_terminal_adoption() { + let options = SurfaceOptions::default(); + let mux = Mux::new_for_test("adoption-handoff", options.clone()); + let workspace = mux.create_empty_workspace(Some("survivor".into()), None, None).unwrap(); + let terminal_id = TerminalId::random().unwrap(); + let terminal_hex = terminal_id.to_hex(); + let incarnation = crate::terminal_host::HostIncarnation::random().unwrap().to_hex(); + let record_path = std::env::temp_dir().join(format!("{terminal_hex}.json")); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 2, + terminal_id: terminal_hex.clone(), + incarnation: incarnation.clone(), + endpoint: record_path.with_extension("sock").to_string_lossy().into_owned(), + owner_token: "00".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: workspace.key.clone(), + supports_set_defaults: true, + supports_terminate_only: true, + }; + { + let mut registry = mux.workspace_registry.lock().unwrap(); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + "adoption-handoff-reserve", + &RegistryTerminal { + terminal_id: terminal_hex.clone(), + workspace_key: workspace.key, + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({}), + exit: None, + }, + ) + .unwrap(); + commit_terminal_lifecycle( + &mut registry, + "terminal-adopting", + "adoption-handoff-adopting", + &terminal_hex, + TerminalLifecycle::Adopting, + Some(&incarnation), + None, + ) + .unwrap(); + } + let kill_attempts = Arc::new(Mutex::new(None)); + *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ + let mux = Arc::downgrade(&mux); + let options = options.clone(); + let kill_attempts = kill_attempts.clone(); + move |id| { + let surface = Surface::spawn_for_test(id, options.clone(), mux.clone())?; + let (_, attempts) = surface.set_recovering_server_shutdown_for_test(); + *kill_attempts.lock().unwrap() = Some(attempts); + Ok(surface) + } + })); + let (attached_tx, attached_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let release_rx = Arc::new(Mutex::new(release_rx)); + *mux.terminal_adoption_after_attach.lock().unwrap() = Some(Arc::new({ + move || { + attached_tx.send(()).unwrap(); + release_rx.lock().unwrap().recv().unwrap(); + } + })); + + mux.schedule_terminal_adoption(options, record.clone(), record_path.clone()); + attached_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + mux.request_daemon_shutdown(); + release_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while !mux.terminal_adoptions.lock().unwrap().is_empty() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let attempts = kill_attempts + .lock() + .unwrap() + .clone() + .expect("adoption surface was not created") + .load(Ordering::Acquire); + + assert_eq!( + attempts, 0, + "daemon handoff terminated the surface attached to the durable host" + ); + } + #[test] fn ordinary_browser_close_terminates_the_owner_staged_during_removal() { fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { From c2f1101e494d3762755f225b721b4c04d8755b87 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:22:02 -0700 Subject: [PATCH 063/521] fix(tui): close shutdown ownership races --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 89 +++++++++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 150 ++++++++++++++++++--- 2 files changed, 216 insertions(+), 23 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 919fbfedde76..939ed4482d30 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -275,6 +275,71 @@ enum Outbound { Flush(Sender<()>), } +struct DeadlineTcpStream { + stream: TcpStream, + deadline: Option, +} + +impl DeadlineTcpStream { + fn new(stream: TcpStream, deadline: Instant) -> Self { + Self { stream, deadline: Some(deadline) } + } + + fn clear_deadline(&mut self) { + self.deadline = None; + } + + fn remaining(&self) -> std::io::Result> { + let Some(deadline) = self.deadline else { + return Ok(None); + }; + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .map(Some) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "CDP connection deadline expired") + }) + } + + fn set_nodelay(&self, nodelay: bool) -> std::io::Result<()> { + self.stream.set_nodelay(nodelay) + } + + fn set_read_timeout(&self, timeout: Option) -> std::io::Result<()> { + self.stream.set_read_timeout(timeout) + } + + fn set_write_timeout(&self, timeout: Option) -> std::io::Result<()> { + self.stream.set_write_timeout(timeout) + } +} + +impl Read for DeadlineTcpStream { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if let Some(remaining) = self.remaining()? { + self.stream.set_read_timeout(Some(remaining))?; + } + self.stream.read(buffer) + } +} + +impl Write for DeadlineTcpStream { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + if let Some(remaining) = self.remaining()? { + self.stream.set_write_timeout(Some(remaining))?; + } + self.stream.write(buffer) + } + + fn flush(&mut self) -> std::io::Result<()> { + if let Some(remaining) = self.remaining()? { + self.stream.set_write_timeout(Some(remaining))?; + } + self.stream.flush() + } +} + impl CdpClient { pub fn connect(web_socket_url: &str, events: SyncSender) -> anyhow::Result { Self::connect_with_timeout(web_socket_url, events, Duration::from_secs(5)) @@ -318,17 +383,27 @@ impl CdpClient { if timeout.is_zero() { anyhow::bail!("CDP connection deadline expired"); } - let stream = TcpStream::connect_timeout(&addr, timeout)?; + let deadline = Instant::now() + .checked_add(timeout) + .ok_or_else(|| anyhow::anyhow!("CDP connection deadline overflow"))?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(|| anyhow::anyhow!("CDP connection deadline expired"))?; + let stream = TcpStream::connect_timeout(&addr, remaining)?; + let stream = DeadlineTcpStream::new(stream, deadline); stream.set_nodelay(true)?; - stream.set_read_timeout(Some(timeout))?; - stream.set_write_timeout(Some(timeout))?; let request = web_socket_url.into_client_request()?; - let (ws, _) = client(request, stream)?; + let (mut ws, _) = client(request, stream)?; + ws.get_mut().clear_deadline(); // The reader thread owns the socket and drains queued outbound // writes before each read poll. A message enqueued just after a // read starts can wait for this window, but writers never contend // on the socket itself. ws.get_ref().set_read_timeout(Some(Duration::from_millis(20)))?; + anyhow::ensure!( + deadline.checked_duration_since(Instant::now()).is_some(), + "CDP connection deadline expired" + ); ws.get_ref().set_write_timeout(Some(timeout))?; let (outbound_tx, outbound_rx) = channel(); let event_queue = Arc::new(EventQueue::new()); @@ -352,7 +427,7 @@ impl CdpClient { fn spawn_reader( &self, - ws: WebSocket, + ws: WebSocket, outbound: Receiver, event_output: SyncSender, ) -> anyhow::Result<()> { @@ -727,7 +802,7 @@ pub fn discover_browser_ws_url(ports: &[u16]) -> Option { fn reader_loop( weak: &Weak, - mut ws: WebSocket, + mut ws: WebSocket, outbound: &Receiver, event_output: &SyncSender, ) { @@ -776,7 +851,7 @@ fn reader_loop( } fn drain_outbound( - ws: &mut WebSocket, + ws: &mut WebSocket, outbound: &Receiver, ) -> anyhow::Result<()> { loop { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e79e53b3539f..8317751be8db 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -35,6 +35,11 @@ use crate::{ }; pub type SurfaceResizeReporter = Arc) + Send + Sync>; +#[cfg(test)] +type ShutdownAttemptHook = Arc; +#[cfg(all(test, unix))] +type TerminalAdoptionSurfaceFactory = + Arc anyhow::Result> + Send + Sync>; const TERMINAL_DIMENSION_MAX: u16 = 10_000; const WORKSPACE_REGISTRY_LIMIT: usize = 4_096; @@ -147,7 +152,7 @@ struct ShutdownOwnerReconciler { wake: std::sync::Condvar, mux: std::sync::OnceLock>, #[cfg(test)] - after_attempt: Mutex>>, + after_attempt: Mutex>, } impl ShutdownOwnerReconciler { @@ -229,6 +234,17 @@ impl ShutdownOwnerReconciler { } if attempts >= SHUTDOWN_RECONCILE_MAX_ATTEMPTS { let mut state = self.state.lock().unwrap(); + if state.pending { + // The final sweep did not include work scheduled + // while it was running. Give that new ownership batch + // its own bounded retry budget. + state.pending = false; + state.degraded = false; + attempts = 0; + delay = SHUTDOWN_RECONCILE_INITIAL_DELAY; + drop(state); + continue; + } state.pending = false; state.worker_started = false; state.degraded = true; @@ -879,6 +895,27 @@ impl Drop for SurfaceCreationGuard<'_> { } } +struct SurfaceOwnerReservation<'a> { + mux: &'a Mux, + active: bool, +} + +impl SurfaceOwnerReservation<'_> { + fn release(&mut self) { + if self.active { + let previous = self.mux.surface_owner_reservations.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "surface owner reservation accounting underflowed"); + self.active = false; + } + } +} + +impl Drop for SurfaceOwnerReservation<'_> { + fn drop(&mut self) { + self.release(); + } +} + #[derive(Default)] struct AsyncSurfaceCreationState { shutting_down: bool, @@ -1136,6 +1173,7 @@ pub struct Mux { shutdown_coordinator: Mutex<()>, shutdown_owners: ShutdownOwnerLedger, shutdown_owner_reconciler: Arc, + surface_owner_reservations: AtomicUsize, next_id: AtomicU64, next_notification_id: AtomicU64, next_active_at: AtomicU64, @@ -1166,8 +1204,7 @@ pub struct Mux { #[cfg(test)] terminal_adoption_after_attach: Mutex>>, #[cfg(all(test, unix))] - terminal_adoption_surface_factory: - Mutex anyhow::Result> + Send + Sync>>>, + terminal_adoption_surface_factory: Mutex>, #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, #[cfg(test)] @@ -1376,6 +1413,7 @@ impl Mux { shutdown_coordinator: Mutex::new(()), shutdown_owners: ShutdownOwnerLedger::default(), shutdown_owner_reconciler: Arc::new(ShutdownOwnerReconciler::default()), + surface_owner_reservations: AtomicUsize::new(0), next_id: AtomicU64::new(next_id), next_notification_id: AtomicU64::new(1), next_active_at: AtomicU64::new(1), @@ -1548,6 +1586,15 @@ impl Mux { handled_terminals.insert(terminal_id.clone()); continue; } + let _creation = self.begin_surface_creation()?; + let mut owner_reservation = match self.reserve_surface_owner() { + Ok(reservation) => reservation, + Err(_) => { + handled_terminals.insert(terminal_id.clone()); + self.schedule_terminal_adoption(options.clone(), record, record_path); + continue; + } + }; let id = self.next_id(); // Bound startup head-of-line blocking per host. One handshake is // enough for healthy hosts; live or indeterminate failures @@ -1589,7 +1636,12 @@ impl Mux { } }; if self - .finish_terminal_adoption(&terminal_id, &record.incarnation, surface.clone()) + .finish_terminal_adoption( + &terminal_id, + &record.incarnation, + surface.clone(), + &mut owner_reservation, + ) .is_err() { surface.kill(); @@ -1778,6 +1830,7 @@ impl Mux { terminal_id: &str, incarnation: &str, surface: Arc, + owner_reservation: &mut SurfaceOwnerReservation<'_>, ) -> anyhow::Result<()> { let _creation = self.begin_surface_creation()?; if surface.is_dead() { @@ -1822,7 +1875,9 @@ impl Mux { } anyhow::bail!("terminal workspace disappeared during adoption"); }; - if let Err(error) = insert_surface_checked(self, &mut state, surface) { + if let Err(error) = + insert_reserved_surface_checked(self, &mut state, surface, owner_reservation) + { drop(state); if let Ok((_, revision)) = commit_terminal_lifecycle( &mut registry, @@ -1966,6 +2021,16 @@ impl Mux { ); break; } + let Ok(_creation) = mux.begin_surface_creation() else { + break; + }; + let mut owner_reservation = match mux.reserve_surface_owner() { + Ok(reservation) => reservation, + Err(_) => { + delay = (delay * 2).min(Duration::from_secs(5)); + continue; + } + }; let id = mux.next_id(); #[cfg(test)] let adopted = if let Some(factory) = @@ -2001,6 +2066,7 @@ impl Mux { &terminal_id, &record.incarnation, surface.clone(), + &mut owner_reservation, ) .is_ok() { @@ -2087,6 +2153,25 @@ impl Mux { self.surface_creations.begin() } + fn reserve_surface_owner(&self) -> anyhow::Result> { + // Every topology removal transfers ownership to the shutdown ledger + // before releasing this same state lock. Counting reservations here + // therefore closes the only interval in which a new runtime could be + // spawned without an owner slot. + let state = self.state.lock().unwrap(); + let used = state + .surfaces + .len() + .saturating_add(self.shutdown_owners.len()) + .saturating_add(self.surface_owner_reservations.load(Ordering::Acquire)); + if used >= self.shutdown_owner_capacity() { + anyhow::bail!("surface_owner_capacity_exhausted"); + } + self.surface_owner_reservations.fetch_add(1, Ordering::AcqRel); + drop(state); + Ok(SurfaceOwnerReservation { mux: self, active: true }) + } + #[cfg(not(test))] fn shutdown_owner_capacity(&self) -> usize { SHUTDOWN_OWNER_CAPACITY @@ -2899,6 +2984,7 @@ impl Mux { if options.command.is_empty() { anyhow::bail!("sidebar plugin command is empty"); } + let mut owner_reservation = self.reserve_surface_owner()?; let id = self.next_id(); let mut opts = self.surface_options.lock().unwrap().clone(); opts.command = Some(options.command.clone()); @@ -2914,7 +3000,12 @@ impl Mux { }; #[cfg(not(test))] let surface = Surface::spawn(id, opts, Arc::downgrade(self))?; - insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone())?; + insert_reserved_surface_checked( + self, + &mut self.state.lock().unwrap(), + surface.clone(), + &mut owner_reservation, + )?; Ok(surface) } @@ -8247,6 +8338,42 @@ fn insert_surface_checked( state: &mut State, surface: Arc, ) -> anyhow::Result<()> { + validate_surface_insertion(state, &surface)?; + if state + .surfaces + .len() + .saturating_add(mux.shutdown_owners.len()) + .saturating_add(mux.surface_owner_reservations.load(Ordering::Acquire)) + >= mux.shutdown_owner_capacity() + { + anyhow::bail!("surface_owner_capacity_exhausted"); + } + state.surfaces.insert(surface.id, surface); + Ok(()) +} + +fn insert_reserved_surface_checked( + mux: &Mux, + state: &mut State, + surface: Arc, + reservation: &mut SurfaceOwnerReservation<'_>, +) -> anyhow::Result<()> { + validate_surface_insertion(state, &surface)?; + if state + .surfaces + .len() + .saturating_add(mux.shutdown_owners.len()) + .saturating_add(mux.surface_owner_reservations.load(Ordering::Acquire)) + > mux.shutdown_owner_capacity() + { + anyhow::bail!("surface_owner_capacity_exhausted"); + } + state.surfaces.insert(surface.id, surface); + reservation.release(); + Ok(()) +} + +fn validate_surface_insertion(state: &State, surface: &Surface) -> anyhow::Result<()> { if state.surfaces.contains_key(&surface.id) { anyhow::bail!("duplicate_surface_id"); } @@ -8261,12 +8388,6 @@ fn insert_surface_checked( { anyhow::bail!("duplicate_terminal_id"); } - if state.surfaces.len().saturating_add(mux.shutdown_owners.len()) - >= mux.shutdown_owner_capacity() - { - anyhow::bail!("surface_owner_capacity_exhausted"); - } - state.surfaces.insert(surface.id, surface); Ok(()) } @@ -10760,9 +10881,6 @@ mod tests { let injected = Arc::new(AtomicBool::new(false)); *mux.shutdown_owner_reconciler.after_attempt.lock().unwrap() = Some(Arc::new({ let mux = mux.clone(); - let second = second.clone(); - let first_failing = first_failing.clone(); - let second_failing = second_failing.clone(); let injected = injected.clone(); move |attempt| { if attempt != SHUTDOWN_RECONCILE_MAX_ATTEMPTS @@ -10906,7 +11024,7 @@ mod tests { } })); - mux.schedule_terminal_adoption(options, record.clone(), record_path.clone()); + mux.schedule_terminal_adoption(options, record, record_path); attached_rx.recv_timeout(Duration::from_secs(2)).unwrap(); mux.request_daemon_shutdown(); release_tx.send(()).unwrap(); From 0a8220b91bf0880cfe3feccb569177d1cb8a6916 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:05:51 -0700 Subject: [PATCH 064/521] test(tui): cover final shutdown ownership gaps --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 62 ++++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 50 ++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 61 +++++++++++++++++++ cmux-tui/dist/scripts/test_package_pypi.py | 46 +++++++++++++++ 4 files changed, 219 insertions(+) create mode 100644 cmux-tui/dist/scripts/test_package_pypi.py diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 7b9167b18cd2..51faeeea1370 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -3043,6 +3043,68 @@ mod tests { assert!(!terminated, "Target.closeTarget success=false discarded the target owner"); } + #[test] + fn ordinary_launched_target_close_requires_confirmation() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); + let mut ws = accept(stream).unwrap(); + let version = read_ws_json(&mut ws); + assert_eq!(version["method"], "Browser.getVersion"); + write_ws_json( + &mut ws, + json!({"id": version["id"], "error": {"code": -32000, "message": "unavailable"}}), + ); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + write_ws_json(&mut ws, json!({"id": discover["id"], "result": {}})); + + let first = read_ws_json(&mut ws); + if first["method"] != "Target.getTargets" { + return false; + } + write_ws_json( + &mut ws, + json!({ + "id": first["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + let close = read_ws_json(&mut ws); + assert_eq!(close["method"], "Target.closeTarget"); + write_ws_json(&mut ws, json!({"id": close["id"], "result": {"success": false}})); + let confirmation = read_ws_json(&mut ws); + assert_eq!(confirmation["method"], "Target.getTargets"); + write_ws_json( + &mut ws, + json!({ + "id": confirmation["id"], + "result": {"targetInfos": [{"targetId": "target-1"}]} + }), + ); + true + }); + let runtime = super::BrowserRuntime::connect_to_endpoint( + &format!("ws://{addr}/devtools/browser/fake"), + None, + BrowserSource::Launched, + ) + .unwrap(); + + let terminated = runtime.close_surface_for_shutdown( + "target-1", + "session-1", + Instant::now() + Duration::from_secs(1), + ); + + runtime.shutdown(); + let queried = server.join().unwrap(); + assert!(queried, "launched target close skipped the authoritative target query"); + assert!(!terminated, "unconfirmed launched target was discarded during ordinary close"); + } + #[test] fn server_shutdown_retries_an_unconfirmed_external_target() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8317751be8db..6250fea65af1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10953,6 +10953,56 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn rejected_terminal_admission_does_not_launch_a_process() { + let root = std::env::temp_dir() + .join(format!("cmux-terminal-admission-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let pid_path = root.join("terminal.pid"); + let mux = Mux::new( + "terminal-admission", + SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!("echo $$ > {}; exec sleep 30", pid_path.display()), + ]), + ..SurfaceOptions::default() + }, + ); + mux.set_shutdown_owner_capacity_for_test(0); + + let state = mux.state.lock().unwrap(); + let spawn = std::thread::spawn({ + let mux = mux.clone(); + move || mux.spawn_surface_with(None, None, Some((80, 24)), None, None) + }); + let deadline = Instant::now() + Duration::from_secs(1); + while !pid_path.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let spawned_pid = std::fs::read_to_string(&pid_path) + .ok() + .and_then(|pid| pid.trim().parse::().ok()); + drop(state); + let error = spawn.join().unwrap().unwrap_err(); + if let Some(pid) = spawned_pid { + // SAFETY: this PID was written by the test-owned terminal command. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + } + drop(mux); + let _ = std::fs::remove_dir_all(root); + + assert_eq!(error.to_string(), "surface_owner_capacity_exhausted"); + assert!( + spawned_pid.is_none(), + "capacity rejection launched terminal process {spawned_pid:?}" + ); + } + #[cfg(unix)] #[test] fn daemon_handoff_does_not_terminate_an_in_flight_terminal_adoption() { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 9b0991af20a2..d95d990af3f8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -3170,6 +3170,67 @@ mod tests { assert_eq!(signals.load(Ordering::Relaxed), 0); } + #[cfg(unix)] + #[test] + fn normal_child_exit_cleans_same_session_descendants_before_reaping() { + let root = std::env::temp_dir() + .join(format!("cmux-local-reap-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let pid_path = root.join("descendant.pid"); + let mux = Mux::new_for_test("local-reap-descendants", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "trap '' HUP TERM; (exec sleep 30) /dev/null 2>&1 & echo $! > {}; exit 0", + pid_path.display() + ), + ]), + ..SurfaceOptions::default() + }; + let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); + let process = + match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; + let deadline = Instant::now() + Duration::from_secs(3); + while (!pid_path.exists() || !process.child_reaped.load(Ordering::Acquire)) + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let descendant = + std::fs::read_to_string(&pid_path).unwrap().trim().parse::().unwrap(); + + let terminated = + surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); + let mut descendant_alive = true; + let exit_deadline = Instant::now() + Duration::from_secs(1); + while descendant_alive && Instant::now() < exit_deadline { + // SAFETY: signal 0 only probes the test-owned PID. + descendant_alive = unsafe { libc::kill(descendant, 0) } == 0; + if descendant_alive { + std::thread::sleep(Duration::from_millis(10)); + } + } + if descendant_alive { + // SAFETY: this PID was written by the test-owned descendant. + unsafe { + libc::kill(descendant, libc::SIGKILL); + } + } + let _ = std::fs::remove_dir_all(root); + + assert!(process.child_reaped.load(Ordering::Acquire)); + assert!(terminated); + assert!( + !descendant_alive, + "normal PTY exit reaped its leader while a same-session descendant remained alive" + ); + } + #[test] fn local_shutdown_does_not_wait_for_the_pty_writer_lock() { let mux = Mux::new_for_test("shutdown-writer-lock", SurfaceOptions::default()); diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py new file mode 100644 index 000000000000..f182bdb4834e --- /dev/null +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Regression tests for the generated PyPI launcher.""" + +from __future__ import annotations + +import unittest + +from package_pypi import wheel_bytes + + +def launcher_command(version: str, argv0: str) -> str: + files = wheel_bytes(version, "test-platform", b"binary") + source = next(data for name, data, _ in files if name == "cmux_tui/_main.py") + namespace: dict[str, object] = {} + exec(compile(source, "cmux_tui/_main.py", "exec"), namespace) + return namespace["_launcher_command"](argv0) # type: ignore[operator] + + +class LauncherCommandTests(unittest.TestCase): + def test_uv_cache_restarts_the_same_version(self) -> None: + self.assertEqual( + launcher_command( + "1.2.3", + "/Users/test/Library/Caches/uv/archive-v0/abcdefgh/bin/cmux", + ), + "uvx cmux==1.2.3", + ) + + def test_pipx_run_cache_restarts_the_same_version(self) -> None: + self.assertEqual( + launcher_command( + "1.2.3", + "/Users/test/Library/Caches/pipx/0123456789abcde/bin/cmux", + ), + "pipx run --spec cmux==1.2.3 cmux", + ) + + def test_stable_install_path_is_reused_directly(self) -> None: + self.assertEqual( + launcher_command("1.2.3", "/Users/test/.local/bin/cmux"), + "/Users/test/.local/bin/cmux", + ) + + +if __name__ == "__main__": + unittest.main() From 2501af84b901e13f39f42904db4028a8bd5a79d9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:22:52 -0700 Subject: [PATCH 065/521] fix(tui): close final shutdown ownership gaps --- .github/workflows/cmux-tui-build-package.yml | 1 + cmux-tui/crates/cmux-tui-core/src/browser.rs | 8 - cmux-tui/crates/cmux-tui-core/src/lib.rs | 5 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 336 ++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 89 +++-- .../src/terminal_host_runtime.rs | 119 +++++-- cmux-tui/crates/cmux-tui/src/main.rs | 21 +- .../crates/cmux-tui/src/server_lifecycle.rs | 5 +- cmux-tui/dist/scripts/package_pypi.py | 10 + 9 files changed, 465 insertions(+), 129 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index d2d7f1bfe624..04d76164fc50 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -296,6 +296,7 @@ jobs: - name: Build PyPI wheels if: inputs.package_pypi run: | + python3 cmux-tui/dist/scripts/test_package_pypi.py python3 cmux-tui/dist/scripts/package_pypi.py \ --binaries-dir dist/binaries \ --version "$PYPI_VERSION" \ diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 51faeeea1370..897125fc4109 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -554,14 +554,6 @@ impl BrowserRuntime { deadline: Instant, ) -> bool { self.unregister(target_id, session_id); - if self.source == BrowserSource::Launched { - if !self.is_closed() { - let _ = self.client.close_target_detached(target_id); - } - // The shared runtime confirms the owned browser process itself - // exited before shutdown can succeed. - return true; - } if !self.is_closed() { match Self::close_external_target_until(&self.client, target_id, deadline) { ExternalTargetShutdown::Closed => return true, diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index a9f1b6799f1f..74d722c1b557 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -41,8 +41,9 @@ pub use mux::{ CellPixelUpdateFailure, Direction, LayoutLeafSpec, LayoutSpec, Mux, MuxEvent, NotificationEvent, NotificationLevel, ProviderWorkspaceAuthority, ProviderWorkspaceAuthorityStatus, ProviderWorkspaceAuthorityUpdateError, RunPlacement, - SidebarPluginOptions, SidebarPluginStatus, SurfaceNotification, SurfaceResizeReporter, - TreeDelta, TreeDeltaKind, WorkspaceMutationResult, WorkspacePlacement, ZoomMode, ZoomState, + ShutdownRequestWatch, SidebarPluginOptions, SidebarPluginStatus, SurfaceNotification, + SurfaceResizeReporter, TreeDelta, TreeDeltaKind, WorkspaceMutationResult, WorkspacePlacement, + ZoomMode, ZoomState, }; pub use pairing::{PairingChallenge, PairingDecision, PairingError}; pub use short_id::assign_short_ids; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 6250fea65af1..a0158faa8b44 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6,7 +6,7 @@ use std::fmt; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; -use std::sync::{Arc, Mutex, MutexGuard, TryLockError, Weak}; +use std::sync::{Arc, Mutex, MutexGuard, Weak}; use std::thread::ThreadId; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -916,6 +916,113 @@ impl Drop for SurfaceOwnerReservation<'_> { } } +#[derive(Default)] +struct ShutdownCoordinatorState { + held: bool, + poisoned: bool, +} + +#[derive(Default)] +struct ShutdownCoordinator { + state: Mutex, + available: std::sync::Condvar, +} + +impl ShutdownCoordinator { + fn lock_until(&self, deadline: Instant) -> anyhow::Result> { + let mut state = self + .state + .lock() + .map_err(|_| anyhow::anyhow!("shutdown coordinator is unavailable"))?; + loop { + if state.poisoned { + anyhow::bail!("shutdown coordinator is unavailable"); + } + if !state.held { + state.held = true; + return Ok(ShutdownCoordinatorGuard { coordinator: self }); + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + anyhow::bail!("another shutdown request exceeded the shutdown deadline"); + }; + let (next, timeout) = self + .available + .wait_timeout(state, remaining) + .map_err(|_| anyhow::anyhow!("shutdown coordinator is unavailable"))?; + state = next; + if timeout.timed_out() && state.held { + anyhow::bail!("another shutdown request exceeded the shutdown deadline"); + } + } + } +} + +struct ShutdownCoordinatorGuard<'a> { + coordinator: &'a ShutdownCoordinator, +} + +impl Drop for ShutdownCoordinatorGuard<'_> { + fn drop(&mut self) { + let mut state = + self.coordinator.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.poisoned |= std::thread::panicking(); + state.held = false; + self.coordinator.available.notify_one(); + } +} + +#[derive(Clone, Copy, Default, PartialEq, Eq)] +enum ShutdownRequestWatchState { + #[default] + Pending, + Requested, + Cancelled, +} + +#[derive(Default)] +struct ShutdownRequestWatchInner { + state: Mutex, + changed: std::sync::Condvar, +} + +impl ShutdownRequestWatchInner { + fn transition(&self, next: ShutdownRequestWatchState) { + let mut state = self.state.lock().unwrap(); + if *state == ShutdownRequestWatchState::Pending { + *state = next; + self.changed.notify_all(); + } + } +} + +/// One-shot notification for a successful server-shutdown request. +#[derive(Clone)] +pub struct ShutdownRequestWatch { + inner: Arc, +} + +impl ShutdownRequestWatch { + /// Block until shutdown is requested or this watch is cancelled. + pub fn wait(&self) -> bool { + let state = self.inner.state.lock().unwrap(); + let state = self + .inner + .changed + .wait_while(state, |state| *state == ShutdownRequestWatchState::Pending) + .unwrap(); + *state == ShutdownRequestWatchState::Requested + } + + /// Wake a waiter that no longer needs to observe shutdown. + pub fn cancel(&self) { + self.inner.transition(ShutdownRequestWatchState::Cancelled); + } + + fn request(&self) { + self.inner.transition(ShutdownRequestWatchState::Requested); + } +} + #[derive(Default)] struct AsyncSurfaceCreationState { shutting_down: bool, @@ -1168,9 +1275,10 @@ pub struct Mux { state: Mutex, subscribers: MuxEventBroadcaster, shutdown_requested: AtomicBool, + shutdown_request_watchers: Mutex>>, surface_creations: SurfaceCreationGate, async_surface_creations: AsyncSurfaceCreationGate, - shutdown_coordinator: Mutex<()>, + shutdown_coordinator: ShutdownCoordinator, shutdown_owners: ShutdownOwnerLedger, shutdown_owner_reconciler: Arc, surface_owner_reservations: AtomicUsize, @@ -1408,9 +1516,10 @@ impl Mux { }), subscribers: MuxEventBroadcaster::default(), shutdown_requested: AtomicBool::new(false), + shutdown_request_watchers: Mutex::new(Vec::new()), surface_creations: SurfaceCreationGate::default(), async_surface_creations: AsyncSurfaceCreationGate::default(), - shutdown_coordinator: Mutex::new(()), + shutdown_coordinator: ShutdownCoordinator::default(), shutdown_owners: ShutdownOwnerLedger::default(), shutdown_owner_reconciler: Arc::new(ShutdownOwnerReconciler::default()), surface_owner_reservations: AtomicUsize::new(0), @@ -1644,7 +1753,7 @@ impl Mux { ) .is_err() { - surface.kill(); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); self.mark_terminal_exited_and_materialize( &terminal_id, "terminal-adoption-failed", @@ -2073,7 +2182,7 @@ impl Mux { mux.reap_if_dead(&surface); break; } - surface.kill(); + mux.retire_reserved_surface_runtime(surface, &mut owner_reservation); let _ = mux.mark_terminal_exited_and_materialize( &terminal_id, "terminal-adoption-failed", @@ -2799,6 +2908,7 @@ impl Mux { workspace_key: Option<&str>, reservation: Option, ) -> anyhow::Result> { + let mut owner_reservation = self.reserve_surface_owner()?; let id = self.next_id(); let mut opts = self.surface_options.lock().unwrap().clone(); if cwd.is_some() { @@ -2892,9 +3002,10 @@ impl Mux { return Err(error); } }; - let identity = surface - .terminal_host_identity() - .ok_or_else(|| anyhow::anyhow!("reserved terminal did not return host identity"))?; + let Some(identity) = surface.terminal_host_identity() else { + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + anyhow::bail!("reserved terminal did not return host identity"); + }; if identity.terminal_id != terminal_hex { let _ = self.transition_terminal_lifecycle( "terminal-exited", @@ -2904,7 +3015,7 @@ impl Mux { None, Some(serde_json::json!({"reason":"host-identity-mismatch"})), ); - surface.kill(); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); anyhow::bail!("terminal host changed registry-reserved identity"); } { @@ -2921,14 +3032,22 @@ impl Mux { let (_, ready_revision) = match ready { Ok(ready) => ready, Err(error) => { - surface.kill(); + drop(registry); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } }; self.emit_terminal_registry_changed(®istry, ready_revision); - if let Err(error) = - insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone()) - { + let insertion = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, + surface.clone(), + &mut owner_reservation, + ) + }; + if let Err(error) = insertion { if let Ok((_, revision)) = commit_terminal_lifecycle( &mut registry, "terminal-exited", @@ -2940,7 +3059,8 @@ impl Mux { ) { self.emit_terminal_registry_changed(®istry, revision); } - surface.kill(); + drop(registry); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } } @@ -2966,11 +3086,18 @@ impl Mux { return Err(error); } }; - if let Err(error) = - insert_surface_checked(self, &mut self.state.lock().unwrap(), surface.clone()) - { + let insertion = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, + surface.clone(), + &mut owner_reservation, + ) + }; + if let Err(error) = insertion { self.pending_workspace_surfaces.lock().unwrap().remove(&id); - surface.kill(); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } Ok(surface) @@ -3000,12 +3127,19 @@ impl Mux { }; #[cfg(not(test))] let surface = Surface::spawn(id, opts, Arc::downgrade(self))?; - insert_reserved_surface_checked( - self, - &mut self.state.lock().unwrap(), - surface.clone(), - &mut owner_reservation, - )?; + let insertion = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, + surface.clone(), + &mut owner_reservation, + ) + }; + if let Err(error) = insertion { + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); + } Ok(surface) } @@ -4035,9 +4169,12 @@ impl Mux { reason: &str, ) -> anyhow::Result<()> { let Some(identity) = surface.terminal_host_identity() else { - let removed = self.state.lock().unwrap().surfaces.remove(&surface.id); - if removed.is_some() { - surface.kill(); + let removed = { + let mut state = self.state.lock().unwrap(); + take_surface_for_retirement(self, &mut state, surface.id) + }; + if let Some(removed) = removed { + self.retire_surface_runtime(removed); } return Ok(()); }; @@ -4242,6 +4379,30 @@ impl Mux { self.retire_surface_runtimes([surface]); } + /// Transfer an uninserted runtime from its admission reservation into the + /// retry ledger without opening a capacity-accounting gap. + fn retire_reserved_surface_runtime( + &self, + surface: Arc, + reservation: &mut SurfaceOwnerReservation<'_>, + ) { + let owner = { + let _state = self.state.lock().unwrap(); + let owner = self.shutdown_owners.stage_surface(&surface); + reservation.release(); + owner + }; + let Some(owner) = owner else { return }; + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { + self.shutdown_owner_reconciler.schedule(); + return; + }; + if self.terminate_shutdown_owners_until(vec![owner], deadline) { + self.shutdown_owner_reconciler.schedule(); + } + } + fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { let owners = surfaces .into_iter() @@ -4326,12 +4487,32 @@ impl Mux { pub fn request_shutdown(&self) { self.shutdown_requested.store(true, Ordering::Release); + let watchers = { + let mut watchers = self.shutdown_request_watchers.lock().unwrap(); + watchers.drain(..).filter_map(|watcher| watcher.upgrade()).collect::>() + }; + for watcher in watchers { + ShutdownRequestWatch { inner: watcher }.request(); + } } pub fn shutdown_requested(&self) -> bool { self.shutdown_requested.load(Ordering::Acquire) } + pub fn watch_shutdown_request(&self) -> ShutdownRequestWatch { + let inner = Arc::new(ShutdownRequestWatchInner::default()); + let watch = ShutdownRequestWatch { inner: inner.clone() }; + let mut watchers = self.shutdown_request_watchers.lock().unwrap(); + watchers.retain(|watcher| watcher.strong_count() != 0); + if self.shutdown_requested.load(Ordering::Acquire) { + watch.request(); + } else { + watchers.push(Arc::downgrade(&inner)); + } + watch + } + pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { let pending = self.shutdown_owners.len(); let state = self.shutdown_owner_reconciler.state.lock().unwrap(); @@ -4531,20 +4712,8 @@ impl Mux { fn lock_shutdown_coordinator_until( &self, deadline: Instant, - ) -> anyhow::Result> { - loop { - match self.shutdown_coordinator.try_lock() { - Ok(coordinator) => return Ok(coordinator), - Err(TryLockError::Poisoned(_)) => { - anyhow::bail!("shutdown coordinator is unavailable") - } - Err(TryLockError::WouldBlock) => {} - } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - anyhow::bail!("another shutdown request exceeded the shutdown deadline"); - }; - std::thread::sleep(remaining.min(Duration::from_millis(10))); - } + ) -> anyhow::Result> { + self.shutdown_coordinator.lock_until(deadline) } /// Atomically reserve a daemon handoff while proving no other native @@ -5488,26 +5657,20 @@ impl Mux { } let notifications = self.surface_notifications(); let active_at = self.next_active_at(); - let attached = { + let attached = (|| -> anyhow::Result<(RunPlacement, TreeDelta, bool)> { let mut state = self.state.lock().unwrap(); if !state.surfaces.contains_key(&surface.id) { anyhow::bail!("terminal closed while its topology binding was being created"); } let Some(wi) = state.workspace_index(workspace) else { - state.surfaces.remove(&surface.id); - surface.kill(); anyhow::bail!("workspace disappeared while creating terminal"); }; let target = state.workspaces[wi].active_screen_ref().map(|screen| screen.active_pane); if let Some(target) = target { let Some((_, si)) = state.screen_of(target) else { - state.surfaces.remove(&surface.id); - surface.kill(); anyhow::bail!("workspace active pane disappeared while creating terminal"); }; let Some(pane) = state.panes.get_mut(&target) else { - state.surfaces.remove(&surface.id); - surface.kill(); anyhow::bail!("workspace active pane disappeared while creating terminal"); }; pane.tabs.push(surface.id); @@ -5522,7 +5685,7 @@ impl Mux { surface.id, ) .expect("new terminal tab is present in tree snapshot"); - ( + Ok(( RunPlacement { surface: surface.id, pane: target, screen, workspace }, TreeDelta { kind: TreeDeltaKind::TabAdded, @@ -5535,7 +5698,7 @@ impl Mux { workspace_revision: None, }, true, - ) + )) } else { let (pane_id, pane) = self.make_pane(surface.id); let screen_id = self.next_id(); @@ -5557,7 +5720,7 @@ impl Mux { screen_id, ) .expect("first workspace screen is present in tree snapshot"); - ( + Ok(( RunPlacement { surface: surface.id, pane: pane_id, @@ -5575,7 +5738,19 @@ impl Mux { workspace_revision: None, }, false, - ) + )) + } + })(); + let attached = match attached { + Ok(attached) => attached, + Err(error) => { + drop(pending_surface); + self.fail_hosted_terminal_attachment( + &surface, + "terminal-topology-attach-failed", + "topology-attach-failed", + )?; + return Err(error); } }; drop(pending_surface); @@ -8315,19 +8490,24 @@ fn bounded_shutdown_fanout( let next = AtomicUsize::new(0); let workers = items.len().min(SHUTDOWN_FANOUT_WORKERS); std::thread::scope(|scope| { - for _ in 0..workers { + for worker in 0..workers { let next = &next; let operation = &operation; - scope.spawn(move || { - loop { - if Instant::now() >= deadline { - break; + let spawned = std::thread::Builder::new() + .name(format!("shutdown-owner-{worker}")) + .spawn_scoped(scope, move || { + loop { + if Instant::now() >= deadline { + break; + } + let index = next.fetch_add(1, Ordering::Relaxed); + let Some(item) = items.get(index) else { break }; + operation(item, deadline); } - let index = next.fetch_add(1, Ordering::Relaxed); - let Some(item) = items.get(index) else { break }; - operation(item, deadline); - } - }); + }); + if spawned.is_err() { + break; + } } }); next.load(Ordering::Relaxed).min(items.len()) @@ -8430,6 +8610,11 @@ fn sidebar_retry_delay(failures: u32) -> Duration { impl Drop for Mux { fn drop(&mut self) { self.shutdown_owner_reconciler.stop(); + if let Ok(watchers) = self.shutdown_request_watchers.get_mut() { + for watcher in watchers.drain(..).filter_map(|watcher| watcher.upgrade()) { + ShutdownRequestWatch { inner: watcher }.cancel(); + } + } if let Ok(state) = self.state.get_mut() { for surface in state.surfaces.values() { surface.disconnect_for_daemon_shutdown(); @@ -11200,11 +11385,36 @@ mod tests { assert!(close.join().unwrap()); owned.set_server_shutdown_failure_for_test(false); - assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + mux.close_all_surfaces_for_shutdown().unwrap(); + assert!(mux.shutdown_owners.is_empty()); let error = shutdown.expect_err("server shutdown passed an untracked retirement owner"); assert!(error.to_string().contains("could not terminate 1 surface process")); } + #[test] + fn shutdown_request_watch_is_signaled_and_cancellable() { + let mux = test_mux(); + let watch = mux.watch_shutdown_request(); + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || result_tx.send(watch.wait()).unwrap()); + assert!(result_rx.recv_timeout(Duration::from_millis(50)).is_err()); + + mux.request_shutdown(); + + assert!(result_rx.recv_timeout(Duration::from_secs(1)).unwrap()); + waiter.join().unwrap(); + assert!(mux.watch_shutdown_request().wait()); + + let fresh = test_mux(); + let watch = fresh.watch_shutdown_request(); + let cancel = watch.clone(); + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || result_tx.send(watch.wait()).unwrap()); + cancel.cancel(); + assert!(!result_rx.recv_timeout(Duration::from_secs(1)).unwrap()); + waiter.join().unwrap(); + } + #[test] fn ordinary_surface_close_does_not_retain_terminal_render_state() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index d95d990af3f8..7e03e43a4476 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -337,6 +337,8 @@ impl HostedFrameStager { const ATTACH_STREAM_CAPACITY: usize = 256; const ATTACH_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024; +#[cfg(unix)] +const NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT: Duration = Duration::from_secs(2); pub(crate) const VT_REPLAY_MAX_BYTES: usize = 8 * 1024 * 1024; pub struct AttachFrameReceiver { @@ -822,7 +824,14 @@ impl LocalPtyProcess { let termination_started = process.termination_started.load(Ordering::Acquire); let pty_drained = process.pty_drained.load(Ordering::Acquire); - if escalation_complete || (!termination_started && pty_drained) { + let normal_cleanup_complete = !escalation_complete + && !termination_started + && pty_drained + && process.kill_session_descendants_before_reap( + Instant::now() + NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, + ); + if escalation_complete || normal_cleanup_complete { + process.group_escalation_complete.store(true, Ordering::Release); let _ = child.wait(); process.child_reaped.store(true, Ordering::Release); drop(signal); @@ -830,15 +839,28 @@ impl LocalPtyProcess { } drop(signal); let state = process.exited.0.lock().unwrap(); - let _state = process - .exited - .1 - .wait_while(state, |_| { - !process.group_escalation_complete.load(Ordering::Acquire) - && (process.termination_started.load(Ordering::Acquire) - || !process.pty_drained.load(Ordering::Acquire)) - }) - .unwrap(); + let _state = if !termination_started && pty_drained { + process + .exited + .1 + .wait_while(state, |_| { + !process.termination_started.load(Ordering::Acquire) + && !process + .group_escalation_complete + .load(Ordering::Acquire) + }) + .unwrap() + } else { + process + .exited + .1 + .wait_while(state, |_| { + !process.group_escalation_complete.load(Ordering::Acquire) + && (process.termination_started.load(Ordering::Acquire) + || !process.pty_drained.load(Ordering::Acquire)) + }) + .unwrap() + }; } } else { let _ = child.wait(); @@ -912,12 +934,18 @@ impl LocalPtyProcess { snapshot: Option<&crate::process_session::SessionProcessSnapshot>, ) -> std::io::Result<()> { let _signal = self.child_signal_lock.lock().unwrap(); - if self.child_reaped.load(Ordering::Acquire) { - return Ok(()); - } let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_reaped.load(Ordering::Acquire) { + return match crate::process_session::session_is_empty_until(session, deadline) { + Ok(true) => Ok(()), + Ok(false) => Err(std::io::Error::other( + "PTY session still has members after its leader was reaped", + )), + Err(error) => Err(error), + }; + } match snapshot { Some(snapshot) => { crate::process_session::signal_from_snapshot(snapshot, session, signal) @@ -933,12 +961,12 @@ impl LocalPtyProcess { snapshot: Option<&crate::process_session::SessionProcessSnapshot>, ) -> std::io::Result { let _signal = self.child_signal_lock.lock().unwrap(); - if self.child_reaped.load(Ordering::Acquire) { - return Ok(true); - } let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_reaped.load(Ordering::Acquire) { + return crate::process_session::session_is_empty_until(leader, deadline); + } match snapshot { Some(snapshot) => crate::process_session::kill_until_only_leader_from_snapshot( snapshot, @@ -955,6 +983,19 @@ impl LocalPtyProcess { } } + #[cfg(unix)] + fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return false; + }; + matches!( + crate::process_session::kill_until_only_leader(leader, leader, deadline, || { + self.child_waitable.load(Ordering::Acquire) + }), + Ok(true) + ) + } + fn terminate_and_wait(&self, deadline: Instant) -> bool { self.terminate_and_wait_inner(deadline, None) } @@ -975,12 +1016,20 @@ impl LocalPtyProcess { ) -> bool { #[cfg(unix)] { - { + let child_reaped = { let _signal = self.child_signal_lock.lock().unwrap(); - if self.child_reaped.load(Ordering::Acquire) { - return true; - } self.termination_started.store(true, Ordering::Release); + self.child_reaped.load(Ordering::Acquire) + }; + self.exited.1.notify_all(); + if child_reaped { + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return false; + }; + return matches!( + crate::process_session::session_is_empty_until(session, deadline), + Ok(true) + ); } if self.signal_terminal_process_session(libc::SIGHUP, deadline, snapshot).is_err() { return false; diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 96ed4136ba8c..2b9316ff777a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2066,28 +2066,46 @@ mod unix { // before reaping. Holding it reserves the session leader and its // numeric session id across enumeration and signaling. let _signal = self.child_signal_lock.lock().unwrap(); - if self.child_reaped.load(Ordering::Acquire) { - return Ok(()); - } let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_reaped.load(Ordering::Acquire) { + return match crate::process_session::session_is_empty_until(session, deadline) { + Ok(true) => Ok(()), + Ok(false) => Err(std::io::Error::other( + "PTY session still has members after its leader was reaped", + )), + Err(error) => Err(error), + }; + } crate::process_session::signal_until(session, signal, deadline) } fn kill_terminal_process_session_until(&self, deadline: Instant) -> std::io::Result { let _signal = self.child_signal_lock.lock().unwrap(); - if self.child_reaped.load(Ordering::Acquire) { - return Ok(true); - } let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_reaped.load(Ordering::Acquire) { + return crate::process_session::session_is_empty_until(leader, deadline); + } crate::process_session::kill_until_only_leader(leader, leader, deadline, || { self.child_waitable.load(Ordering::Acquire) }) } + fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return false; + }; + matches!( + crate::process_session::kill_until_only_leader(leader, leader, deadline, || { + self.child_waitable.load(Ordering::Acquire) + }), + Ok(true) + ) + } + fn request_forced_pty_drain(&self) { self.force_pty_drain.store(true, Ordering::Release); // Wake the otherwise blocking poll in the sole PTY reader. The @@ -2103,6 +2121,7 @@ mod unix { let _signal = self.child_signal_lock.lock().unwrap(); self.termination_started.swap(true, Ordering::AcqRel) }; + self.child_exit.1.notify_all(); if already_started { return; } @@ -2135,9 +2154,35 @@ mod unix { } fn terminate_and_wait(&self) { - { + let child_reaped = { let _signal = self.child_signal_lock.lock().unwrap(); self.termination_started.store(true, Ordering::Release); + self.child_reaped.load(Ordering::Acquire) + }; + self.child_exit.1.notify_all(); + if child_reaped { + let session_empty = self + .pid + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .is_some_and(|session| { + matches!( + crate::process_session::session_is_empty_until( + session, + Instant::now() + HOST_KILL_WAIT, + ), + Ok(true) + ) + }); + if session_empty { + let mut exited = self.child_exit.0.lock().unwrap(); + *exited = true; + drop(exited); + self.child_exit.1.notify_all(); + self.publish_exit_if_drained(); + } else { + self.termination_started.store(false, Ordering::Release); + } + return; } // ProcessSignaller only targets the direct child. Start with a // graceful session hangup so foreground and background jobs can @@ -2633,7 +2678,13 @@ mod unix { let termination_started = child_host.termination_started.load(Ordering::Acquire); let pty_drained = child_host.pty_drained.load(Ordering::Acquire); - if escalation_complete || (!termination_started && pty_drained) { + let normal_cleanup_complete = !escalation_complete + && !termination_started + && pty_drained + && child_host + .kill_session_descendants_before_reap(Instant::now() + HOST_KILL_WAIT); + if escalation_complete || normal_cleanup_complete { + child_host.group_escalation_complete.store(true, Ordering::Release); let _ = child.wait(); child_host.child_reaped.store(true, Ordering::Release); drop(signal); @@ -2641,15 +2692,26 @@ mod unix { } drop(signal); let state = child_host.child_exit.0.lock().unwrap(); - let _state = child_host - .child_exit - .1 - .wait_while(state, |_| { - !child_host.group_escalation_complete.load(Ordering::Acquire) - && (child_host.termination_started.load(Ordering::Acquire) - || !child_host.pty_drained.load(Ordering::Acquire)) - }) - .unwrap(); + let _state = if !termination_started && pty_drained { + child_host + .child_exit + .1 + .wait_while(state, |_| { + !child_host.termination_started.load(Ordering::Acquire) + && !child_host.group_escalation_complete.load(Ordering::Acquire) + }) + .unwrap() + } else { + child_host + .child_exit + .1 + .wait_while(state, |_| { + !child_host.group_escalation_complete.load(Ordering::Acquire) + && (child_host.termination_started.load(Ordering::Acquire) + || !child_host.pty_drained.load(Ordering::Acquire)) + }) + .unwrap() + }; } let mut exited = child_host.child_exit.0.lock().unwrap(); *exited = true; @@ -2662,10 +2724,25 @@ mod unix { let _ = child.wait(); child_host.child_reaped.store(true, Ordering::Release); child_host.child_waitable.store(true, Ordering::Release); - let mut exited = child_host.child_exit.0.lock().unwrap(); - *exited = true; - child_host.child_exit.1.notify_all(); - child_host.publish_exit_if_drained(); + let session_empty = child_host + .pid + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .is_some_and(|session| { + matches!( + crate::process_session::session_is_empty_until( + session, + Instant::now() + HOST_KILL_WAIT, + ), + Ok(true) + ) + }); + if session_empty { + let mut exited = child_host.child_exit.0.lock().unwrap(); + *exited = true; + child_host.child_exit.1.notify_all(); + drop(exited); + child_host.publish_exit_if_drained(); + } } })?; Ok(shared) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index a9218c7f1559..e2b0d00f49a9 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -60,7 +60,6 @@ static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); const MACHINE_PROVIDER_TOKEN_ENV: &str = "CMUX_MACHINE_PROVIDER_TOKEN"; const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORITY"; const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(5); -const SERVER_SHUTDOWN_WATCH_POLL: std::time::Duration = std::time::Duration::from_millis(25); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -944,28 +943,21 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result>, worker: Option>, } impl ServerProcessShutdownGuard { fn start(mux: &Arc, socket_path: PathBuf) -> io::Result { - let weak_mux = Arc::downgrade(mux); + let shutdown_watch = mux.watch_shutdown_request(); + let worker_watch = shutdown_watch.clone(); let worker_socket_path = socket_path.clone(); let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::Builder::new().name("cmux-server-exit-watchdog".into()).spawn( move || { - loop { - match completed.recv_timeout(SERVER_SHUTDOWN_WATCH_POLL) { - Ok(()) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} - } - let Some(mux) = weak_mux.upgrade() else { - return; - }; - if mux.shutdown_requested() { - break; - } + if !worker_watch.wait() { + return; } if matches!( @@ -982,7 +974,7 @@ impl ServerProcessShutdownGuard { } }, )?; - Ok(Self { socket_path, completion: Some(completion), worker: Some(worker) }) + Ok(Self { socket_path, shutdown_watch, completion: Some(completion), worker: Some(worker) }) } fn complete(mut self) { @@ -994,6 +986,7 @@ impl ServerProcessShutdownGuard { if let Some(completion) = self.completion.take() { let _ = completion.send(()); } + self.shutdown_watch.cancel(); if let Some(worker) = self.worker.take() { let _ = worker.join(); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index de55f241e39f..49c9e9e8dcbb 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -378,6 +378,7 @@ impl ServerLifecycle { let mut next_request_id = LEGACY_LIST_REQUEST_ID; let mut consecutive_empty_scans = 0; let mut owners = Vec::::new(); + let mut owner_ids = HashSet::::new(); for _ in 0..LEGACY_MAX_SCAN_ROUNDS { ensure_legacy_helper_active()?; if Instant::now() >= deadline { @@ -387,6 +388,7 @@ impl ServerLifecycle { &mut next_request_id, expected, &mut owners, + &mut owner_ids, deadline, ) { Ok(closed) => closed, @@ -436,6 +438,7 @@ impl ServerLifecycle { next_request_id: &mut u64, expected: ProcessIdentity, owners: &mut Vec, + owner_ids: &mut HashSet, deadline: Instant, ) -> anyhow::Result { let list_request_id = take_legacy_request_id(next_request_id)?; @@ -488,7 +491,7 @@ impl ServerLifecycle { .ok_or_else(|| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - if !owners.iter().any(|captured| captured.id() == owner.id()) { + if owner_ids.insert(owner.id()) { owners.push(owner); } } diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index 806b5f6b2169..40f21f89d334 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -108,6 +108,16 @@ def _launcher_command(argv0: str) -> str: for candidate in parts[index + 1 :] ): return "uvx cmux==" + PACKAGE_VERSION + for index, part in enumerate(parts): + if part != "pipx" or index + 2 >= len(parts): + continue + cache_key = parts[index + 1] + if ( + len(cache_key) == 15 + and all(character in "0123456789abcdef" for character in cache_key) + and parts[index + 2] == "bin" + ): + return "pipx run --spec cmux==" + PACKAGE_VERSION + " cmux" return shlex.quote(argv0) if argv0 else "cmux" From a056485132213259bc1909c85a2fd4fd5dc59c89 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:36:50 -0700 Subject: [PATCH 066/521] test(tui): cover pipx dot-cache launcher --- cmux-tui/dist/scripts/test_package_pypi.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index f182bdb4834e..b32c1a714e2b 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -35,6 +35,15 @@ def test_pipx_run_cache_restarts_the_same_version(self) -> None: "pipx run --spec cmux==1.2.3 cmux", ) + def test_pipx_run_dot_cache_restarts_the_same_version(self) -> None: + self.assertEqual( + launcher_command( + "1.2.3", + "/Users/test/.local/pipx/.cache/0123456789abcde/bin/cmux", + ), + "pipx run --spec cmux==1.2.3 cmux", + ) + def test_stable_install_path_is_reused_directly(self) -> None: self.assertEqual( launcher_command("1.2.3", "/Users/test/.local/bin/cmux"), From 948cc948d9b59a8b34f6a5e61f181f1c2d58d731 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 03:37:52 -0700 Subject: [PATCH 067/521] fix(tui): recognize nested pipx run caches --- cmux-tui/dist/scripts/package_pypi.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index 40f21f89d334..9152c387812f 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -108,14 +108,17 @@ def _launcher_command(argv0: str) -> str: for candidate in parts[index + 1 :] ): return "uvx cmux==" + PACKAGE_VERSION + pipx_seen = False for index, part in enumerate(parts): - if part != "pipx" or index + 2 >= len(parts): + if part == "pipx": + pipx_seen = True continue - cache_key = parts[index + 1] if ( - len(cache_key) == 15 - and all(character in "0123456789abcdef" for character in cache_key) - and parts[index + 2] == "bin" + pipx_seen + and len(part) == 15 + and all(character in "0123456789abcdef" for character in part) + and index + 1 < len(parts) + and parts[index + 1] == "bin" ): return "pipx run --spec cmux==" + PACKAGE_VERSION + " cmux" return shlex.quote(argv0) if argv0 else "cmux" From 8a999477bb21fc0c6dd8d0ec7323ccd0443864e8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 04:14:47 -0700 Subject: [PATCH 068/521] test(tui): expose remaining shutdown safety gaps --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 5 ++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 78 +++++++++++++++++++ .../src/terminal_host_runtime.rs | 37 +++++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 62 +++++++++++++++ 4 files changed, 182 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 897125fc4109..d1f2d4c33edb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -410,6 +410,11 @@ impl BrowserRuntime { Self::connect_to_endpoint(web_socket_url, None, BrowserSource::External) } + #[cfg(test)] + pub(crate) fn connect_launched_for_test(web_socket_url: &str) -> anyhow::Result> { + Self::connect_to_endpoint(web_socket_url, None, BrowserSource::Launched) + } + fn connect_to_endpoint( web_socket_url: &str, chrome: Option, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a0158faa8b44..4901ebce663c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -11364,6 +11364,84 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn server_shutdown_uses_launched_runtime_when_target_confirmation_is_unreachable() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let version = read_ws_json(&mut ws); + assert_eq!(version["method"], "Browser.getVersion"); + ws.send(tungstenite::Message::Text( + serde_json::json!({ + "id": version["id"], + "error": {"code": -32000, "message": "unavailable"} + }) + .to_string() + .into(), + )) + .unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + }); + let runtime = BrowserRuntime::connect_launched_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + server.join().unwrap(); + let disconnect_deadline = Instant::now() + Duration::from_secs(1); + while !runtime.is_closed() && Instant::now() < disconnect_deadline { + std::thread::yield_now(); + } + assert!(runtime.is_closed(), "fixture did not make target confirmation unreachable"); + + let mux = test_mux(); + let options = mux.surface_options.lock().unwrap().clone(); + let surface = browser::new_surface( + 999, + "about:blank".to_string(), + (80, 24), + (8, 16), + &options, + Arc::downgrade(&mux), + ); + surface.as_browser().unwrap().install_shutdown_session_for_test( + runtime.clone(), + "target-1", + "session-1", + ); + insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface).unwrap(); + *mux.browser_runtime.lock().unwrap() = Some(runtime); + + let result = mux.close_all_surfaces_for_shutdown(); + + assert!( + result.is_ok(), + "owned launched browser was not used as the authoritative shutdown fallback: {result:?}" + ); + assert!(mux.shutdown_owners.is_empty()); + assert!(mux.browser_runtime.lock().unwrap().is_none()); + } + #[test] fn server_shutdown_cannot_pass_an_in_flight_surface_retirement() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 2b9316ff777a..0e9a78ecd2fa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3491,6 +3491,43 @@ mod unix { let _ = fs::remove_dir_all(record_path.parent().unwrap()); } + #[test] + fn terminate_only_handshake_uses_one_absolute_deadline() { + let (record_path, record, lease) = record_fixture("terminate-deadline"); + let endpoint = PathBuf::from(&record.endpoint); + prepare_private_dir(endpoint.parent().unwrap()).unwrap(); + let _ = fs::remove_file(&endpoint); + let listener = UnixListener::bind(&endpoint).unwrap(); + let stalled = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + for _ in 0..crate::terminal_host_protocol::HEADER_LEN { + if stream.write_all(&[0]).is_err() { + break; + } + thread::sleep(Duration::from_millis(20)); + } + }); + + let started = Instant::now(); + let result = terminate_terminal_host_with_timeout( + &record, + &record_path, + Duration::from_millis(40), + ); + let elapsed = started.elapsed(); + stalled.join().unwrap(); + let _ = fs::remove_file(endpoint); + drop(lease); + assert!(remove_stale_terminal_host_record(&record_path, &record).unwrap()); + let _ = fs::remove_dir_all(record_path.parent().unwrap()); + + assert!(result.is_err()); + assert!( + elapsed < Duration::from_millis(250), + "partial host frames restarted the shutdown timeout: {elapsed:?}" + ); + } + #[test] fn host_tap_byte_overflow_closes_the_client_socket() { let (host_socket, mut client_socket) = UnixStream::pair().unwrap(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index a610e3f48cdc..da1738ba5699 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -233,6 +233,15 @@ struct LegacyServerProcess { #[cfg(unix)] impl LegacyServerProcess { fn start(name: &str, scenario: &str, reported_pid: Option) -> Self { + Self::start_with_surface_pid(name, scenario, reported_pid, None) + } + + fn start_with_surface_pid( + name: &str, + scenario: &str, + reported_pid: Option, + surface_pid: Option, + ) -> Self { let dir = unique_temp_dir(name); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); @@ -248,6 +257,9 @@ impl LegacyServerProcess { if let Some(reported_pid) = reported_pid { command.env("CMUX_TUI_TEST_LEGACY_REPORTED_PID", reported_pid.to_string()); } + if let Some(surface_pid) = surface_pid { + command.env("CMUX_TUI_TEST_LEGACY_SURFACE_PID", surface_pid.to_string()); + } let child = command.spawn().unwrap(); let mut server = Self { child, socket, dir, descendant_pid_file }; let deadline = Instant::now() + Duration::from_secs(15); @@ -294,6 +306,9 @@ fn legacy_server_process_helper() { .ok() .map(|value| value.parse::().unwrap()) .unwrap_or_else(std::process::id); + let reported_surface_pid = std::env::var("CMUX_TUI_TEST_LEGACY_SURFACE_PID") + .ok() + .map(|value| value.parse::().unwrap()); let mut reparent_on_close = None; if matches!( scenario.as_str(), @@ -481,6 +496,7 @@ fn legacy_server_process_helper() { | "persistent-close-error" | "zombie-child" | "reparent-on-close" + | "unowned-surface" | "browser-surface" | "dead-surface" )); @@ -532,6 +548,8 @@ fn legacy_server_process_helper() { assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); let pid = if scenario == "dead-surface" { None + } else if let Some(pid) = reported_surface_pid { + Some(pid) } else { Some( fs::read_to_string( @@ -1726,6 +1744,50 @@ fn server_stop_refuses_a_legacy_server_that_spoofs_another_process_id() { victim.wait().unwrap(); } +#[cfg(unix)] +#[test] +fn server_stop_never_signals_an_unowned_legacy_surface_pid() { + let mut command = Command::new("yes"); + command.stdout(Stdio::null()).stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut victim = command.spawn().unwrap(); + let mut server = LegacyServerProcess::start_with_surface_pid( + "legacy-server-unowned-surface", + "unowned-surface", + None, + Some(victim.id()), + ); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + let victim_survived = victim.try_wait().unwrap().is_none(); + let server_survived = server.child.try_wait().unwrap().is_none(); + if victim_survived { + victim.kill().unwrap(); + victim.wait().unwrap(); + } + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(victim_survived, "legacy cleanup signaled a process outside the server tree"); + assert!(server_survived, "legacy cleanup stopped the server after ownership proof failed"); +} + #[cfg(unix)] #[test] fn surface_attach_uses_the_full_terminal_and_attaches_only_its_target() { From fb2202d041dae3e68771794de21433cab336be0b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 04:33:40 -0700 Subject: [PATCH 069/521] fix(tui): enforce bounded shutdown ownership --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 4 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 64 +++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 11 +- .../src/terminal_host_runtime.rs | 344 +++++++++++++++--- .../crates/cmux-tui/src/server_lifecycle.rs | 12 +- .../src/server_lifecycle/legacy_process.rs | 11 + 6 files changed, 367 insertions(+), 79 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index d1f2d4c33edb..1a8916608f0b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -111,6 +111,10 @@ impl BrowserShutdownOwner { pub(crate) fn terminate_until(&self, deadline: Instant) -> bool { self.0.runtime.close_surface_for_shutdown(&self.0.target_id, &self.0.session_id, deadline) } + + pub(crate) fn runtime(&self) -> &Arc { + &self.0.runtime + } } struct BrowserState { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4901ebce663c..ef8bd13bc6de 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -15,7 +15,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use zeroize::Zeroize; -use crate::browser::{self, BrowserBootstrap, BrowserRuntime}; +use crate::browser::{self, BrowserBootstrap, BrowserRuntime, BrowserSource}; use crate::event_bus::{MuxEventBroadcaster, MuxEventReceiver}; use crate::layout::{Rect, layout_screen}; use crate::model::{Node, Pane, Screen, State, Workspace}; @@ -128,6 +128,12 @@ impl ShutdownOwnerLedger { } } + fn remove_browser_runtime(&self, runtime: &Arc) { + self.owners.lock().unwrap().retain(|_, owner| { + owner.browser_runtime().is_none_or(|owner_runtime| !Arc::ptr_eq(owner_runtime, runtime)) + }); + } + fn len(&self) -> usize { self.owners.lock().unwrap().len() } @@ -4648,7 +4654,47 @@ impl Mux { } self.terminate_staged_shutdown_owners_until(deadline); let retained = self.shutdown_owners.snapshot(); - let browser_surface_failed = retained.iter().any(|(_, owner)| owner.is_browser()); + let current_browser_runtime = self.browser_runtime.lock().unwrap().take(); + let mut browser_runtimes = Vec::new(); + for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { + if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { + browser_runtimes.push(runtime.clone()); + } + } + if let Some(runtime) = ¤t_browser_runtime + && !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) + { + browser_runtimes.push(runtime.clone()); + } + let mut browser_runtime_failed = false; + for runtime in browser_runtimes { + let is_current = current_browser_runtime + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &runtime)); + let surface_failed = retained.iter().any(|(_, owner)| { + owner + .browser_runtime() + .is_some_and(|owner_runtime| Arc::ptr_eq(owner_runtime, &runtime)) + }); + if surface_failed && runtime.source() == BrowserSource::External { + if is_current { + *self.browser_runtime.lock().unwrap() = Some(runtime); + } + continue; + } + if runtime.shutdown_until(deadline) { + if runtime.source() == BrowserSource::Launched { + self.shutdown_owners.remove_browser_runtime(&runtime); + } + } else { + browser_runtime_failed = true; + if is_current { + *self.browser_runtime.lock().unwrap() = Some(runtime); + } + } + } + + let retained = self.shutdown_owners.snapshot(); let mut failed_surface_ids = retained .iter() .filter_map(|(key, _)| match key { @@ -4666,20 +4712,6 @@ impl Mux { .collect::>(); failed_hosts.sort(); - let browser_runtime = (!browser_surface_failed) - .then(|| self.browser_runtime.lock().unwrap().take()) - .flatten(); - let browser_runtime_failed = if let Some(runtime) = browser_runtime { - if runtime.shutdown_until(deadline) { - false - } else { - *self.browser_runtime.lock().unwrap() = Some(runtime); - true - } - } else { - false - }; - let mut failures = Vec::new(); if !failed_surface_ids.is_empty() { failures.push(format!( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 7e03e43a4476..2de4806280bc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -30,7 +30,7 @@ pub use crate::browser::{ BrowserAttachState, BrowserFrame, BrowserFrameStream, BrowserSource, BrowserStatus, }; use crate::browser::{ - BrowserResizeWaiter, BrowserShutdownOwner, BrowserSurface, PendingBrowserResize, + BrowserResizeWaiter, BrowserRuntime, BrowserShutdownOwner, BrowserSurface, PendingBrowserResize, }; #[cfg(unix)] use crate::terminal_host_protocol::{FLAG_COLORS_FOLLOW, Frame, MessageKind, PROTOCOL_VERSION}; @@ -746,8 +746,13 @@ impl SurfaceShutdownOwner { } } - pub(crate) fn is_browser(&self) -> bool { - matches!(self.kind, SurfaceShutdownOwnerKind::Browser(_)) + pub(crate) fn browser_runtime(&self) -> Option<&Arc> { + match &self.kind { + SurfaceShutdownOwnerKind::Browser(owner) => Some(owner.runtime()), + SurfaceShutdownOwnerKind::Local(_) => None, + #[cfg(unix)] + SurfaceShutdownOwnerKind::Hosted(_) => None, + } } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 0e9a78ecd2fa..68beb05df9ac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -275,7 +275,9 @@ mod unix { use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; - use std::os::fd::{AsRawFd, RawFd}; + use std::mem::{offset_of, size_of}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; + use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; use std::os::unix::net::{UnixListener, UnixStream}; use std::os::unix::process::CommandExt; @@ -721,11 +723,11 @@ mod unix { self.send(MessageKind::Terminate, &[]) } - pub(crate) fn terminate_with_timeout(&self, timeout: Duration) -> std::io::Result<()> { + pub(crate) fn terminate_until(&self, deadline: Instant) -> std::io::Result<()> { let mut writer = self.writer.lock().unwrap(); - writer.set_write_timeout(Some(timeout))?; let frame = Frame::new(MessageKind::Terminate, Vec::new()); - let result = write_frame(&mut *writer, &frame).map_err(protocol_io_error); + let result = write_frame(&mut DeadlineStream::new(&mut writer, deadline), &frame) + .map_err(protocol_io_error); if result.is_err() { let _ = writer.shutdown(std::net::Shutdown::Both); } @@ -1140,19 +1142,28 @@ mod unix { connect_record(record, record_path) } - pub(crate) fn adopt_terminal_host_with_timeout( + pub(crate) fn adopt_terminal_host_until( record: TerminalHostRecord, record_path: PathBuf, - timeout: Duration, + deadline: Instant, ) -> anyhow::Result { validate_terminal_host_record(&record_path, &record)?; - connect_record_with_timeout(record, record_path, timeout) + connect_record_until(record, record_path, deadline) } + #[cfg(test)] pub(crate) fn terminate_terminal_host_with_timeout( record: &TerminalHostRecord, record_path: &Path, timeout: Duration, + ) -> anyhow::Result<()> { + terminate_terminal_host_until(record, record_path, Instant::now() + timeout) + } + + fn terminate_terminal_host_until( + record: &TerminalHostRecord, + record_path: &Path, + deadline: Instant, ) -> anyhow::Result<()> { validate_terminal_host_record(record_path, record)?; if !record.supports_terminate_only { @@ -1161,9 +1172,8 @@ mod unix { let terminal_id = TerminalId::from_bytes(decode_hex_array(&record.terminal_id)?); let incarnation = HostIncarnation::from_bytes(decode_hex_array(&record.incarnation)?); let owner_token = CapabilityToken::from_bytes(decode_hex_array(&record.owner_token)?); - let mut stream = connect_with_timeout(Path::new(&record.endpoint), timeout)?; - stream.set_read_timeout(Some(timeout))?; - stream.set_write_timeout(Some(timeout))?; + let mut stream = connect_until(Path::new(&record.endpoint), deadline)?; + let mut stream = DeadlineStream::new(&mut stream, deadline); let hello = ClientHello { min_version: PROTOCOL_VERSION, max_version: PROTOCOL_VERSION, @@ -1479,18 +1489,17 @@ mod unix { if last_terminate_attempt.is_none_or(|attempt: Instant| { now.duration_since(attempt) >= Duration::from_millis(100) }) { - let timeout = deadline - .saturating_duration_since(Instant::now()) - .min(Duration::from_millis(100)); - if !timeout.is_zero() { + let attempt_deadline = deadline.min(Instant::now() + Duration::from_millis(100)); + if Instant::now() < attempt_deadline { if record.supports_terminate_only { - let _ = terminate_terminal_host_with_timeout(record, record_path, timeout); - } else if let Ok(host) = adopt_terminal_host_with_timeout( + let _ = + terminate_terminal_host_until(record, record_path, attempt_deadline); + } else if let Ok(host) = adopt_terminal_host_until( record.clone(), record_path.to_path_buf(), - timeout, + attempt_deadline, ) { - let _ = host.terminate_with_timeout(timeout); + let _ = host.terminate_until(attempt_deadline); host.disconnect(); } last_terminate_attempt = Some(Instant::now()); @@ -1515,13 +1524,19 @@ mod unix { record: TerminalHostRecord, record_path: PathBuf, handshake_timeout: Duration, + ) -> anyhow::Result { + connect_record_until(record, record_path, Instant::now() + handshake_timeout) + } + + fn connect_record_until( + record: TerminalHostRecord, + record_path: PathBuf, + deadline: Instant, ) -> anyhow::Result { let terminal_id = TerminalId::from_bytes(decode_hex_array(&record.terminal_id)?); let incarnation = HostIncarnation::from_bytes(decode_hex_array(&record.incarnation)?); let owner_token = CapabilityToken::from_bytes(decode_hex_array(&record.owner_token)?); - let mut stream = connect_with_timeout(Path::new(&record.endpoint), handshake_timeout)?; - stream.set_read_timeout(Some(handshake_timeout))?; - stream.set_write_timeout(Some(handshake_timeout))?; + let mut stream = connect_until(Path::new(&record.endpoint), deadline)?; let hello = ClientHello { min_version: PROTOCOL_VERSION, max_version: PROTOCOL_VERSION, @@ -1530,35 +1545,42 @@ mod unix { terminal_id, token: owner_token, }; - write_frame(&mut stream, &hello.into_frame(1))?; - let hello_frame = read_required_frame(&mut stream, "host hello")?; - if hello_frame.kind != MessageKind::HostHello { - anyhow::bail!("terminal host rejected owner handshake"); - } - let host_hello = HostHello::decode(&hello_frame.payload)?; - if host_hello.terminal_id != terminal_id || host_hello.incarnation != incarnation { - anyhow::bail!("terminal-host record identity does not match live host"); - } - let snapshot_frame = read_required_frame(&mut stream, "terminal snapshot")?; - if snapshot_frame.kind != MessageKind::Snapshot - || snapshot_frame.flags != 0 - || snapshot_frame.request_id != 0 - { - anyhow::bail!("terminal host did not send an initial snapshot"); - } - let mut snapshot = decode_snapshot(&snapshot_frame.payload)?; - let colors_frame = read_required_frame(&mut stream, "terminal color state")?; - if colors_frame.kind != MessageKind::Colors - || colors_frame.flags != 0 - || colors_frame.sequence != snapshot_frame.sequence - || colors_frame.request_id != 0 - { - anyhow::bail!("terminal host did not send Colors at the snapshot sequence boundary"); - } - snapshot.sequence_boundary = snapshot_frame.sequence; - snapshot.colors = decode_terminal_color_overrides(&colors_frame.payload)?; - let snapshot_size = (snapshot.cols, snapshot.rows); + let (snapshot, snapshot_size) = { + let mut handshake = DeadlineStream::new(&mut stream, deadline); + write_frame(&mut handshake, &hello.into_frame(1))?; + let hello_frame = read_required_frame(&mut handshake, "host hello")?; + if hello_frame.kind != MessageKind::HostHello { + anyhow::bail!("terminal host rejected owner handshake"); + } + let host_hello = HostHello::decode(&hello_frame.payload)?; + if host_hello.terminal_id != terminal_id || host_hello.incarnation != incarnation { + anyhow::bail!("terminal-host record identity does not match live host"); + } + let snapshot_frame = read_required_frame(&mut handshake, "terminal snapshot")?; + if snapshot_frame.kind != MessageKind::Snapshot + || snapshot_frame.flags != 0 + || snapshot_frame.request_id != 0 + { + anyhow::bail!("terminal host did not send an initial snapshot"); + } + let mut snapshot = decode_snapshot(&snapshot_frame.payload)?; + let colors_frame = read_required_frame(&mut handshake, "terminal color state")?; + if colors_frame.kind != MessageKind::Colors + || colors_frame.flags != 0 + || colors_frame.sequence != snapshot_frame.sequence + || colors_frame.request_id != 0 + { + anyhow::bail!( + "terminal host did not send Colors at the snapshot sequence boundary" + ); + } + snapshot.sequence_boundary = snapshot_frame.sequence; + snapshot.colors = decode_terminal_color_overrides(&colors_frame.payload)?; + let snapshot_size = (snapshot.cols, snapshot.rows); + (snapshot, snapshot_size) + }; stream.set_read_timeout(None)?; + stream.set_write_timeout(Some(HOST_HANDSHAKE_TIMEOUT))?; // Keep bounded writes for the lifetime of the disposable admin // mirror. A stopped or wedged host must not block a mux/control thread // forever while it sends input, mouse, resize, or Terminate. Reads are @@ -1586,18 +1608,228 @@ mod unix { Ok(attachment) } - fn connect_with_timeout(path: &Path, timeout: Duration) -> anyhow::Result { - let deadline = Instant::now() + timeout; + struct DeadlineStream<'a> { + stream: &'a mut UnixStream, + deadline: Instant, + } + + impl<'a> DeadlineStream<'a> { + fn new(stream: &'a mut UnixStream, deadline: Instant) -> Self { + Self { stream, deadline } + } + + fn remaining(&self) -> std::io::Result { + self.deadline + .checked_duration_since(Instant::now()) + .filter(|value| !value.is_zero()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host operation exceeded its deadline", + ) + }) + } + + fn shutdown(&self, how: std::net::Shutdown) -> std::io::Result<()> { + self.stream.shutdown(how) + } + } + + impl Read for DeadlineStream<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.stream.set_read_timeout(Some(self.remaining()?))?; + self.stream.read(buffer) + } + } + + impl Write for DeadlineStream<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.stream.set_write_timeout(Some(self.remaining()?))?; + self.stream.write(buffer) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.stream.set_write_timeout(Some(self.remaining()?))?; + self.stream.flush() + } + } + + fn connect_until(path: &Path, deadline: Instant) -> anyhow::Result { + let mut last_error = None; loop { - let error = match UnixStream::connect(path) { + if Instant::now() >= deadline { + return Err(last_error + .unwrap_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host connection exceeded its deadline", + ) + }) + .into()); + } + match connect_once_until(path, deadline) { Ok(stream) => return Ok(stream), - Err(error) => error, - }; + Err(error) => last_error = Some(error), + } + let now = Instant::now(); + if now >= deadline { + continue; + } + thread::sleep(deadline.saturating_duration_since(now).min(Duration::from_millis(10))); + } + } + + fn connect_once_until(path: &Path, deadline: Instant) -> std::io::Result { + let (address, address_len) = unix_socket_address(path)?; + // SAFETY: socket has no pointer arguments and returns a new owned + // descriptor on success. + let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; + if descriptor < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: descriptor is a fresh successful socket result and this + // OwnedFd takes its sole ownership. + let descriptor = unsafe { OwnedFd::from_raw_fd(descriptor) }; + // SAFETY: F_GETFD only reads flags from this valid descriptor. + let descriptor_flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFD) }; + if descriptor_flags < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: F_SETFD updates flags on this valid descriptor. + if unsafe { + libc::fcntl(descriptor.as_raw_fd(), libc::F_SETFD, descriptor_flags | libc::FD_CLOEXEC) + } < 0 + { + return Err(std::io::Error::last_os_error()); + } + let stream = UnixStream::from(descriptor); + stream.set_nonblocking(true)?; + // SAFETY: address is an initialized sockaddr_un with its exact + // kernel-visible length, and stream owns a valid AF_UNIX socket. + let connected = unsafe { + libc::connect( + stream.as_raw_fd(), + (&raw const address).cast::(), + address_len, + ) + }; + if connected != 0 { + let error = std::io::Error::last_os_error(); + let code = error.raw_os_error(); + if code != Some(libc::EINPROGRESS) + && code != Some(libc::EAGAIN) + && code != Some(libc::EWOULDBLOCK) + { + return Err(error); + } + wait_for_connect(&stream, deadline)?; + } + stream.set_nonblocking(false)?; + Ok(stream) + } + + fn wait_for_connect(stream: &UnixStream, deadline: Instant) -> std::io::Result<()> { + loop { let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Err(error.into()); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host connection exceeded its deadline", + )); }; - thread::sleep(remaining.min(Duration::from_millis(10))); + if remaining.is_zero() { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host connection exceeded its deadline", + )); + } + let timeout_ms = remaining.as_millis().max(1).min(i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: stream.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for the + // duration of this call. + let result = unsafe { libc::poll(&mut descriptor, 1, timeout_ms) }; + if result == 0 { + continue; + } + if result < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if descriptor.revents & libc::POLLNVAL != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "terminal host connection socket became invalid", + )); + } + let mut socket_error = 0; + let mut socket_error_len = libc::socklen_t::try_from(size_of::()).unwrap(); + // SAFETY: socket_error and its length describe a writable c_int, + // and stream owns a valid socket descriptor. + if unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_ERROR, + (&raw mut socket_error).cast(), + &raw mut socket_error_len, + ) + } != 0 + { + return Err(std::io::Error::last_os_error()); + } + return if socket_error == 0 { + Ok(()) + } else { + Err(std::io::Error::from_raw_os_error(socket_error)) + }; + } + } + + fn unix_socket_address(path: &Path) -> std::io::Result<(libc::sockaddr_un, libc::socklen_t)> { + const SUN_PATH_CAPACITY: usize = + size_of::() - offset_of!(libc::sockaddr_un, sun_path); + let path = path.as_os_str().as_bytes(); + if path.is_empty() || path.len() >= SUN_PATH_CAPACITY || path.contains(&0) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "invalid terminal host Unix socket path", + )); } + // SAFETY: all-zero is a valid starting representation for + // sockaddr_un; family, path, and platform length are set below. + let mut address = unsafe { std::mem::zeroed::() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (destination, source) in address.sun_path.iter_mut().zip(path) { + *destination = *source as libc::c_char; + } + let address_len = offset_of!(libc::sockaddr_un, sun_path) + path.len() + 1; + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" + ))] + { + address.sun_len = u8::try_from(address_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "terminal host Unix socket path is too long", + ) + })?; + } + Ok(( + address, + libc::socklen_t::try_from(address_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "terminal host Unix socket path is too long", + ) + })?, + )) } fn read_required_frame(reader: &mut impl Read, context: &str) -> anyhow::Result { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 49c9e9e8dcbb..f10701a97b5c 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -373,6 +373,7 @@ impl ServerLifecycle { fn close_legacy_surfaces_until_stable( &mut self, expected: ProcessIdentity, + captured: &CapturedProcessTree, deadline: Instant, ) -> anyhow::Result> { let mut next_request_id = LEGACY_LIST_REQUEST_ID; @@ -387,6 +388,7 @@ impl ServerLifecycle { let closed = match self.close_legacy_surface_snapshot( &mut next_request_id, expected, + captured, &mut owners, &mut owner_ids, deadline, @@ -437,6 +439,7 @@ impl ServerLifecycle { &mut self, next_request_id: &mut u64, expected: ProcessIdentity, + captured: &CapturedProcessTree, owners: &mut Vec, owner_ids: &mut HashSet, deadline: Instant, @@ -484,7 +487,7 @@ impl ServerLifecycle { .ok_or_else(|| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - let owner = capture_process_session(pid, expected, deadline) + let owner = capture_process_session(pid, expected, captured, deadline) .map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })? @@ -647,9 +650,10 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { let captured = capture_legacy_process_tree(actual, deadline)?; ensure_legacy_helper_active()?; - let owners = lifecycle.close_legacy_surfaces_until_stable(actual, deadline).map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; + let owners = + lifecycle.close_legacy_surfaces_until_stable(actual, &captured, deadline).map_err( + |_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed), + )?; for owner in owners { ensure_legacy_helper_active()?; owner.terminate_until(deadline).map_err(|_| { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index f719992a496a..1144c41f2ffe 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -193,6 +193,10 @@ pub(super) struct CapturedProcessTree { } impl CapturedProcessTree { + fn contains_process(&self, process: ProcessIdentity) -> bool { + self.sessions.iter().any(|session| session.members.contains(&process)) + } + pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { for session in &self.sessions { if !session.kill_until_empty(deadline)? { @@ -226,6 +230,7 @@ impl CapturedProcessSession { pub(super) fn capture_process_session( pid: libc::pid_t, server: ProcessIdentity, + captured: &CapturedProcessTree, deadline: Instant, ) -> io::Result> { if pid <= 1 @@ -235,6 +240,12 @@ pub(super) fn capture_process_session( return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid PTY owner process")); } let Some(process) = ProcessIdentity::capture(pid)? else { return Ok(None) }; + if !captured.contains_process(process) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PTY owner was not captured in the verified server process tree", + )); + } if process.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { return Ok(None); } From ff1eb97955728cf2dfd2d8612104c452e8fb0e98 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:38:55 -0700 Subject: [PATCH 070/521] test(tui): expose remaining lifecycle races --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 46 ++++++++++++++++++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 47 ++++++++++++++++++++++ cmux-tui/crates/cmux-tui/src/main.rs | 38 +++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 110cb07a5dc6..69c0be18bb0f 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -12,6 +12,8 @@ static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); #[cfg(test)] thread_local! { static FORCE_KILL_TIMEOUT: std::cell::Cell = const { std::cell::Cell::new(false) }; + static FORCE_REAPER_SPAWN_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; + static REAP_CHILD_CALLED_ON_THREAD: std::cell::Cell = const { std::cell::Cell::new(false) }; } #[derive(Debug, Clone)] @@ -157,12 +159,19 @@ struct ReapRequest { fn reap_child_detached(child: Child, profile_dir: Option) { let request = std::sync::Arc::new(Mutex::new(Some(ReapRequest { child, profile_dir }))); let worker_request = request.clone(); - let spawned = + #[cfg(test)] + let force_spawn_failure = FORCE_REAPER_SPAWN_FAILURE.get(); + #[cfg(not(test))] + let force_spawn_failure = false; + let spawned = if force_spawn_failure { + Err(std::io::Error::other("forced Chrome reaper spawn failure")) + } else { std::thread::Builder::new().name("cmux-tui-cdp-chrome-reaper".into()).spawn(move || { if let Some(request) = worker_request.lock().unwrap().take() { reap_child(request); } - }); + }) + }; if spawned.is_err() && let Some(request) = request.lock().unwrap().take() { @@ -171,6 +180,8 @@ fn reap_child_detached(child: Child, profile_dir: Option) { } fn reap_child(mut request: ReapRequest) { + #[cfg(test)] + REAP_CHILD_CALLED_ON_THREAD.set(true); let _ = request.child.kill(); if request.child.wait().is_ok() && let Some(profile_dir) = request.profile_dir @@ -399,4 +410,35 @@ mod tests { assert!(reaped, "dropping Chrome discarded an unconfirmed child without reaping it"); } + + #[cfg(unix)] + #[test] + fn reaper_spawn_failure_never_waits_on_the_caller_thread() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let chrome = Chrome { + child: Mutex::new(Some(child)), + profile_dir: make_profile_dir().unwrap(), + profile_ephemeral: true, + web_socket_url: "ws://127.0.0.1/unused".to_string(), + }; + + REAP_CHILD_CALLED_ON_THREAD.set(false); + FORCE_KILL_TIMEOUT.set(true); + FORCE_REAPER_SPAWN_FAILURE.set(true); + drop(chrome); + FORCE_REAPER_SPAWN_FAILURE.set(false); + FORCE_KILL_TIMEOUT.set(false); + let reaped_inline = REAP_CHILD_CALLED_ON_THREAD.get(); + + let cleanup = Command::new("true").spawn().unwrap(); + reap_child_detached(cleanup, None); + + assert!(!reaped_inline, "reaper spawn failure fell back to an unbounded caller wait"); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ef8bd13bc6de..61a5e4d30a23 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1316,6 +1316,8 @@ pub struct Mux { #[cfg(test)] terminal_create_after_workspace_reservation: Mutex>>, #[cfg(test)] + new_pane_after_spawn: Mutex) + Send + Sync>>>, + #[cfg(test)] terminal_adoption_after_attach: Mutex>>, #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex>, @@ -1557,6 +1559,8 @@ impl Mux { #[cfg(test)] terminal_create_after_workspace_reservation: Mutex::new(None), #[cfg(test)] + new_pane_after_spawn: Mutex::new(None), + #[cfg(test)] terminal_adoption_after_attach: Mutex::new(None), #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex::new(None), @@ -6354,6 +6358,10 @@ impl Mux { eprintln!("cmux-tui: pane PTY creation failed: {error:#}"); anyhow::anyhow!("pane creation failed") })?; + #[cfg(test)] + if let Some(hook) = self.new_pane_after_spawn.lock().unwrap().clone() { + hook(surface.clone()); + } let pane_id = self.next_id(); let active_at = self.next_active_at(); let mut changed_screen = None; @@ -10555,6 +10563,45 @@ mod tests { }); } + #[test] + fn failed_new_pane_attachment_retains_shutdown_ownership() { + let mux = test_mux(); + let first = mux.new_workspace(None, None).unwrap(); + let target = mux.with_state(|state| state.pane_of(first.id).unwrap()); + let (spawned_tx, spawned_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let release_rx = Arc::new(Mutex::new(release_rx)); + *mux.new_pane_after_spawn.lock().unwrap() = Some(Arc::new({ + move |surface| { + surface.set_server_shutdown_failure_for_test(true); + spawned_tx.send(surface).unwrap(); + release_rx.lock().unwrap().recv().unwrap(); + } + })); + + let create = std::thread::spawn({ + let mux = mux.clone(); + move || mux.new_pane(target, None) + }); + let abandoned = spawned_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + assert!(mux.close_pane(target).unwrap()); + release_tx.send(()).unwrap(); + + let error = create.join().unwrap().unwrap_err(); + assert!(error.to_string().contains("not found")); + assert!(mux.surface(abandoned.id).is_none()); + assert_eq!( + mux.shutdown_owners.len(), + 1, + "failed pane attachment discarded its process owner instead of staging a retry" + ); + + *mux.new_pane_after_spawn.lock().unwrap() = None; + abandoned.set_server_shutdown_failure_for_test(false); + let _ = mux.close_all_surfaces_for_shutdown(); + assert!(mux.shutdown_owners.is_empty()); + } + #[test] fn zellij_new_pane_exits_zoom_before_focusing_the_new_pane() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index e2b0d00f49a9..6cf9e6c795cd 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1415,6 +1415,44 @@ mod tests { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } + #[cfg(unix)] + #[test] + fn completed_server_guard_never_unlinks_a_replacement_socket() { + use std::os::unix::net::{UnixListener, UnixStream}; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let socket_path = + std::env::temp_dir().join(format!("cg-{}-{suffix:x}.sock", std::process::id())); + let original = UnixListener::bind(&socket_path).unwrap(); + let mux = Mux::new("server-guard-test", SurfaceOptions::default()); + let shutdown_watch = mux.watch_shutdown_request(); + let (completion, completed) = std::sync::mpsc::channel(); + let (replacement_tx, replacement_rx) = std::sync::mpsc::sync_channel(1); + let replacement_path = socket_path.clone(); + let worker = std::thread::spawn(move || { + completed.recv().unwrap(); + let replacement = UnixListener::bind(&replacement_path).unwrap(); + replacement_tx.send(replacement).unwrap(); + }); + let guard = ServerProcessShutdownGuard { + socket_path: socket_path.clone(), + shutdown_watch, + completion: Some(completion), + worker: Some(worker), + }; + + guard.complete(); + + let replacement = replacement_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let client = UnixStream::connect(&socket_path) + .expect("guard completion unlinked the replacement server socket"); + drop(client); + drop(replacement); + drop(original); + cmux_tui_core::server::cleanup(&socket_path); + } + #[test] fn surface_only_attach_does_not_publish_session_default_colors() { let mux = Mux::new("surface-only-color-test", SurfaceOptions::default()); From da8eb23d9a2c266e0afd5fdd9a66d5d2a92d4980 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:46:25 -0700 Subject: [PATCH 071/521] fix(tui): close remaining lifecycle races --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 36 ++++++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/mux.rs | 12 +++++--- cmux-tui/crates/cmux-tui/src/main.rs | 5 ++- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 69c0be18bb0f..456312b1a4cd 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -2,12 +2,15 @@ use std::ffi::OsString; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; -use std::sync::Mutex; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; +use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); +// Thread exhaustion must not turn browser shutdown into an unbounded caller +// wait. Retain exact Child ownership here and retry it with the next reaper. +static REAPER_BACKLOG: OnceLock>> = OnceLock::new(); #[cfg(test)] thread_local! { @@ -157,7 +160,25 @@ struct ReapRequest { } fn reap_child_detached(child: Child, profile_dir: Option) { - let request = std::sync::Arc::new(Mutex::new(Some(ReapRequest { child, profile_dir }))); + let mut requests = { + let mut backlog = REAPER_BACKLOG.get_or_init(|| Mutex::new(Vec::new())).lock().unwrap(); + let mut requests = std::mem::take(&mut *backlog); + requests.push(ReapRequest { child, profile_dir }); + requests + }; + let mut pending = requests.drain(..); + while let Some(request) = pending.next() { + if let Err(request) = spawn_reap_child(request) { + let mut backlog = REAPER_BACKLOG.get_or_init(|| Mutex::new(Vec::new())).lock().unwrap(); + backlog.push(request); + backlog.extend(pending); + break; + } + } +} + +fn spawn_reap_child(request: ReapRequest) -> Result<(), ReapRequest> { + let request = std::sync::Arc::new(Mutex::new(Some(request))); let worker_request = request.clone(); #[cfg(test)] let force_spawn_failure = FORCE_REAPER_SPAWN_FAILURE.get(); @@ -172,11 +193,14 @@ fn reap_child_detached(child: Child, profile_dir: Option) { } }) }; - if spawned.is_err() - && let Some(request) = request.lock().unwrap().take() - { - reap_child(request); + if spawned.is_err() { + return Err(request + .lock() + .unwrap() + .take() + .expect("reaper request remains after spawn error")); } + Ok(()) } fn reap_child(mut request: ReapRequest) { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 61a5e4d30a23..54a46f7280b2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -40,6 +40,8 @@ type ShutdownAttemptHook = Arc; #[cfg(all(test, unix))] type TerminalAdoptionSurfaceFactory = Arc anyhow::Result> + Send + Sync>; +#[cfg(test)] +type NewPaneAfterSpawnHook = Arc) + Send + Sync>; const TERMINAL_DIMENSION_MAX: u16 = 10_000; const WORKSPACE_REGISTRY_LIMIT: usize = 4_096; @@ -1316,7 +1318,7 @@ pub struct Mux { #[cfg(test)] terminal_create_after_workspace_reservation: Mutex>>, #[cfg(test)] - new_pane_after_spawn: Mutex) + Send + Sync>>>, + new_pane_after_spawn: Mutex>, #[cfg(test)] terminal_adoption_after_attach: Mutex>>, #[cfg(all(test, unix))] @@ -6427,12 +6429,14 @@ impl Mux { entity, workspace_revision: None, }); - } else { - state.surfaces.remove(&surface.id); } } let Some(screen) = changed_screen else { - surface.kill(); + self.fail_hosted_terminal_attachment( + &surface, + "terminal-pane-attach-failed", + "pane-disappeared-before-attach", + )?; anyhow::bail!("pane {target} not found"); }; self.emit(MuxEvent::TreeDelta(delta.expect("successful new pane has a tree delta"))); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 6cf9e6c795cd..fe35798cae3a 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -982,10 +982,9 @@ impl ServerProcessShutdownGuard { } fn finish(&mut self) { + let Some(completion) = self.completion.take() else { return }; cmux_tui_core::server::cleanup(&self.socket_path); - if let Some(completion) = self.completion.take() { - let _ = completion.send(()); - } + let _ = completion.send(()); self.shutdown_watch.cancel(); if let Some(worker) = self.worker.take() { let _ = worker.join(); From a1e265de2912e6088324ae76f3960cd9bc6d493e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:25:08 -0700 Subject: [PATCH 072/521] test(tui): expose final Chrome reaper stall --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 456312b1a4cd..f70cd1985480 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -465,4 +465,51 @@ mod tests { assert!(!reaped_inline, "reaper spawn failure fell back to an unbounded caller wait"); } + + #[cfg(unix)] + #[test] + fn final_reaper_request_survives_a_shutdown_thread_spawn_failure() { + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + } + + let warmup = Command::new("true").spawn().unwrap(); + reap_child_detached(warmup, None); + + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + let profile_dir = make_profile_dir().unwrap(); + + FORCE_REAPER_SPAWN_FAILURE.set(true); + reap_child_detached(child, Some(profile_dir.clone())); + FORCE_REAPER_SPAWN_FAILURE.set(false); + + let deadline = Instant::now() + Duration::from_secs(1); + let cleaned = loop { + if unsafe { kill(pid, 0) } != 0 && !profile_dir.exists() { + break true; + } + if Instant::now() >= deadline { + break false; + } + std::thread::sleep(Duration::from_millis(10)); + }; + if !cleaned { + let mut status = 0; + unsafe { + kill(pid, 9); + waitpid(pid, &mut status, 0); + } + let _ = std::fs::remove_dir_all(&profile_dir); + } + + assert!(cleaned, "the final Chrome cleanup remained parked until another browser shutdown"); + } } From 45dfb13b97322b0213ab9a4cd5a45d74ce36b264 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:28:12 -0700 Subject: [PATCH 073/521] fix(tui): preflight persistent Chrome reaper --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 137 ++++++++++++++------- 1 file changed, 92 insertions(+), 45 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index f70cd1985480..dc8ef9d32087 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -5,12 +5,12 @@ use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use std::sync::{Mutex, OnceLock}; +use std::thread::JoinHandle; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); -// Thread exhaustion must not turn browser shutdown into an unbounded caller -// wait. Retain exact Child ownership here and retry it with the next reaper. -static REAPER_BACKLOG: OnceLock>> = OnceLock::new(); +static CHROME_REAPER: OnceLock>> = OnceLock::new(); +const REAPER_POLL_INTERVAL: Duration = Duration::from_millis(10); #[cfg(test)] thread_local! { @@ -40,6 +40,7 @@ pub struct Chrome { profile_dir: PathBuf, profile_ephemeral: bool, web_socket_url: String, + reaper: ChromeReaperHandle, } impl Chrome { @@ -56,6 +57,10 @@ impl Chrome { } pub fn launch_with(options: &ChromeLaunchOptions) -> anyhow::Result { + // Browser shutdown must never depend on creating a new thread. Refuse + // to launch the child until the process-wide reaper owns a worker. + let reaper = chrome_reaper_handle() + .map_err(|error| anyhow::anyhow!("failed to start Chrome reaper: {error}"))?; let (profile_dir, profile_ephemeral) = profile_dir_for(options)?; std::fs::create_dir_all(&profile_dir)?; let mut command = Command::new(&options.binary); @@ -100,7 +105,11 @@ impl Chrome { let _ = std::fs::remove_dir_all(&profile_dir); } } else { - reap_child_detached(child, profile_ephemeral.then(|| profile_dir.clone())); + reap_child_detached( + &reaper, + child, + profile_ephemeral.then(|| profile_dir.clone()), + ); } anyhow::bail!( "Chrome did not publish a DevTools endpoint within 10s (binary: {})", @@ -114,6 +123,7 @@ impl Chrome { profile_dir, profile_ephemeral, web_socket_url, + reaper, }) } @@ -147,7 +157,11 @@ impl Drop for Chrome { } let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); if let Some(child) = child { - reap_child_detached(child, self.profile_ephemeral.then(|| self.profile_dir.clone())); + reap_child_detached( + &self.reaper, + child, + self.profile_ephemeral.then(|| self.profile_dir.clone()), + ); } else if self.profile_ephemeral { let _ = std::fs::remove_dir_all(&self.profile_dir); } @@ -159,59 +173,87 @@ struct ReapRequest { profile_dir: Option, } -fn reap_child_detached(child: Child, profile_dir: Option) { - let mut requests = { - let mut backlog = REAPER_BACKLOG.get_or_init(|| Mutex::new(Vec::new())).lock().unwrap(); - let mut requests = std::mem::take(&mut *backlog); - requests.push(ReapRequest { child, profile_dir }); - requests - }; - let mut pending = requests.drain(..); - while let Some(request) = pending.next() { - if let Err(request) = spawn_reap_child(request) { - let mut backlog = REAPER_BACKLOG.get_or_init(|| Mutex::new(Vec::new())).lock().unwrap(); - backlog.push(request); - backlog.extend(pending); - break; - } +#[derive(Clone)] +struct ChromeReaperHandle { + sender: mpsc::Sender, +} + +struct ChromeReaper { + handle: ChromeReaperHandle, + _worker: JoinHandle<()>, +} + +impl ChromeReaper { + fn start() -> std::io::Result { + let (sender, receiver) = mpsc::channel(); + let handle = ChromeReaperHandle { sender }; + let worker = spawn_reaper_worker(receiver)?; + Ok(Self { handle, _worker: worker }) } } -fn spawn_reap_child(request: ReapRequest) -> Result<(), ReapRequest> { - let request = std::sync::Arc::new(Mutex::new(Some(request))); - let worker_request = request.clone(); +fn spawn_reaper_worker(receiver: mpsc::Receiver) -> std::io::Result> { #[cfg(test)] let force_spawn_failure = FORCE_REAPER_SPAWN_FAILURE.get(); #[cfg(not(test))] let force_spawn_failure = false; - let spawned = if force_spawn_failure { - Err(std::io::Error::other("forced Chrome reaper spawn failure")) - } else { - std::thread::Builder::new().name("cmux-tui-cdp-chrome-reaper".into()).spawn(move || { - if let Some(request) = worker_request.lock().unwrap().take() { - reap_child(request); + if force_spawn_failure { + return Err(std::io::Error::other("forced Chrome reaper spawn failure")); + } + std::thread::Builder::new() + .name("cmux-tui-cdp-chrome-reaper".into()) + .spawn(move || run_reaper(receiver)) +} + +fn chrome_reaper_handle() -> std::io::Result { + let mut slot = CHROME_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); + if slot.is_none() { + *slot = Some(ChromeReaper::start()?); + } + Ok(slot.as_ref().expect("Chrome reaper initialized").handle.clone()) +} + +fn reap_child_detached(reaper: &ChromeReaperHandle, child: Child, profile_dir: Option) { + reaper + .sender + .send(ReapRequest { child, profile_dir }) + .expect("Chrome reaper worker remains alive while its service is registered"); +} + +fn run_reaper(receiver: mpsc::Receiver) { + let mut pending = Vec::new(); + loop { + let received = if pending.is_empty() { + receiver.recv().map_err(|_| mpsc::RecvTimeoutError::Disconnected) + } else { + receiver.recv_timeout(REAPER_POLL_INTERVAL) + }; + match received { + Ok(request) => pending.push(request), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) if pending.is_empty() => return, + Err(mpsc::RecvTimeoutError::Disconnected) => { + std::thread::sleep(REAPER_POLL_INTERVAL); } - }) - }; - if spawned.is_err() { - return Err(request - .lock() - .unwrap() - .take() - .expect("reaper request remains after spawn error")); + } + while let Ok(request) = receiver.try_recv() { + pending.push(request); + } + pending.retain_mut(|request| !poll_reap_request(request)); } - Ok(()) } -fn reap_child(mut request: ReapRequest) { +fn poll_reap_request(request: &mut ReapRequest) -> bool { #[cfg(test)] REAP_CHILD_CALLED_ON_THREAD.set(true); let _ = request.child.kill(); - if request.child.wait().is_ok() - && let Some(profile_dir) = request.profile_dir - { + if !matches!(request.child.try_wait(), Ok(Some(_))) { + return false; + } + if let Some(profile_dir) = request.profile_dir.take() { let _ = std::fs::remove_dir_all(profile_dir); } + true } fn kill_child_until(child: &mut Child, deadline: Instant) -> bool { @@ -383,6 +425,7 @@ mod tests { profile_dir, profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), + reaper: chrome_reaper_handle().unwrap(), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); @@ -410,6 +453,7 @@ mod tests { profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), + reaper: chrome_reaper_handle().unwrap(), }; FORCE_KILL_TIMEOUT.set(true); @@ -445,11 +489,13 @@ mod tests { .stderr(Stdio::null()) .spawn() .unwrap(); + let reaper = chrome_reaper_handle().unwrap(); let chrome = Chrome { child: Mutex::new(Some(child)), profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), + reaper: reaper.clone(), }; REAP_CHILD_CALLED_ON_THREAD.set(false); @@ -461,7 +507,7 @@ mod tests { let reaped_inline = REAP_CHILD_CALLED_ON_THREAD.get(); let cleanup = Command::new("true").spawn().unwrap(); - reap_child_detached(cleanup, None); + reap_child_detached(&reaper, cleanup, None); assert!(!reaped_inline, "reaper spawn failure fell back to an unbounded caller wait"); } @@ -474,8 +520,9 @@ mod tests { fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; } + let reaper = chrome_reaper_handle().unwrap(); let warmup = Command::new("true").spawn().unwrap(); - reap_child_detached(warmup, None); + reap_child_detached(&reaper, warmup, None); let child = Command::new("sleep") .arg("60") @@ -488,7 +535,7 @@ mod tests { let profile_dir = make_profile_dir().unwrap(); FORCE_REAPER_SPAWN_FAILURE.set(true); - reap_child_detached(child, Some(profile_dir.clone())); + reap_child_detached(&reaper, child, Some(profile_dir.clone())); FORCE_REAPER_SPAWN_FAILURE.set(false); let deadline = Instant::now() + Duration::from_secs(1); From cfb33e9e3b96094e058b7bbe233ed22cb1ede0a8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 06:59:16 -0700 Subject: [PATCH 074/521] test(tui): expose remaining bounded shutdown gaps --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 104 ++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 48 ++++++++ .../src/terminal_host_runtime.rs | 6 + cmux-tui/crates/cmux-tui/tests/cli.rs | 37 +++++++ 4 files changed, 195 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index dc8ef9d32087..a5d13f4ad792 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -2,6 +2,8 @@ use std::ffi::OsString; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +#[cfg(test)] +use std::sync::atomic::{AtomicBool, AtomicUsize}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use std::sync::{Mutex, OnceLock}; @@ -12,6 +14,15 @@ static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); static CHROME_REAPER: OnceLock>> = OnceLock::new(); const REAPER_POLL_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(test)] +static FORCE_REAPER_PENDING: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +static FORCE_REAPER_WAIT_ERROR: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +static REAPER_POLL_ATTEMPTS: AtomicUsize = AtomicUsize::new(0); +#[cfg(test)] +static REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); + #[cfg(test)] thread_local! { static FORCE_KILL_TIMEOUT: std::cell::Cell = const { std::cell::Cell::new(false) }; @@ -246,6 +257,17 @@ fn run_reaper(receiver: mpsc::Receiver) { fn poll_reap_request(request: &mut ReapRequest) -> bool { #[cfg(test)] REAP_CHILD_CALLED_ON_THREAD.set(true); + #[cfg(test)] + { + REAPER_POLL_ATTEMPTS.fetch_add(1, Ordering::Relaxed); + if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { + let _ = request.child.kill(); + return false; + } + if FORCE_REAPER_PENDING.load(Ordering::Acquire) { + return false; + } + } let _ = request.child.kill(); if !matches!(request.child.try_wait(), Ok(Some(_))) { return false; @@ -559,4 +581,86 @@ mod tests { assert!(cleaned, "the final Chrome cleanup remained parked until another browser shutdown"); } + + #[cfg(unix)] + #[test] + fn persistent_reaper_backs_off_unconfirmed_children() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + } + + let reaper = chrome_reaper_handle().unwrap(); + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + FORCE_REAPER_PENDING.store(true, Ordering::Release); + REAPER_POLL_ATTEMPTS.store(0, Ordering::Release); + reap_child_detached(&reaper, child, None); + std::thread::sleep(Duration::from_millis(160)); + let attempts = REAPER_POLL_ATTEMPTS.load(Ordering::Acquire); + FORCE_REAPER_PENDING.store(false, Ordering::Release); + + let deadline = Instant::now() + Duration::from_secs(1); + while unsafe { kill(pid, 0) } == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + if unsafe { kill(pid, 0) } == 0 { + let mut status = 0; + unsafe { + kill(pid, 9); + waitpid(pid, &mut status, 0); + } + } + + assert!(attempts <= 6, "unconfirmed child was polled {attempts} times in 160ms"); + } + + #[cfg(unix)] + #[test] + fn terminal_reaper_wait_error_releases_the_profile() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); + unsafe extern "C" { + fn kill(pid: i32, signal: i32) -> i32; + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + } + + let reaper = chrome_reaper_handle().unwrap(); + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + let profile_dir = make_profile_dir().unwrap(); + FORCE_REAPER_WAIT_ERROR.store(true, Ordering::Release); + reap_child_detached(&reaper, child, Some(profile_dir.clone())); + let deadline = Instant::now() + Duration::from_millis(200); + while profile_dir.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let released = !profile_dir.exists(); + FORCE_REAPER_WAIT_ERROR.store(false, Ordering::Release); + + let cleanup_deadline = Instant::now() + Duration::from_secs(1); + while profile_dir.exists() && Instant::now() < cleanup_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let mut status = 0; + unsafe { + kill(pid, 9); + waitpid(pid, &mut status, 0); + } + let _ = std::fs::remove_dir_all(&profile_dir); + + assert!(released, "terminal wait error retained the Chrome profile indefinitely"); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 54a46f7280b2..67b01a9f7450 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1324,6 +1324,8 @@ pub struct Mux { #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex>, #[cfg(test)] + terminal_adoption_workers_started: AtomicUsize, + #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, #[cfg(test)] shutdown_owner_capacity: AtomicUsize, @@ -1567,6 +1569,8 @@ impl Mux { #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex::new(None), #[cfg(test)] + terminal_adoption_workers_started: AtomicUsize::new(0), + #[cfg(test)] browser_bootstrap_before_runtime: Mutex::new(None), #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), @@ -2051,6 +2055,8 @@ impl Mux { let spawn_result = std::thread::Builder::new() .name(format!("terminal-adopt-{terminal_id}")) .spawn(move || { + #[cfg(test)] + mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); let mut delay = Duration::from_millis(100); loop { if mux.shutting_down.load(Ordering::Acquire) { @@ -11271,6 +11277,48 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn terminal_adoption_overflow_uses_one_shared_worker() { + let options = SurfaceOptions::default(); + let mux = Mux::new_for_test("adoption-worker-bound", options.clone()); + let root = + std::env::temp_dir().join(format!("cmux-adoption-worker-bound-{}", std::process::id())); + + for _ in 0..3 { + let terminal_id = TerminalId::random().unwrap().to_hex(); + let record_path = root.join(format!("{terminal_id}.json")); + mux.schedule_terminal_adoption( + options.clone(), + crate::terminal_host_runtime::TerminalHostRecord { + record_version: 2, + terminal_id, + incarnation: crate::terminal_host::HostIncarnation::random().unwrap().to_hex(), + endpoint: record_path.with_extension("sock").to_string_lossy().into_owned(), + owner_token: "00".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: String::new(), + supports_set_defaults: true, + supports_terminate_only: true, + }, + record_path, + ); + } + + let deadline = Instant::now() + Duration::from_millis(250); + while mux.terminal_adoption_workers_started.load(Ordering::Acquire) < 3 + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let workers = mux.terminal_adoption_workers_started.load(Ordering::Acquire); + mux.request_daemon_shutdown(); + let _ = std::fs::remove_dir_all(root); + + assert!(workers <= 1, "overflow adoption started {workers} retry threads"); + } + #[cfg(unix)] #[test] fn daemon_handoff_does_not_terminate_an_in_flight_terminal_adoption() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 68beb05df9ac..495d478c5c27 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2631,6 +2631,12 @@ mod unix { thread::sleep(Duration::from_millis(delay.min(5_000))); } let bootstrapped = crate::terminal_host::bootstrap_stdio_once(reader, writer)?; + if let Ok(delay) = std::env::var("CMUX_TUI_TEST_STALL_AFTER_BOOTSTRAP_READY_MS") + && let Ok(delay) = delay.parse::() + && delay > 0 + { + thread::sleep(Duration::from_millis(delay.min(5_000))); + } let Some(launch_frame) = read_frame(reader, MAX_LAUNCH_PAYLOAD)? else { // Keep the one-frame bootstrap probe useful for compatibility and // packaging diagnostics. Production launchers always follow it diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index da1738ba5699..792258568457 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1185,6 +1185,43 @@ fn server_stop_cancels_a_blocked_terminal_host_launch() { assert!(terminal_host_pids(&host_root).is_empty()); } +#[cfg(unix)] +#[test] +fn server_stop_cancels_a_blocked_terminal_host_launch_write() { + let mut server = HeadlessServer::start_with( + "server-stop-blocked-launch-write", + false, + &[("CMUX_TUI_TEST_STALL_AFTER_BOOTSTRAP_READY_MS", "3000")], + ); + let command = format!(": #{}", "x".repeat(128 * 1024)); + let mut create = Command::new(bin()) + .args(["--socket"]) + .arg(&server.socket) + .args(["run", "--new-workspace", "--command"]) + .arg(command) + .env_remove("CMUX_TUI_SOCKET") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); + std::thread::sleep(Duration::from_millis(250)); + assert!(create.try_wait().unwrap().is_none(), "terminal creation did not block on Launch"); + + let stop_started = Instant::now(); + let stop = cli(&server, &["--json", "server", "stop"]); + let stop_elapsed = stop_started.elapsed(); + + assert_success(&stop); + assert!(server.child.wait().unwrap().success()); + assert!(!create.wait().unwrap().success()); + assert!(terminal_host_pids(&host_root).is_empty()); + assert!( + stop_elapsed < Duration::from_secs(2), + "shutdown waited for the blocked Launch write: {stop_elapsed:?}" + ); +} + #[cfg(unix)] #[test] fn cancelled_published_host_is_terminated_through_its_record() { From d27ca432e582261d2bd5f58ac1569be8e614f8d9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:11:15 -0700 Subject: [PATCH 075/521] fix(tui): bound remaining shutdown workers --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 194 +++++-- cmux-tui/crates/cmux-tui-core/src/mux.rs | 489 ++++++++++++------ .../src/terminal_host_runtime.rs | 95 +++- 3 files changed, 551 insertions(+), 227 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index a5d13f4ad792..5205cd063a26 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -3,16 +3,18 @@ use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; #[cfg(test)] -use std::sync::atomic::{AtomicBool, AtomicUsize}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc; -use std::sync::{Mutex, OnceLock}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::JoinHandle; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; static PROFILE_SEQ: AtomicU64 = AtomicU64::new(1); static CHROME_REAPER: OnceLock>> = OnceLock::new(); -const REAPER_POLL_INTERVAL: Duration = Duration::from_millis(10); +const REAPER_CAPACITY: usize = 4_096; +const REAPER_INITIAL_BACKOFF: Duration = Duration::from_millis(10); +const REAPER_MAX_BACKOFF: Duration = Duration::from_secs(1); #[cfg(test)] static FORCE_REAPER_PENDING: AtomicBool = AtomicBool::new(false); @@ -51,7 +53,7 @@ pub struct Chrome { profile_dir: PathBuf, profile_ephemeral: bool, web_socket_url: String, - reaper: ChromeReaperHandle, + reaper: Option, } impl Chrome { @@ -69,9 +71,11 @@ impl Chrome { pub fn launch_with(options: &ChromeLaunchOptions) -> anyhow::Result { // Browser shutdown must never depend on creating a new thread. Refuse - // to launch the child until the process-wide reaper owns a worker. - let reaper = chrome_reaper_handle() - .map_err(|error| anyhow::anyhow!("failed to start Chrome reaper: {error}"))?; + // to launch the child until the process-wide reaper owns a worker and + // has reserved bounded cleanup capacity for this exact process. + let reaper = chrome_reaper_lease().map_err(|error| { + anyhow::anyhow!("cannot launch Chrome without bounded cleanup ownership: {error}") + })?; let (profile_dir, profile_ephemeral) = profile_dir_for(options)?; std::fs::create_dir_all(&profile_dir)?; let mut command = Command::new(&options.binary); @@ -90,23 +94,35 @@ impl Chrome { .take() .ok_or_else(|| anyhow::anyhow!("failed to capture Chrome stderr"))?; let (tx, rx) = mpsc::channel(); - std::thread::Builder::new().name("cmux-tui-cdp-chrome-stderr".into()).spawn(move || { - let mut reader = BufReader::new(stderr); - let mut line = String::new(); - let mut sent = false; - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) | Err(_) => break, - Ok(_) => { - if !sent && let Some(url) = parse_devtools_url(&line) { - let _ = tx.send(url); - sent = true; + let stderr_worker = std::thread::Builder::new() + .name("cmux-tui-cdp-chrome-stderr".into()) + .spawn(move || { + let mut reader = BufReader::new(stderr); + let mut line = String::new(); + let mut sent = false; + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) => { + if !sent && let Some(url) = parse_devtools_url(&line) { + let _ = tx.send(url); + sent = true; + } } } } + }); + if let Err(error) = stderr_worker { + if kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)) { + if profile_ephemeral { + let _ = std::fs::remove_dir_all(&profile_dir); + } + } else { + reap_child_detached(reaper, child, profile_ephemeral.then(|| profile_dir.clone())); } - })?; + return Err(error.into()); + } let web_socket_url = match rx.recv_timeout(Duration::from_secs(10)) { Ok(url) => url, @@ -117,7 +133,7 @@ impl Chrome { } } else { reap_child_detached( - &reaper, + reaper, child, profile_ephemeral.then(|| profile_dir.clone()), ); @@ -134,7 +150,7 @@ impl Chrome { profile_dir, profile_ephemeral, web_socket_url, - reaper, + reaper: Some(reaper), }) } @@ -169,7 +185,7 @@ impl Drop for Chrome { let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); if let Some(child) = child { reap_child_detached( - &self.reaper, + self.reaper.take().expect("live Chrome retains its reaper lease"), child, self.profile_ephemeral.then(|| self.profile_dir.clone()), ); @@ -182,24 +198,37 @@ impl Drop for Chrome { struct ReapRequest { child: Child, profile_dir: Option, + _lease: ChromeReaperLease, + next_attempt: Instant, + retry_delay: Duration, } -#[derive(Clone)] -struct ChromeReaperHandle { +struct ChromeReaperLease { sender: mpsc::Sender, + active: Arc, +} + +impl Drop for ChromeReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } } struct ChromeReaper { - handle: ChromeReaperHandle, + sender: mpsc::Sender, + active: Arc, _worker: JoinHandle<()>, } impl ChromeReaper { fn start() -> std::io::Result { let (sender, receiver) = mpsc::channel(); - let handle = ChromeReaperHandle { sender }; let worker = spawn_reaper_worker(receiver)?; - Ok(Self { handle, _worker: worker }) + Ok(Self { sender, active: Arc::new(AtomicUsize::new(0)), _worker: worker }) + } + + fn lease(&self) -> std::io::Result { + reserve_reaper_lease(self.sender.clone(), self.active.clone(), REAPER_CAPACITY) } } @@ -216,41 +245,79 @@ fn spawn_reaper_worker(receiver: mpsc::Receiver) -> std::io::Result .spawn(move || run_reaper(receiver)) } -fn chrome_reaper_handle() -> std::io::Result { +fn reserve_reaper_lease( + sender: mpsc::Sender, + active: Arc, + capacity: usize, +) -> std::io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < capacity).then_some(current + 1) + }) + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::WouldBlock, "Chrome reaper capacity exhausted") + })?; + Ok(ChromeReaperLease { sender, active }) +} + +fn chrome_reaper_lease() -> std::io::Result { let mut slot = CHROME_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); if slot.is_none() { *slot = Some(ChromeReaper::start()?); } - Ok(slot.as_ref().expect("Chrome reaper initialized").handle.clone()) + slot.as_ref().expect("Chrome reaper initialized").lease() } -fn reap_child_detached(reaper: &ChromeReaperHandle, child: Child, profile_dir: Option) { - reaper - .sender - .send(ReapRequest { child, profile_dir }) +fn reap_child_detached(reaper: ChromeReaperLease, child: Child, profile_dir: Option) { + let sender = reaper.sender.clone(); + sender + .send(ReapRequest { + child, + profile_dir, + _lease: reaper, + next_attempt: Instant::now(), + retry_delay: REAPER_INITIAL_BACKOFF, + }) .expect("Chrome reaper worker remains alive while its service is registered"); } fn run_reaper(receiver: mpsc::Receiver) { - let mut pending = Vec::new(); + let mut pending: Vec = Vec::new(); loop { let received = if pending.is_empty() { receiver.recv().map_err(|_| mpsc::RecvTimeoutError::Disconnected) } else { - receiver.recv_timeout(REAPER_POLL_INTERVAL) + let now = Instant::now(); + let wait = pending + .iter() + .map(|request| request.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(REAPER_MAX_BACKOFF); + receiver.recv_timeout(wait) }; match received { Ok(request) => pending.push(request), Err(mpsc::RecvTimeoutError::Timeout) => {} Err(mpsc::RecvTimeoutError::Disconnected) if pending.is_empty() => return, Err(mpsc::RecvTimeoutError::Disconnected) => { - std::thread::sleep(REAPER_POLL_INTERVAL); + std::thread::sleep(REAPER_MAX_BACKOFF); } } while let Ok(request) = receiver.try_recv() { pending.push(request); } - pending.retain_mut(|request| !poll_reap_request(request)); + let now = Instant::now(); + pending.retain_mut(|request| { + if request.next_attempt > now { + return true; + } + if poll_reap_request(request) { + return false; + } + request.next_attempt = now + request.retry_delay; + request.retry_delay = (request.retry_delay * 2).min(REAPER_MAX_BACKOFF); + true + }); } } @@ -262,14 +329,17 @@ fn poll_reap_request(request: &mut ReapRequest) -> bool { REAPER_POLL_ATTEMPTS.fetch_add(1, Ordering::Relaxed); if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { let _ = request.child.kill(); - return false; + if let Some(profile_dir) = request.profile_dir.take() { + let _ = std::fs::remove_dir_all(profile_dir); + } + return true; } if FORCE_REAPER_PENDING.load(Ordering::Acquire) { return false; } } let _ = request.child.kill(); - if !matches!(request.child.try_wait(), Ok(Some(_))) { + if let Ok(None) = request.child.try_wait() { return false; } if let Some(profile_dir) = request.profile_dir.take() { @@ -447,7 +517,7 @@ mod tests { profile_dir, profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: chrome_reaper_handle().unwrap(), + reaper: Some(chrome_reaper_lease().unwrap()), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); @@ -457,6 +527,7 @@ mod tests { #[cfg(unix)] #[test] fn drop_transfers_an_unconfirmed_child_to_a_reaper() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; @@ -475,7 +546,7 @@ mod tests { profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: chrome_reaper_handle().unwrap(), + reaper: Some(chrome_reaper_lease().unwrap()), }; FORCE_KILL_TIMEOUT.set(true); @@ -504,6 +575,7 @@ mod tests { #[cfg(unix)] #[test] fn reaper_spawn_failure_never_waits_on_the_caller_thread() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); let child = Command::new("sleep") .arg("60") .stdin(Stdio::null()) @@ -511,13 +583,13 @@ mod tests { .stderr(Stdio::null()) .spawn() .unwrap(); - let reaper = chrome_reaper_handle().unwrap(); + let reaper = chrome_reaper_lease().unwrap(); let chrome = Chrome { child: Mutex::new(Some(child)), profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: reaper.clone(), + reaper: Some(reaper), }; REAP_CHILD_CALLED_ON_THREAD.set(false); @@ -529,7 +601,7 @@ mod tests { let reaped_inline = REAP_CHILD_CALLED_ON_THREAD.get(); let cleanup = Command::new("true").spawn().unwrap(); - reap_child_detached(&reaper, cleanup, None); + reap_child_detached(chrome_reaper_lease().unwrap(), cleanup, None); assert!(!reaped_inline, "reaper spawn failure fell back to an unbounded caller wait"); } @@ -537,14 +609,14 @@ mod tests { #[cfg(unix)] #[test] fn final_reaper_request_survives_a_shutdown_thread_spawn_failure() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; } - let reaper = chrome_reaper_handle().unwrap(); let warmup = Command::new("true").spawn().unwrap(); - reap_child_detached(&reaper, warmup, None); + reap_child_detached(chrome_reaper_lease().unwrap(), warmup, None); let child = Command::new("sleep") .arg("60") @@ -557,7 +629,7 @@ mod tests { let profile_dir = make_profile_dir().unwrap(); FORCE_REAPER_SPAWN_FAILURE.set(true); - reap_child_detached(&reaper, child, Some(profile_dir.clone())); + reap_child_detached(chrome_reaper_lease().unwrap(), child, Some(profile_dir.clone())); FORCE_REAPER_SPAWN_FAILURE.set(false); let deadline = Instant::now() + Duration::from_secs(1); @@ -591,7 +663,6 @@ mod tests { fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; } - let reaper = chrome_reaper_handle().unwrap(); let child = Command::new("sleep") .arg("60") .stdin(Stdio::null()) @@ -602,7 +673,7 @@ mod tests { let pid = i32::try_from(child.id()).unwrap(); FORCE_REAPER_PENDING.store(true, Ordering::Release); REAPER_POLL_ATTEMPTS.store(0, Ordering::Release); - reap_child_detached(&reaper, child, None); + reap_child_detached(chrome_reaper_lease().unwrap(), child, None); std::thread::sleep(Duration::from_millis(160)); let attempts = REAPER_POLL_ATTEMPTS.load(Ordering::Acquire); FORCE_REAPER_PENDING.store(false, Ordering::Release); @@ -631,7 +702,6 @@ mod tests { fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; } - let reaper = chrome_reaper_handle().unwrap(); let child = Command::new("sleep") .arg("60") .stdin(Stdio::null()) @@ -642,7 +712,7 @@ mod tests { let pid = i32::try_from(child.id()).unwrap(); let profile_dir = make_profile_dir().unwrap(); FORCE_REAPER_WAIT_ERROR.store(true, Ordering::Release); - reap_child_detached(&reaper, child, Some(profile_dir.clone())); + reap_child_detached(chrome_reaper_lease().unwrap(), child, Some(profile_dir.clone())); let deadline = Instant::now() + Duration::from_millis(200); while profile_dir.exists() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); @@ -663,4 +733,22 @@ mod tests { assert!(released, "terminal wait error retained the Chrome profile indefinitely"); } + + #[test] + fn reaper_admission_is_bounded_by_live_leases() { + let (sender, _receiver) = mpsc::channel::(); + let active = Arc::new(AtomicUsize::new(0)); + let first = reserve_reaper_lease(sender.clone(), active.clone(), 2).unwrap(); + let second = reserve_reaper_lease(sender.clone(), active.clone(), 2).unwrap(); + + assert_eq!(active.load(Ordering::Acquire), 2); + assert!(reserve_reaper_lease(sender.clone(), active.clone(), 2).is_err()); + + drop(first); + let replacement = reserve_reaper_lease(sender, active.clone(), 2).unwrap(); + assert_eq!(active.load(Ordering::Acquire), 2); + drop(second); + drop(replacement); + assert_eq!(active.load(Ordering::Acquire), 0); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 67b01a9f7450..f70a3e14cdec 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -54,11 +54,51 @@ const SHUTDOWN_FANOUT_WORKERS: usize = 32; const SHUTDOWN_RECONCILE_INITIAL_DELAY: Duration = Duration::from_millis(25); const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; +#[cfg(unix)] +const TERMINAL_ADOPTION_QUEUE_CAPACITY: usize = SHUTDOWN_OWNER_CAPACITY; #[cfg(not(test))] const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 8; #[cfg(test)] const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 3; +#[cfg(unix)] +struct TerminalAdoptionTask { + options: SurfaceOptions, + record: crate::terminal_host_runtime::TerminalHostRecord, + record_path: std::path::PathBuf, + next_attempt: Instant, + delay: Duration, +} + +#[cfg(unix)] +struct TerminalAdoptionQueueState { + tasks: Vec, + rescan_required: bool, + next_rescan: Option, + rescan_delay: Duration, + worker_running: bool, +} + +#[cfg(unix)] +impl Default for TerminalAdoptionQueueState { + fn default() -> Self { + Self { + tasks: Vec::new(), + rescan_required: false, + next_rescan: None, + rescan_delay: Duration::from_millis(100), + worker_running: false, + } + } +} + +#[cfg(unix)] +#[derive(Default)] +struct TerminalAdoptionCoordinator { + state: Mutex, + wake: std::sync::Condvar, +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] enum ShutdownOwnerKey { Surface(SurfaceId), @@ -1336,6 +1376,8 @@ pub struct Mux { agent_records: Mutex>, surface_notifications: Mutex>, terminal_adoptions: Mutex>, + #[cfg(unix)] + terminal_adoption_coordinator: Arc, /// Fences the interval between accepting a browser daemon-handoff request /// and queueing its acknowledgement. ClientRegistry consults this under /// its own lock so a new native-browser owner cannot race the shutdown. @@ -1581,6 +1623,8 @@ impl Mux { agent_records: Mutex::new(HashMap::new()), surface_notifications: Mutex::new(HashMap::new()), terminal_adoptions: Mutex::new(HashSet::new()), + #[cfg(unix)] + terminal_adoption_coordinator: Arc::new(TerminalAdoptionCoordinator::default()), daemon_handoff_pending: AtomicBool::new(false), shutting_down: AtomicBool::new(false), daemon_shutdown_requested: AtomicBool::new(false), @@ -1592,7 +1636,12 @@ impl Mux { }); mux.shutdown_owner_reconciler.bind(Arc::downgrade(&mux)); #[cfg(unix)] - mux.adopt_terminal_hosts()?; + { + if mux.surface_options.lock().unwrap().terminal_host_root.is_some() { + mux.ensure_terminal_adoption_worker()?; + } + mux.adopt_terminal_hosts()?; + } Ok(mux) } @@ -2039,6 +2088,35 @@ impl Mux { Ok(()) } + #[cfg(unix)] + fn ensure_terminal_adoption_worker(self: &Arc) -> std::io::Result<()> { + let coordinator = self.terminal_adoption_coordinator.clone(); + { + let mut state = coordinator.state.lock().unwrap(); + if state.worker_running { + return Ok(()); + } + state.worker_running = true; + } + let mux = Arc::downgrade(self); + let worker_coordinator = coordinator.clone(); + match std::thread::Builder::new().name("terminal-adoption-coordinator".into()).spawn( + move || { + #[cfg(test)] + if let Some(mux) = mux.upgrade() { + mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); + } + Self::run_terminal_adoption_worker(mux, worker_coordinator); + }, + ) { + Ok(_) => Ok(()), + Err(error) => { + coordinator.state.lock().unwrap().worker_running = false; + Err(error) + } + } + } + #[cfg(unix)] fn schedule_terminal_adoption( self: &Arc, @@ -2046,179 +2124,248 @@ impl Mux { record: crate::terminal_host_runtime::TerminalHostRecord, record_path: std::path::PathBuf, ) { - let terminal_id = record.terminal_id.clone(); - if !self.terminal_adoptions.lock().unwrap().insert(terminal_id.clone()) { + if self.shutting_down.load(Ordering::Acquire) { return; } - let cleanup_id = terminal_id.clone(); - let mux = self.clone(); - let spawn_result = std::thread::Builder::new() - .name(format!("terminal-adopt-{terminal_id}")) - .spawn(move || { - #[cfg(test)] - mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); - let mut delay = Duration::from_millis(100); - loop { - if mux.shutting_down.load(Ordering::Acquire) { - break; - } - std::thread::sleep(delay); - if mux.shutting_down.load(Ordering::Acquire) { - break; - } - let terminal = mux - .workspace_registry - .lock() - .unwrap() - .terminal_record(&terminal_id) - .ok() - .flatten(); - let Some(terminal) = terminal else { - if cleanup_terminal_host_record(&record, &record_path) { - break; - } - delay = (delay * 2).min(Duration::from_secs(5)); - continue; - }; - if matches!( - terminal.lifecycle, - TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned - ) { - if terminal.lifecycle == TerminalLifecycle::Exited { - let _ = mux.materialize_exited_terminal(&terminal_id, &options); - } - if cleanup_terminal_host_record(&record, &record_path) { - break; - } - delay = (delay * 2).min(Duration::from_secs(5)); - continue; - } - if terminal - .incarnation - .as_deref() - .is_some_and(|incarnation| incarnation != record.incarnation) - { - let _ = mux.mark_terminal_exited_and_materialize( - &terminal_id, - "terminal-incarnation-mismatch", - "host-incarnation-mismatch", - &options, - ); - if cleanup_terminal_host_record(&record, &record_path) { - break; - } - delay = (delay * 2).min(Duration::from_secs(5)); - continue; - } - if terminal.lifecycle == TerminalLifecycle::Running { - let already_live = mux - .resolve_terminal(&terminal_id) - .ok() - .flatten() - .is_some_and(|resolution| resolution.surface.is_some()); - if already_live { - break; - } - if mux - .transition_terminal_lifecycle( - "terminal-adopting", - "retry-terminal-adoption", - &terminal_id, - TerminalLifecycle::Adopting, - Some(&record.incarnation), - None, - ) - .is_err() - { - break; - } - } - if terminal_host_record_liveness(&record_path, &record) - == TerminalHostLiveness::Dead - { - let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( - &record_path, - &record, - ); - let _ = mux.mark_terminal_exited_and_materialize( - &terminal_id, - "terminal-host-proven-dead", - "host-process-ended-before-adoption", - &options, - ); - break; - } - let Ok(_creation) = mux.begin_surface_creation() else { - break; - }; - let mut owner_reservation = match mux.reserve_surface_owner() { - Ok(reservation) => reservation, - Err(_) => { - delay = (delay * 2).min(Duration::from_secs(5)); - continue; - } - }; - let id = mux.next_id(); - #[cfg(test)] - let adopted = if let Some(factory) = - mux.terminal_adoption_surface_factory.lock().unwrap().clone() - { - factory(id) - } else { - Surface::adopt_hosted( - id, - options.clone(), - Arc::downgrade(&mux), - record.clone(), - record_path.clone(), - ) - }; - #[cfg(not(test))] - let adopted = Surface::adopt_hosted( - id, - options.clone(), - Arc::downgrade(&mux), - record.clone(), - record_path.clone(), - ); - if let Ok(surface) = adopted { - #[cfg(test)] - if let Some(hook) = - mux.terminal_adoption_after_attach.lock().unwrap().clone() - { - hook(); - } - if mux - .finish_terminal_adoption( - &terminal_id, - &record.incarnation, - surface.clone(), - &mut owner_reservation, - ) - .is_ok() - { - mux.reap_if_dead(&surface); - break; - } - mux.retire_reserved_surface_runtime(surface, &mut owner_reservation); - let _ = mux.mark_terminal_exited_and_materialize( - &terminal_id, - "terminal-adoption-failed", - "host-adoption-topology-failed", - &options, - ); - delay = (delay * 2).min(Duration::from_secs(5)); - continue; - } - delay = (delay * 2).min(Duration::from_secs(5)); + let terminal_id = record.terminal_id.clone(); + { + let mut tracked = self.terminal_adoptions.lock().unwrap(); + if tracked.contains(&terminal_id) { + return; + } + if tracked.len() >= TERMINAL_ADOPTION_QUEUE_CAPACITY { + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + state.rescan_required = true; + state.next_rescan.get_or_insert_with(Instant::now); + self.terminal_adoption_coordinator.wake.notify_one(); + return; + } + tracked.insert(terminal_id.clone()); + } + + self.terminal_adoption_coordinator.state.lock().unwrap().tasks.push(TerminalAdoptionTask { + options, + record, + record_path, + next_attempt: Instant::now() + Duration::from_millis(100), + delay: Duration::from_millis(100), + }); + self.terminal_adoption_coordinator.wake.notify_one(); + if self.ensure_terminal_adoption_worker().is_err() { + self.terminal_adoptions.lock().unwrap().remove(&terminal_id); + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + state.tasks.retain(|task| task.record.terminal_id != terminal_id); + state.rescan_required = true; + state.next_rescan.get_or_insert_with(Instant::now); + } + } + + #[cfg(unix)] + fn rescan_terminal_adoptions(self: &Arc) -> bool { + let options = self.surface_options.lock().unwrap().clone(); + let Some(root) = options.terminal_host_root.as_deref() else { return true }; + let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(root) else { + return false; + }; + for (record_path, record) in records { + let already_live = self + .resolve_terminal(&record.terminal_id) + .ok() + .flatten() + .is_some_and(|resolution| resolution.surface.is_some()); + if already_live { + continue; + } + self.schedule_terminal_adoption(options.clone(), record, record_path); + } + true + } + + #[cfg(unix)] + fn run_terminal_adoption_worker( + mux: Weak, + coordinator: Arc, + ) { + loop { + let Some(mux) = mux.upgrade() else { + coordinator.state.lock().unwrap().worker_running = false; + return; + }; + if mux.shutting_down.load(Ordering::Acquire) { + mux.terminal_adoptions.lock().unwrap().clear(); + let mut state = coordinator.state.lock().unwrap(); + state.tasks.clear(); + state.worker_running = false; + return; + } + + let tracked = mux.terminal_adoptions.lock().unwrap().len(); + let mut state = coordinator.state.lock().unwrap(); + let now = Instant::now(); + let rescan_due = state.next_rescan.is_none_or(|next_rescan| next_rescan <= now); + if state.rescan_required && rescan_due && tracked < TERMINAL_ADOPTION_QUEUE_CAPACITY { + state.rescan_required = false; + state.next_rescan = None; + drop(state); + if !mux.rescan_terminal_adoptions() { + let mut state = coordinator.state.lock().unwrap(); + state.rescan_required = true; + state.next_rescan = Some(Instant::now() + state.rescan_delay); + state.rescan_delay = (state.rescan_delay * 2).min(Duration::from_secs(5)); + } else { + coordinator.state.lock().unwrap().rescan_delay = Duration::from_millis(100); } - mux.terminal_adoptions.lock().unwrap().remove(&terminal_id); - }); - if spawn_result.is_err() { - self.terminal_adoptions.lock().unwrap().remove(&cleanup_id); + continue; + } + + if let Some(index) = state.tasks.iter().position(|task| task.next_attempt <= now) { + let mut task = state.tasks.swap_remove(index); + drop(state); + let complete = mux.try_terminal_adoption(&task); + if complete { + mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); + coordinator.wake.notify_one(); + } else { + task.delay = (task.delay * 2).min(Duration::from_secs(5)); + task.next_attempt = Instant::now() + task.delay; + coordinator.state.lock().unwrap().tasks.push(task); + } + continue; + } + + let wait = state + .tasks + .iter() + .map(|task| task.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(Duration::from_secs(1)) + .min(Duration::from_secs(1)); + drop(mux); + let _ = coordinator.wake.wait_timeout(state, wait).unwrap(); } } + #[cfg(unix)] + fn try_terminal_adoption(self: &Arc, task: &TerminalAdoptionTask) -> bool { + let terminal_id = &task.record.terminal_id; + let terminal = + self.workspace_registry.lock().unwrap().terminal_record(terminal_id).ok().flatten(); + let Some(terminal) = terminal else { + return cleanup_terminal_host_record(&task.record, &task.record_path); + }; + if matches!(terminal.lifecycle, TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned) { + if terminal.lifecycle == TerminalLifecycle::Exited { + let _ = self.materialize_exited_terminal(terminal_id, &task.options); + } + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if terminal + .incarnation + .as_deref() + .is_some_and(|incarnation| incarnation != task.record.incarnation) + { + let _ = self.mark_terminal_exited_and_materialize( + terminal_id, + "terminal-incarnation-mismatch", + "host-incarnation-mismatch", + &task.options, + ); + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if terminal.lifecycle == TerminalLifecycle::Running { + let already_live = self + .resolve_terminal(terminal_id) + .ok() + .flatten() + .is_some_and(|resolution| resolution.surface.is_some()); + if already_live { + return true; + } + if self + .transition_terminal_lifecycle( + "terminal-adopting", + "retry-terminal-adoption", + terminal_id, + TerminalLifecycle::Adopting, + Some(&task.record.incarnation), + None, + ) + .is_err() + { + return true; + } + } + if terminal_host_record_liveness(&task.record_path, &task.record) + == TerminalHostLiveness::Dead + { + let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( + &task.record_path, + &task.record, + ); + let _ = self.mark_terminal_exited_and_materialize( + terminal_id, + "terminal-host-proven-dead", + "host-process-ended-before-adoption", + &task.options, + ); + return true; + } + let Ok(_creation) = self.begin_surface_creation() else { + return true; + }; + let mut owner_reservation = match self.reserve_surface_owner() { + Ok(reservation) => reservation, + Err(_) => return false, + }; + let id = self.next_id(); + #[cfg(test)] + let adopted = + if let Some(factory) = self.terminal_adoption_surface_factory.lock().unwrap().clone() { + factory(id) + } else { + Surface::adopt_hosted( + id, + task.options.clone(), + Arc::downgrade(self), + task.record.clone(), + task.record_path.clone(), + ) + }; + #[cfg(not(test))] + let adopted = Surface::adopt_hosted( + id, + task.options.clone(), + Arc::downgrade(self), + task.record.clone(), + task.record_path.clone(), + ); + let Ok(surface) = adopted else { return false }; + #[cfg(test)] + if let Some(hook) = self.terminal_adoption_after_attach.lock().unwrap().clone() { + hook(); + } + if self + .finish_terminal_adoption( + terminal_id, + &task.record.incarnation, + surface.clone(), + &mut owner_reservation, + ) + .is_ok() + { + self.reap_if_dead(&surface); + return true; + } + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + let _ = self.mark_terminal_exited_and_materialize( + terminal_id, + "terminal-adoption-failed", + "host-adoption-topology-failed", + &task.options, + ); + false + } + #[cfg(test)] pub(crate) fn new_for_test( session: impl Into, @@ -4489,6 +4636,8 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); + #[cfg(unix)] + self.terminal_adoption_coordinator.wake.notify_all(); let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; let _ = self.surface_creations.stop_and_wait_until(deadline); @@ -4551,6 +4700,8 @@ impl Mux { let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; self.shutting_down.store(true, Ordering::Release); + #[cfg(unix)] + self.terminal_adoption_coordinator.wake.notify_all(); if !self.surface_creations.stop_and_wait_until(deadline) { anyhow::bail!("surface creation did not stop before the shutdown deadline"); } @@ -4779,6 +4930,8 @@ impl Mux { self.surface_creations.stop(); self.async_surface_creations.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); + #[cfg(unix)] + self.terminal_adoption_coordinator.wake.notify_all(); } pub fn daemon_shutdown_requested(&self) -> bool { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 495d478c5c27..019c5fb3de33 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -997,15 +997,16 @@ mod unix { let child = command.spawn().context("spawn terminal-host process")?; let mut process = SpawnedHostProcess { child: Some(child) }; let host_pid = process.child_mut().id(); - let mut stdin = + let stdin = process.child_mut().stdin.take().context("open terminal-host bootstrap stdin")?; let stdout = process.child_mut().stdout.take().context("open terminal-host bootstrap stdout")?; - let mut stdout = CancellableHostReader { - reader: stdout, - deadline: Instant::now() + HOST_LAUNCH_TIMEOUT, - cancelled, - }; + set_nonblocking(stdin.as_raw_fd())?; + let launch_deadline = Instant::now() + HOST_LAUNCH_TIMEOUT; + let mut stdin = + CancellableHostWriter { writer: stdin, deadline: launch_deadline, cancelled }; + let mut stdout = + CancellableHostReader { reader: stdout, deadline: launch_deadline, cancelled }; let bootstrap = HostBootstrap { min_version: PROTOCOL_VERSION, @@ -1082,6 +1083,88 @@ mod unix { Ok(attachment) } + fn set_nonblocking(descriptor: RawFd) -> std::io::Result<()> { + // SAFETY: F_GETFL reads flags from the valid owned pipe descriptor. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFL) }; + if flags < 0 { + return Err(std::io::Error::last_os_error()); + } + if flags & libc::O_NONBLOCK != 0 { + return Ok(()); + } + // SAFETY: F_SETFL updates status flags on the same valid descriptor. + if unsafe { libc::fcntl(descriptor, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } + + struct CancellableHostWriter<'a, W> { + writer: W, + deadline: Instant, + cancelled: &'a dyn Fn() -> bool, + } + + impl Write for CancellableHostWriter<'_, W> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + if buffer.is_empty() { + return Ok(0); + } + loop { + if (self.cancelled)() { + return Err(std::io::Error::new( + std::io::ErrorKind::ConnectionAborted, + "terminal host launch cancelled during server shutdown", + )); + } + let Some(remaining) = self.deadline.checked_duration_since(Instant::now()) else { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "terminal host launch timed out", + )); + }; + let wait = remaining.min(HOST_LAUNCH_CANCEL_POLL); + let timeout_ms = wait.as_millis().max(1).min(i32::MAX as u128) as i32; + let mut descriptor = libc::pollfd { + fd: self.writer.as_raw_fd(), + events: libc::POLLOUT | libc::POLLHUP, + revents: 0, + }; + let result = unsafe { libc::poll(&mut descriptor, 1, timeout_ms) }; + if result > 0 { + if descriptor.revents & libc::POLLNVAL != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "terminal host bootstrap pipe became invalid", + )); + } + match self.writer.write(buffer) { + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::Interrupted + ) => + { + continue; + } + result => return result, + } + } + if result == 0 { + continue; + } + let error = std::io::Error::last_os_error(); + if error.kind() != std::io::ErrorKind::Interrupted { + return Err(error); + } + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } + } + struct CancellableHostReader<'a, R> { reader: R, deadline: Instant, From 98e2bd9d0ce462b7a53552496fae2f7c2df86254 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:38:00 -0700 Subject: [PATCH 076/521] test(tui): expose final scalability regressions --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 210 +++++++++++++++++++++++ tests/test_tui_package_launchers.py | 8 + 2 files changed, 218 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f70a3e14cdec..049932cae0ad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -11472,6 +11472,108 @@ mod tests { assert!(workers <= 1, "overflow adoption started {workers} retry threads"); } + #[cfg(unix)] + #[test] + fn blocked_terminal_adoption_does_not_stall_an_unrelated_terminal() { + let options = SurfaceOptions::default(); + let mux = Mux::new_for_test("adoption-parallel-progress", options.clone()); + let workspace = + mux.create_empty_workspace(Some("parallel-progress".into()), None, None).unwrap(); + let root = std::env::temp_dir().join(format!( + "cmux-adoption-parallel-progress-{}", + crate::workspace_registry::new_uuid_v4() + )); + let mut records = Vec::new(); + { + let mut registry = mux.workspace_registry.lock().unwrap(); + for index in 0..2 { + let terminal_id = TerminalId::random().unwrap().to_hex(); + let incarnation = crate::terminal_host::HostIncarnation::random().unwrap().to_hex(); + let record_path = root.join(format!("{terminal_id}.json")); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + &format!("parallel-adoption-reserve-{index}"), + &RegistryTerminal { + terminal_id: terminal_id.clone(), + workspace_key: workspace.key.clone(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({}), + exit: None, + }, + ) + .unwrap(); + commit_terminal_lifecycle( + &mut registry, + "terminal-adopting", + &format!("parallel-adoption-start-{index}"), + &terminal_id, + TerminalLifecycle::Adopting, + Some(&incarnation), + None, + ) + .unwrap(); + records.push(( + crate::terminal_host_runtime::TerminalHostRecord { + record_version: 2, + terminal_id, + incarnation, + endpoint: record_path.with_extension("sock").to_string_lossy().into_owned(), + owner_token: "00".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: workspace.key.clone(), + supports_set_defaults: true, + supports_terminate_only: true, + }, + record_path, + )); + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let (first_started_tx, first_started_rx) = std::sync::mpsc::sync_channel(1); + let (second_started_tx, second_started_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let release_rx = Arc::new(Mutex::new(release_rx)); + *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ + let mux = Arc::downgrade(&mux); + let options = options.clone(); + let calls = calls.clone(); + move |id| { + let call = calls.fetch_add(1, Ordering::AcqRel); + if call == 0 { + first_started_tx.send(()).unwrap(); + release_rx.lock().unwrap().recv().unwrap(); + } else { + second_started_tx.send(()).unwrap(); + } + Surface::spawn_for_test(id, options.clone(), mux.clone()) + } + })); + + let (first_record, first_path) = records.remove(0); + mux.schedule_terminal_adoption(options.clone(), first_record, first_path); + first_started_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let (second_record, second_path) = records.remove(0); + mux.schedule_terminal_adoption(options, second_record, second_path); + let second_progressed = second_started_rx.recv_timeout(Duration::from_millis(300)).is_ok(); + release_tx.send(()).unwrap(); + + let deadline = Instant::now() + Duration::from_secs(2); + while !mux.terminal_adoptions.lock().unwrap().is_empty() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + mux.shutdown(); + let _ = std::fs::remove_dir_all(root); + + assert!( + second_progressed, + "one blocked terminal adoption stalled an unrelated queued terminal" + ); + } + #[cfg(unix)] #[test] fn daemon_handoff_does_not_terminate_an_in_flight_terminal_adoption() { @@ -11648,6 +11750,114 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn shutdown_batches_target_discovery_per_browser_runtime() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (query_count_tx, query_count_rx) = std::sync::mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + + let mut query_count = 0; + let mut closed_targets = HashSet::new(); + while closed_targets.len() < 2 { + let request = read_ws_json(&mut ws); + match request["method"].as_str().unwrap() { + "Target.getTargets" => { + query_count += 1; + ws.send(tungstenite::Message::Text( + serde_json::json!({ + "id": request["id"], + "result": { + "targetInfos": [ + {"targetId": "target-1"}, + {"targetId": "target-2"} + ] + } + }) + .to_string() + .into(), + )) + .unwrap(); + } + "Target.closeTarget" => { + closed_targets + .insert(request["params"]["targetId"].as_str().unwrap().to_string()); + ws.send(tungstenite::Message::Text( + serde_json::json!({ + "id": request["id"], + "result": {"success": true} + }) + .to_string() + .into(), + )) + .unwrap(); + } + method => panic!("unexpected browser shutdown request: {method}"), + } + } + query_count_tx.send(query_count).unwrap(); + }); + let runtime = BrowserRuntime::connect_external_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + let mux = test_mux(); + let options = mux.surface_options.lock().unwrap().clone(); + for (id, target_id, session_id) in + [(999, "target-1", "session-1"), (1000, "target-2", "session-2")] + { + let surface = browser::new_surface( + id, + "about:blank".to_string(), + (80, 24), + (8, 16), + &options, + Arc::downgrade(&mux), + ); + surface.as_browser().unwrap().install_shutdown_session_for_test( + runtime.clone(), + target_id, + session_id, + ); + assert!(mux.shutdown_owners.stage_surface(&surface).is_some()); + } + + let failed = + mux.terminate_staged_shutdown_owners_until(Instant::now() + Duration::from_secs(1)); + let query_count = query_count_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + runtime.shutdown(); + server.join().unwrap(); + assert!(!failed); + assert!(mux.shutdown_owners.is_empty()); + assert_eq!( + query_count, 1, + "shutdown repeated full target discovery for each browser surface" + ); + } + #[test] fn server_shutdown_uses_launched_runtime_when_target_confirmation_is_unreachable() { fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index c2fa2cc0fcf1..7e1f2e923270 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -2,6 +2,7 @@ import os import platform +import runpy import shutil import subprocess import sys @@ -37,6 +38,12 @@ ) +def test_pypi_fixture_targets_match_packager() -> None: + packager = runpy.run_path(str(PYPI_PACKAGER)) + declared = tuple(target.rust_target for target in packager["TARGETS"]) + assert RUST_TARGETS == declared + + def normalized_platform() -> tuple[str, str]: machine = platform.machine().lower() if machine in {"amd64", "x64"}: @@ -187,6 +194,7 @@ def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: def main() -> None: import tempfile + test_pypi_fixture_targets_match_packager() with tempfile.TemporaryDirectory() as directory: test_identity_verifier_executes_artifact_and_rejects_mismatch( Path(directory) From 7ca549b0677ea98804a7b0a7fb8c6802063cd228 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 07:50:56 -0700 Subject: [PATCH 077/521] fix(tui): batch bounded lifecycle cleanup --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 7 +- cmux-tui/crates/cmux-tui-core/src/browser.rs | 144 +++++++--- cmux-tui/crates/cmux-tui-core/src/mux.rs | 260 +++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 9 + tests/test_tui_package_launchers.py | 4 +- 5 files changed, 322 insertions(+), 102 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 939ed4482d30..bc1f7db80dd8 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -547,6 +547,10 @@ impl CdpClient { target_id: &str, timeout: Duration, ) -> anyhow::Result { + Ok(self.target_ids_with_timeout(timeout)?.iter().any(|candidate| candidate == target_id)) + } + + pub fn target_ids_with_timeout(&self, timeout: Duration) -> anyhow::Result> { let result = self.call_with_timeout("Target.getTargets", json!({}), None, timeout)?; let targets = result .get("targetInfos") @@ -554,7 +558,8 @@ impl CdpClient { .ok_or_else(|| anyhow::anyhow!("Target.getTargets response missing targetInfos"))?; Ok(targets .iter() - .any(|target| target.get("targetId").and_then(Value::as_str) == Some(target_id))) + .filter_map(|target| target.get("targetId").and_then(Value::as_str).map(str::to_string)) + .collect()) } pub fn close_target_detached(&self, target_id: &str) -> anyhow::Result<()> { diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 1a8916608f0b..825d357b5bb9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, Sender, SyncSender, TrySendError, sync_channel}; @@ -278,12 +278,6 @@ struct QueuedSurfaceEvent { retained_bytes: usize, } -enum ExternalTargetShutdown { - Closed, - StillOpen, - Unreachable, -} - impl SurfaceRoute { fn new() -> Self { Self { state: Mutex::new(SurfaceRouteState::default()), ready: Condvar::new() } @@ -562,64 +556,128 @@ impl BrowserRuntime { session_id: &str, deadline: Instant, ) -> bool { - self.unregister(target_id, session_id); - if !self.is_closed() { - match Self::close_external_target_until(&self.client, target_id, deadline) { - ExternalTargetShutdown::Closed => return true, - ExternalTargetShutdown::StillOpen => return false, - ExternalTargetShutdown::Unreachable => {} - } + self.close_surfaces_for_shutdown(&[(target_id, session_id)], deadline) + .into_iter() + .next() + .unwrap_or(true) + } + + pub(crate) fn close_owners_for_shutdown( + &self, + owners: &[&BrowserShutdownOwner], + deadline: Instant, + ) -> Vec { + debug_assert!(owners.iter().all(|owner| std::ptr::eq(self, owner.runtime().as_ref()))); + let surfaces = owners + .iter() + .map(|owner| (owner.0.target_id.as_str(), owner.0.session_id.as_str())) + .collect::>(); + self.close_surfaces_for_shutdown(&surfaces, deadline) + } + + fn close_surfaces_for_shutdown( + &self, + surfaces: &[(&str, &str)], + deadline: Instant, + ) -> Vec { + for (target_id, session_id) in surfaces { + self.unregister(target_id, session_id); + } + if surfaces.is_empty() { + return Vec::new(); + } + let targets = surfaces.iter().map(|(target_id, _)| *target_id).collect::>(); + let mut outcomes = if self.is_closed() { + vec![None; targets.len()] + } else { + Self::close_external_targets_until(&self.client, &targets, deadline) + }; + let unresolved = outcomes + .iter() + .enumerate() + .filter_map(|(index, outcome)| outcome.is_none().then_some(index)) + .collect::>(); + if unresolved.is_empty() { + return outcomes.into_iter().map(|outcome| outcome.unwrap_or(false)).collect(); } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return false; + return outcomes.into_iter().map(|outcome| outcome.unwrap_or(false)).collect(); }; let (event_tx, _event_rx) = sync_channel(CDP_EVENT_QUEUE_CAPACITY); let Ok(client) = self.client.reconnect_with_timeout(event_tx, remaining) else { - return false; + return outcomes.into_iter().map(|outcome| outcome.unwrap_or(false)).collect(); }; - matches!( - Self::close_external_target_until(&client, target_id, deadline), - ExternalTargetShutdown::Closed - ) + let retry_targets = unresolved.iter().map(|index| targets[*index]).collect::>(); + let retry_outcomes = Self::close_external_targets_until(&client, &retry_targets, deadline); + for (index, outcome) in unresolved.into_iter().zip(retry_outcomes) { + outcomes[index] = outcome; + } + outcomes.into_iter().map(|outcome| outcome.unwrap_or(false)).collect() } - fn close_external_target_until( + fn close_external_targets_until( client: &CdpClient, - target_id: &str, + target_ids: &[&str], deadline: Instant, - ) -> ExternalTargetShutdown { + ) -> Vec> { + let mut outcomes = vec![None; target_ids.len()]; let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return ExternalTargetShutdown::Unreachable; + return outcomes; }; let query_timeout = remaining / 3; if query_timeout.is_zero() { - return ExternalTargetShutdown::Unreachable; + return outcomes; } - match client.target_exists_with_timeout(target_id, query_timeout) { - Ok(false) => return ExternalTargetShutdown::Closed, - Ok(true) => {} - Err(_) => return ExternalTargetShutdown::Unreachable, - } - - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return ExternalTargetShutdown::StillOpen; + let Ok(open_targets) = client.target_ids_with_timeout(query_timeout) else { + return outcomes; }; - let close_timeout = remaining / 3; - if close_timeout.is_zero() { - return ExternalTargetShutdown::StillOpen; + let open_targets = open_targets.into_iter().collect::>(); + let pending = target_ids + .iter() + .enumerate() + .filter_map(|(index, target_id)| { + if open_targets.contains(*target_id) { + Some(index) + } else { + outcomes[index] = Some(true); + None + } + }) + .collect::>(); + let mut ambiguous = Vec::new(); + for (position, index) in pending.iter().copied().enumerate() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + ambiguous.extend_from_slice(&pending[position..]); + break; + }; + let slots = u32::try_from(pending.len() - position + 1).unwrap_or(u32::MAX); + let close_timeout = remaining / slots; + if close_timeout.is_zero() { + ambiguous.push(index); + continue; + } + if client.close_target_with_timeout(target_ids[index], close_timeout).is_ok() { + outcomes[index] = Some(true); + } else { + ambiguous.push(index); + } } - if client.close_target_with_timeout(target_id, close_timeout).is_ok() { - return ExternalTargetShutdown::Closed; + if ambiguous.is_empty() { + return outcomes; } let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return ExternalTargetShutdown::Unreachable; + return outcomes; + }; + let Ok(open_targets) = client.target_ids_with_timeout(remaining) else { + return outcomes; }; - match client.target_exists_with_timeout(target_id, remaining) { - Ok(false) => ExternalTargetShutdown::Closed, - Ok(true) => ExternalTargetShutdown::StillOpen, - Err(_) => ExternalTargetShutdown::Unreachable, + let open_targets = open_targets.into_iter().collect::>(); + for index in ambiguous { + outcomes[index] = Some(!open_targets.contains(target_ids[index])); } + outcomes } pub fn shutdown(&self) { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 049932cae0ad..e68861a8789c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -56,6 +56,8 @@ const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; #[cfg(unix)] const TERMINAL_ADOPTION_QUEUE_CAPACITY: usize = SHUTDOWN_OWNER_CAPACITY; +#[cfg(unix)] +const TERMINAL_ADOPTION_WORKERS: usize = 4; #[cfg(not(test))] const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 8; #[cfg(test)] @@ -76,7 +78,8 @@ struct TerminalAdoptionQueueState { rescan_required: bool, next_rescan: Option, rescan_delay: Duration, - worker_running: bool, + workers_running: usize, + stopping: bool, } #[cfg(unix)] @@ -87,7 +90,8 @@ impl Default for TerminalAdoptionQueueState { rescan_required: false, next_rescan: None, rescan_delay: Duration::from_millis(100), - worker_running: false, + workers_running: 0, + stopping: false, } } } @@ -105,6 +109,27 @@ enum ShutdownOwnerKey { Hosted { terminal_id: String, incarnation: String }, } +enum ShutdownOwnerWork { + Single((ShutdownOwnerKey, Arc)), + BrowserBatch { + runtime: Arc, + owners: Vec<(ShutdownOwnerKey, Arc)>, + }, +} + +impl ShutdownOwnerWork { + fn record_failed_keys(&self, failed: &mut HashSet) { + match self { + Self::Single((key, _)) => { + failed.insert(key.clone()); + } + Self::BrowserBatch { owners, .. } => { + failed.extend(owners.iter().map(|(key, _)| key.clone())); + } + } + } +} + fn shutdown_owner_key(surface: SurfaceId, owner: &SurfaceShutdownOwner) -> ShutdownOwnerKey { owner .hosted_identity() @@ -1638,7 +1663,7 @@ impl Mux { #[cfg(unix)] { if mux.surface_options.lock().unwrap().terminal_host_root.is_some() { - mux.ensure_terminal_adoption_worker()?; + mux.ensure_terminal_adoption_workers()?; } mux.adopt_terminal_hosts()?; } @@ -2089,32 +2114,53 @@ impl Mux { } #[cfg(unix)] - fn ensure_terminal_adoption_worker(self: &Arc) -> std::io::Result<()> { + fn ensure_terminal_adoption_workers(self: &Arc) -> std::io::Result<()> { let coordinator = self.terminal_adoption_coordinator.clone(); - { + let (existing, missing) = { let mut state = coordinator.state.lock().unwrap(); - if state.worker_running { + if state.stopping { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "terminal adoption is stopping", + )); + } + if state.workers_running >= TERMINAL_ADOPTION_WORKERS { return Ok(()); } - state.worker_running = true; - } - let mux = Arc::downgrade(self); - let worker_coordinator = coordinator.clone(); - match std::thread::Builder::new().name("terminal-adoption-coordinator".into()).spawn( - move || { - #[cfg(test)] - if let Some(mux) = mux.upgrade() { - mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); + let existing = state.workers_running; + let missing = TERMINAL_ADOPTION_WORKERS - existing; + state.workers_running += missing; + (existing, missing) + }; + let mut started = 0; + let mut first_error = None; + for worker in 0..missing { + let worker = existing + worker; + let mux = Arc::downgrade(self); + let worker_coordinator = coordinator.clone(); + match std::thread::Builder::new().name(format!("terminal-adoption-{worker}")).spawn( + move || { + #[cfg(test)] + if let Some(mux) = mux.upgrade() { + mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); + } + Self::run_terminal_adoption_worker(mux, worker_coordinator); + }, + ) { + Ok(_) => started += 1, + Err(error) => { + let mut state = coordinator.state.lock().unwrap(); + state.workers_running = state.workers_running.saturating_sub(1); + coordinator.wake.notify_all(); + first_error.get_or_insert(error); } - Self::run_terminal_adoption_worker(mux, worker_coordinator); - }, - ) { - Ok(_) => Ok(()), - Err(error) => { - coordinator.state.lock().unwrap().worker_running = false; - Err(error) } } + if existing + started > 0 { + Ok(()) + } else { + Err(first_error.expect("at least one worker spawn failed")) + } } #[cfg(unix)] @@ -2151,7 +2197,7 @@ impl Mux { delay: Duration::from_millis(100), }); self.terminal_adoption_coordinator.wake.notify_one(); - if self.ensure_terminal_adoption_worker().is_err() { + if self.ensure_terminal_adoption_workers().is_err() { self.terminal_adoptions.lock().unwrap().remove(&terminal_id); let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); state.tasks.retain(|task| task.record.terminal_id != terminal_id); @@ -2187,15 +2233,19 @@ impl Mux { coordinator: Arc, ) { loop { + let stopping = coordinator.state.lock().unwrap().stopping; + if stopping { + Self::terminal_adoption_worker_stopped(&coordinator); + return; + } let Some(mux) = mux.upgrade() else { - coordinator.state.lock().unwrap().worker_running = false; + Self::terminal_adoption_worker_stopped(&coordinator); return; }; if mux.shutting_down.load(Ordering::Acquire) { mux.terminal_adoptions.lock().unwrap().clear(); - let mut state = coordinator.state.lock().unwrap(); - state.tasks.clear(); - state.worker_running = false; + drop(mux); + Self::terminal_adoption_worker_stopped(&coordinator); return; } @@ -2225,6 +2275,8 @@ impl Mux { if complete { mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); coordinator.wake.notify_one(); + } else if mux.shutting_down.load(Ordering::Acquire) { + mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); } else { task.delay = (task.delay * 2).min(Duration::from_secs(5)); task.next_attempt = Instant::now() + task.delay; @@ -2245,6 +2297,39 @@ impl Mux { } } + #[cfg(unix)] + fn terminal_adoption_worker_stopped(coordinator: &TerminalAdoptionCoordinator) { + let mut state = coordinator.state.lock().unwrap(); + state.workers_running = state.workers_running.saturating_sub(1); + coordinator.wake.notify_all(); + } + + #[cfg(unix)] + fn request_terminal_adoption_stop(&self) { + self.terminal_adoptions.lock().unwrap().clear(); + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + state.stopping = true; + state.tasks.clear(); + self.terminal_adoption_coordinator.wake.notify_all(); + } + + #[cfg(unix)] + fn wait_for_terminal_adoption_workers_until(&self, deadline: Instant) -> bool { + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + while state.workers_running != 0 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = + self.terminal_adoption_coordinator.wake.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.workers_running != 0 { + return false; + } + } + true + } + #[cfg(unix)] fn try_terminal_adoption(self: &Arc, task: &TerminalAdoptionTask) -> bool { let terminal_id = &task.record.terminal_id; @@ -2319,18 +2404,20 @@ impl Mux { }; let id = self.next_id(); #[cfg(test)] - let adopted = - if let Some(factory) = self.terminal_adoption_surface_factory.lock().unwrap().clone() { - factory(id) - } else { - Surface::adopt_hosted( - id, - task.options.clone(), - Arc::downgrade(self), - task.record.clone(), - task.record_path.clone(), - ) - }; + let adoption_surface_factory = + self.terminal_adoption_surface_factory.lock().unwrap().clone(); + #[cfg(test)] + let adopted = if let Some(factory) = adoption_surface_factory { + factory(id) + } else { + Surface::adopt_hosted( + id, + task.options.clone(), + Arc::downgrade(self), + task.record.clone(), + task.record_path.clone(), + ) + }; #[cfg(not(test))] let adopted = Surface::adopt_hosted( id, @@ -2341,7 +2428,9 @@ impl Mux { ); let Ok(surface) = adopted else { return false }; #[cfg(test)] - if let Some(hook) = self.terminal_adoption_after_attach.lock().unwrap().clone() { + let after_attach = self.terminal_adoption_after_attach.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = after_attach { hook(); } if self @@ -4602,18 +4691,62 @@ impl Mux { deadline, ) .ok(); + let mut single_work = Vec::new(); + let mut browser_batches = HashMap::< + *const BrowserRuntime, + (Arc, Vec<(ShutdownOwnerKey, Arc)>), + >::new(); + for (key, owner) in &owners { + if let Some(runtime) = owner.browser_runtime() { + browser_batches + .entry(Arc::as_ptr(runtime)) + .or_insert_with(|| (runtime.clone(), Vec::new())) + .1 + .push((key.clone(), owner.clone())); + } else { + single_work.push(ShutdownOwnerWork::Single((key.clone(), owner.clone()))); + } + } + // Start each high-fan-in browser batch before individual process work, + // so a saturated fanout cannot strand every target behind the deadline. + let mut work = browser_batches + .into_values() + .map(|(runtime, owners)| ShutdownOwnerWork::BrowserBatch { runtime, owners }) + .collect::>(); + work.extend(single_work); + let failed = Mutex::new(HashSet::new()); - let started = bounded_shutdown_fanout(&owners, deadline, |(key, owner), deadline| { - #[cfg(unix)] - let terminated = owner.terminate_until_in_batch(deadline, process_snapshot.as_ref()); - #[cfg(not(unix))] - let terminated = owner.terminate_until(deadline); - if !terminated { - failed.lock().unwrap().insert(key.clone()); + let started = bounded_shutdown_fanout(&work, deadline, |work, deadline| match work { + ShutdownOwnerWork::Single((key, owner)) => { + #[cfg(unix)] + let terminated = + owner.terminate_until_in_batch(deadline, process_snapshot.as_ref()); + #[cfg(not(unix))] + let terminated = owner.terminate_until(deadline); + if !terminated { + failed.lock().unwrap().insert(key.clone()); + } + } + ShutdownOwnerWork::BrowserBatch { runtime, owners } => { + let browser_owners = owners + .iter() + .map(|(_, owner)| { + owner.browser_shutdown_owner().expect("browser batch has browser owners") + }) + .collect::>(); + let outcomes = runtime.close_owners_for_shutdown(&browser_owners, deadline); + let mut failed = failed.lock().unwrap(); + for (index, (key, _)) in owners.iter().enumerate() { + if !outcomes.get(index).copied().unwrap_or(false) { + failed.insert(key.clone()); + } + } } }); let mut failed = failed.into_inner().unwrap(); - failed.extend(owners[started..].iter().map(|(key, _)| key.clone())); + for work in &work[started..] { + work.record_failed_keys(&mut failed); + } let confirmed = owners.into_iter().filter(|(key, _)| !failed.contains(key)).collect::>(); self.shutdown_owners.remove_confirmed(&confirmed); @@ -4637,11 +4770,13 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); #[cfg(unix)] - self.terminal_adoption_coordinator.wake.notify_all(); + self.request_terminal_adoption_stop(); let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; let _ = self.surface_creations.stop_and_wait_until(deadline); let _ = self.async_surface_creations.stop_and_wait_until(deadline); + #[cfg(unix)] + let _ = self.wait_for_terminal_adoption_workers_until(deadline); let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); for surface in &surfaces { surface.disconnect_for_daemon_shutdown(); @@ -4701,13 +4836,17 @@ impl Mux { let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; self.shutting_down.store(true, Ordering::Release); #[cfg(unix)] - self.terminal_adoption_coordinator.wake.notify_all(); + self.request_terminal_adoption_stop(); if !self.surface_creations.stop_and_wait_until(deadline) { anyhow::bail!("surface creation did not stop before the shutdown deadline"); } if !self.async_surface_creations.stop_and_wait_until(deadline) { anyhow::bail!("browser bootstrap did not stop before the shutdown deadline"); } + #[cfg(unix)] + if !self.wait_for_terminal_adoption_workers_until(deadline) { + anyhow::bail!("terminal adoption did not stop before the shutdown deadline"); + } #[cfg(unix)] let terminal_host_records = { @@ -4931,7 +5070,7 @@ impl Mux { self.async_surface_creations.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); #[cfg(unix)] - self.terminal_adoption_coordinator.wake.notify_all(); + self.request_terminal_adoption_stop(); } pub fn daemon_shutdown_requested(&self) -> bool { @@ -11432,7 +11571,7 @@ mod tests { #[cfg(unix)] #[test] - fn terminal_adoption_overflow_uses_one_shared_worker() { + fn terminal_adoption_overflow_uses_one_bounded_worker_pool() { let options = SurfaceOptions::default(); let mux = Mux::new_for_test("adoption-worker-bound", options.clone()); let root = @@ -11459,17 +11598,27 @@ mod tests { ); } - let deadline = Instant::now() + Duration::from_millis(250); - while mux.terminal_adoption_workers_started.load(Ordering::Acquire) < 3 + let deadline = Instant::now() + Duration::from_secs(1); + while mux.terminal_adoption_workers_started.load(Ordering::Acquire) + < TERMINAL_ADOPTION_WORKERS && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); } let workers = mux.terminal_adoption_workers_started.load(Ordering::Acquire); + let workers_running = + mux.terminal_adoption_coordinator.state.lock().unwrap().workers_running; mux.request_daemon_shutdown(); let _ = std::fs::remove_dir_all(root); - assert!(workers <= 1, "overflow adoption started {workers} retry threads"); + assert_eq!( + workers, TERMINAL_ADOPTION_WORKERS, + "adoption did not preflight its fixed worker pool" + ); + assert!( + workers_running <= TERMINAL_ADOPTION_WORKERS, + "overflow adoption retained {workers_running} concurrent retry threads" + ); } #[cfg(unix)] @@ -11540,7 +11689,6 @@ mod tests { *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ let mux = Arc::downgrade(&mux); let options = options.clone(); - let calls = calls.clone(); move |id| { let call = calls.fetch_add(1, Ordering::AcqRel); if call == 0 { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 2de4806280bc..04abe271a77f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -755,6 +755,15 @@ impl SurfaceShutdownOwner { } } + pub(crate) fn browser_shutdown_owner(&self) -> Option<&BrowserShutdownOwner> { + match &self.kind { + SurfaceShutdownOwnerKind::Browser(owner) => Some(owner), + SurfaceShutdownOwnerKind::Local(_) => None, + #[cfg(unix)] + SurfaceShutdownOwnerKind::Hosted(_) => None, + } + } + #[cfg(unix)] pub(crate) fn hosted( record: crate::terminal_host_runtime::TerminalHostRecord, diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index 7e1f2e923270..2aa7c28de28f 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -33,8 +33,8 @@ RUST_TARGETS = ( "aarch64-apple-darwin", "x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", ) From 8de2e95ea86720473fd34fffc95dbd53fa7d2190 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 08:34:01 -0700 Subject: [PATCH 078/521] test(tui): expose lifecycle retry and framing gaps --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 9 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 119 ++++++++++++++++++ .../src/terminal_host_runtime.rs | 74 +++++++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 76 +++++++++++ 4 files changed, 274 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 5205cd063a26..3baed026e084 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -695,7 +695,7 @@ mod tests { #[cfg(unix)] #[test] - fn terminal_reaper_wait_error_releases_the_profile() { + fn terminal_reaper_wait_error_retains_ownership_for_retry() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; @@ -712,12 +712,13 @@ mod tests { let pid = i32::try_from(child.id()).unwrap(); let profile_dir = make_profile_dir().unwrap(); FORCE_REAPER_WAIT_ERROR.store(true, Ordering::Release); + REAPER_POLL_ATTEMPTS.store(0, Ordering::Release); reap_child_detached(chrome_reaper_lease().unwrap(), child, Some(profile_dir.clone())); let deadline = Instant::now() + Duration::from_millis(200); - while profile_dir.exists() && Instant::now() < deadline { + while REAPER_POLL_ATTEMPTS.load(Ordering::Acquire) == 0 && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); } - let released = !profile_dir.exists(); + let retained = profile_dir.exists(); FORCE_REAPER_WAIT_ERROR.store(false, Ordering::Release); let cleanup_deadline = Instant::now() + Duration::from_secs(1); @@ -731,7 +732,7 @@ mod tests { } let _ = std::fs::remove_dir_all(&profile_dir); - assert!(released, "terminal wait error retained the Chrome profile indefinitely"); + assert!(retained, "terminal wait error discarded Chrome cleanup ownership"); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 04abe271a77f..d0bc9c0dea83 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -790,6 +790,14 @@ struct LocalPtyProcess { child_reaped: AtomicBool, #[cfg(unix)] group_escalation_complete: AtomicBool, + #[cfg(all(unix, test))] + normal_cleanup_failures: AtomicUsize, + #[cfg(all(unix, test))] + normal_cleanup_attempts: AtomicUsize, + #[cfg(all(unix, test))] + normal_cleanup_delay_ms: AtomicU64, + #[cfg(all(unix, test))] + normal_cleanup_started: AtomicBool, } impl LocalPtyProcess { @@ -813,6 +821,14 @@ impl LocalPtyProcess { child_reaped: AtomicBool::new(false), #[cfg(unix)] group_escalation_complete: AtomicBool::new(false), + #[cfg(all(unix, test))] + normal_cleanup_failures: AtomicUsize::new(0), + #[cfg(all(unix, test))] + normal_cleanup_attempts: AtomicUsize::new(0), + #[cfg(all(unix, test))] + normal_cleanup_delay_ms: AtomicU64::new(0), + #[cfg(all(unix, test))] + normal_cleanup_started: AtomicBool::new(false), }) } @@ -999,6 +1015,24 @@ impl LocalPtyProcess { #[cfg(unix)] fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + #[cfg(test)] + { + self.normal_cleanup_attempts.fetch_add(1, Ordering::AcqRel); + self.normal_cleanup_started.store(true, Ordering::Release); + let delay_ms = self.normal_cleanup_delay_ms.load(Ordering::Acquire); + if delay_ms > 0 { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + if self + .normal_cleanup_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return false; + } + } let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return false; }; @@ -3294,6 +3328,91 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn normal_child_exit_retries_a_failed_session_cleanup() { + let root = std::env::temp_dir() + .join(format!("cmux-local-reap-retry-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let release_path = root.join("release"); + let mux = Mux::new_for_test("local-reap-retry", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!("while [ ! -e {} ]; do sleep 0.01; done", release_path.display()), + ]), + ..SurfaceOptions::default() + }; + let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); + let process = + match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; + process.normal_cleanup_failures.store(1, Ordering::Release); + std::fs::write(&release_path, b"ready").unwrap(); + + let retry_deadline = Instant::now() + Duration::from_secs(1); + while (!process.child_reaped.load(Ordering::Acquire) + || process.normal_cleanup_attempts.load(Ordering::Acquire) < 2) + && Instant::now() < retry_deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let retried = process.normal_cleanup_attempts.load(Ordering::Acquire) >= 2; + let reaped = process.child_reaped.load(Ordering::Acquire); + let _ = process.terminate_and_wait(Instant::now() + Duration::from_secs(1)); + let _ = std::fs::remove_dir_all(root); + + assert!(retried, "natural PTY cleanup was not retried after a transient failure"); + assert!(reaped, "the PTY leader remained unreaped after cleanup could succeed"); + } + + #[cfg(unix)] + #[test] + fn explicit_shutdown_deadline_does_not_wait_for_natural_cleanup_sweep() { + let root = std::env::temp_dir() + .join(format!("cmux-local-reap-lock-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let release_path = root.join("release"); + let mux = Mux::new_for_test("local-reap-lock", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!("while [ ! -e {} ]; do sleep 0.01; done", release_path.display()), + ]), + ..SurfaceOptions::default() + }; + let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); + let process = + match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; + process.normal_cleanup_delay_ms.store(300, Ordering::Release); + std::fs::write(&release_path, b"ready").unwrap(); + let sweep_deadline = Instant::now() + Duration::from_secs(1); + while !process.normal_cleanup_started.load(Ordering::Acquire) + && Instant::now() < sweep_deadline + { + std::thread::sleep(Duration::from_millis(5)); + } + assert!(process.normal_cleanup_started.load(Ordering::Acquire)); + + let started = Instant::now(); + let _ = process.terminate_and_wait(started + Duration::from_millis(50)); + let elapsed = started.elapsed(); + let _ = process.terminate_and_wait(Instant::now() + Duration::from_secs(1)); + let _ = std::fs::remove_dir_all(root); + + assert!( + elapsed < Duration::from_millis(150), + "explicit shutdown waited behind the natural cleanup sweep: {elapsed:?}" + ); + } + #[test] fn local_shutdown_does_not_wait_for_the_pty_writer_lock() { let mux = Mux::new_for_test("shutdown-writer-lock", SurfaceOptions::default()); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 019c5fb3de33..02afc5aa716f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -302,6 +302,10 @@ mod unix { const HOST_LAUNCH_ROLLBACK_WAIT: Duration = Duration::from_secs(4); const HOST_LAUNCH_TIMEOUT: Duration = Duration::from_secs(10); const HOST_LAUNCH_CANCEL_POLL: Duration = Duration::from_millis(25); + #[cfg(test)] + static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); + #[cfg(test)] + static NEXT_HOST_NORMAL_CLEANUP_FAILURES: AtomicUsize = AtomicUsize::new(0); struct SpawnedHostProcess { child: Option, @@ -2099,6 +2103,10 @@ mod unix { child_signal_lock: Mutex<()>, child_reaped: AtomicBool, group_escalation_complete: AtomicBool, + #[cfg(test)] + normal_cleanup_failures: AtomicUsize, + #[cfg(test)] + normal_cleanup_attempts: AtomicUsize, } fn publish_host_frames( @@ -2410,6 +2418,19 @@ mod unix { } fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + #[cfg(test)] + { + self.normal_cleanup_attempts.fetch_add(1, Ordering::AcqRel); + if self + .normal_cleanup_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return false; + } + } let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return false; }; @@ -2913,6 +2934,12 @@ mod unix { child_signal_lock: Mutex::new(()), child_reaped: AtomicBool::new(false), group_escalation_complete: AtomicBool::new(false), + #[cfg(test)] + normal_cleanup_failures: AtomicUsize::new( + NEXT_HOST_NORMAL_CLEANUP_FAILURES.swap(0, Ordering::AcqRel), + ), + #[cfg(test)] + normal_cleanup_attempts: AtomicUsize::new(0), }); let reader_host = shared.clone(); @@ -3783,6 +3810,53 @@ mod unix { assert_eq!(*viewers.lock().unwrap(), HashMap::from([(1, (80, 24))])); } + #[test] + fn hosted_child_exit_retries_a_failed_session_cleanup() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + NEXT_HOST_NORMAL_CLEANUP_FAILURES.store(1, Ordering::Release); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: "/tmp/cmux-host-reap-retry.sock".into(), + record_path: "/tmp/cmux-host-reap-retry.json".into(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec!["/bin/sh".into(), "-c".into(), "exit 0".into()], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + let host = spawn_host_runtime(&launch, &bootstrapped).unwrap(); + + let retry_deadline = Instant::now() + Duration::from_secs(1); + while (!host.child_reaped.load(Ordering::Acquire) + || host.normal_cleanup_attempts.load(Ordering::Acquire) < 2) + && Instant::now() < retry_deadline + { + thread::sleep(Duration::from_millis(10)); + } + let retried = host.normal_cleanup_attempts.load(Ordering::Acquire) >= 2; + let reaped = host.child_reaped.load(Ordering::Acquire); + host.request_termination(); + let _ = host.wait_for_child_exit(Duration::from_secs(1)); + + assert!(retried, "hosted PTY cleanup was not retried after a transient failure"); + assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); + } + #[test] fn stalled_host_handshake_is_time_bounded() { let (record_path, record, lease) = record_fixture("handshake-timeout"); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index f10701a97b5c..40bff13d7291 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -977,11 +977,68 @@ fn shell_quote(path: &Path) -> String { #[cfg(test)] mod tests { + use std::io::{self, Read}; + use std::net::Shutdown; #[cfg(unix)] use std::process::{Command, Stdio}; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; + struct FloodStream { + remaining_chunks: usize, + reads: Arc, + } + + impl Read for FloodStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if self.remaining_chunks == 0 { + return Ok(0); + } + self.remaining_chunks -= 1; + self.reads.fetch_add(1, Ordering::AcqRel); + buffer.fill(b'x'); + Ok(buffer.len()) + } + } + + impl Write for FloodStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl transport::Stream for FloodStream { + fn try_clone_box(&self) -> io::Result> { + Err(io::Error::new(io::ErrorKind::Unsupported, "test stream is not cloneable")) + } + + fn read_timeout(&self) -> io::Result> { + Ok(None) + } + + fn set_read_timeout(&self, _timeout: Option) -> io::Result<()> { + Ok(()) + } + + fn write_timeout(&self) -> io::Result> { + Ok(None) + } + + fn set_write_timeout(&self, _timeout: Option) -> io::Result<()> { + Ok(()) + } + + fn shutdown(&self, _how: Shutdown) -> io::Result<()> { + Ok(()) + } + } + #[test] fn mismatch_message_contains_both_release_identities_and_stop_command() { let identity = ServerIdentity { @@ -1225,4 +1282,23 @@ mod tests { ); assert_eq!(error.to_string(), crate::localization::catalog().server.transport_failed); } + + #[test] + fn response_reader_rejects_an_unterminated_frame_at_its_size_limit() { + const CHUNK_BYTES: usize = 1024 * 1024; + let reads = Arc::new(AtomicUsize::new(0)); + let stream = FloodStream { remaining_chunks: 20, reads: reads.clone() }; + let mut reader = + BufReader::with_capacity(CHUNK_BYTES, Box::new(stream) as Box); + + let error = + read_matching_response_with_timeout(&mut reader, 7, false, Duration::from_secs(1)) + .unwrap_err(); + + assert_eq!(error.to_string(), crate::localization::catalog().server.response_invalid); + assert!( + reads.load(Ordering::Acquire) <= 17, + "response reader consumed an unbounded unterminated frame" + ); + } } From 982472b07933d1839f1a53489fdba43e6f04da8f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 08:41:32 -0700 Subject: [PATCH 079/521] fix(tui): make lifecycle ownership retryable --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 10 +- .../cmux-tui-core/src/process_session.rs | 86 +++++++++++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 75 +++++------- .../src/terminal_host_runtime.rs | 74 +++++------- .../crates/cmux-tui/src/server_lifecycle.rs | 112 ++++++++++++------ 5 files changed, 220 insertions(+), 137 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 3baed026e084..14f735ce3239 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -329,18 +329,16 @@ fn poll_reap_request(request: &mut ReapRequest) -> bool { REAPER_POLL_ATTEMPTS.fetch_add(1, Ordering::Relaxed); if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { let _ = request.child.kill(); - if let Some(profile_dir) = request.profile_dir.take() { - let _ = std::fs::remove_dir_all(profile_dir); - } - return true; + return false; } if FORCE_REAPER_PENDING.load(Ordering::Acquire) { return false; } } let _ = request.child.kill(); - if let Ok(None) = request.child.try_wait() { - return false; + match request.child.try_wait() { + Ok(Some(_)) => {} + Ok(None) | Err(_) => return false, } if let Some(profile_dir) = request.profile_dir.take() { let _ = std::fs::remove_dir_all(profile_dir); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2b8f56fb07ef..1746a756c2b1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -7,10 +7,92 @@ use std::collections::{HashMap, HashSet}; use std::io; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Condvar, Mutex}; use std::time::{Duration, Instant}; const SESSION_KILL_POLL: Duration = Duration::from_millis(5); +const NATURAL_REAP_RETRY_INITIAL: Duration = Duration::from_millis(25); +const NATURAL_REAP_RETRY_MAX: Duration = Duration::from_secs(1); + +/// Synchronization shared by a WNOWAIT child observer and explicit PTY +/// shutdown. The session leader remains reserved until this state machine +/// confirms that cleanup completed and performs the sole final reap. +pub(crate) struct ReservedChildReap<'a> { + pub(crate) changed: &'a (Mutex, Condvar), + pub(crate) signal_lock: &'a Mutex<()>, + pub(crate) pty_drained: &'a AtomicBool, + pub(crate) termination_started: &'a AtomicBool, + pub(crate) cleanup_complete: &'a AtomicBool, + pub(crate) child_reaped: &'a AtomicBool, +} + +/// Reap a waitable PTY session leader without holding its signal lock during +/// process-table scans. A transient natural-cleanup failure is retried even +/// when no other lifecycle event arrives. +pub(crate) fn reap_reserved_session_leader( + sync: ReservedChildReap<'_>, + cleanup_timeout: Duration, + mut cleanup: impl FnMut(Instant) -> bool, + reap: impl FnOnce(), +) { + let mut reap = Some(reap); + let mut retry_delay = NATURAL_REAP_RETRY_INITIAL; + loop { + let should_attempt_cleanup = { + let _signal = sync.signal_lock.lock().unwrap(); + if sync.cleanup_complete.load(Ordering::Acquire) { + reap.take().expect("reserved child is reaped once")(); + sync.child_reaped.store(true, Ordering::Release); + return; + } + !sync.termination_started.load(Ordering::Acquire) + && sync.pty_drained.load(Ordering::Acquire) + }; + + let cleanup_succeeded = should_attempt_cleanup && cleanup(Instant::now() + cleanup_timeout); + if should_attempt_cleanup { + let _signal = sync.signal_lock.lock().unwrap(); + let cleanup_claimed = sync.cleanup_complete.load(Ordering::Acquire); + let termination_started = sync.termination_started.load(Ordering::Acquire); + if cleanup_claimed || (cleanup_succeeded && !termination_started) { + sync.cleanup_complete.store(true, Ordering::Release); + reap.take().expect("reserved child is reaped once")(); + sync.child_reaped.store(true, Ordering::Release); + return; + } + } + + let state = sync.changed.0.lock().unwrap(); + let retry_natural_cleanup = should_attempt_cleanup + && !cleanup_succeeded + && !sync.termination_started.load(Ordering::Acquire) + && !sync.cleanup_complete.load(Ordering::Acquire); + if retry_natural_cleanup { + let (_state, _) = sync + .changed + .1 + .wait_timeout_while(state, retry_delay, |_| { + !sync.termination_started.load(Ordering::Acquire) + && !sync.cleanup_complete.load(Ordering::Acquire) + && sync.pty_drained.load(Ordering::Acquire) + }) + .unwrap(); + retry_delay = (retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + } else { + let _state = sync + .changed + .1 + .wait_while(state, |_| { + !sync.cleanup_complete.load(Ordering::Acquire) + && (sync.termination_started.load(Ordering::Acquire) + || !sync.pty_drained.load(Ordering::Acquire)) + }) + .unwrap(); + retry_delay = NATURAL_REAP_RETRY_INITIAL; + } + } +} #[cfg(test)] thread_local! { @@ -127,7 +209,7 @@ impl SessionProcessSnapshot { drop(state); #[cfg(test)] - self.scan_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + self.scan_count.fetch_add(1, Ordering::Relaxed); let scanned = scan_sessions(&self.sessions, Some(deadline)); let mut state = self.state.lock().unwrap(); state.refreshing = false; @@ -149,7 +231,7 @@ impl SessionProcessSnapshot { #[cfg(test)] fn scan_count(&self) -> usize { - self.scan_count.load(std::sync::atomic::Ordering::Relaxed) + self.scan_count.load(Ordering::Relaxed) } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index d0bc9c0dea83..a57bf6ad7c93 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -847,51 +847,21 @@ impl LocalPtyProcess { if observed_without_reaping { process.child_waitable.store(true, Ordering::Release); process.exited.1.notify_all(); - loop { - let signal = process.child_signal_lock.lock().unwrap(); - let escalation_complete = - process.group_escalation_complete.load(Ordering::Acquire); - let termination_started = - process.termination_started.load(Ordering::Acquire); - let pty_drained = process.pty_drained.load(Ordering::Acquire); - let normal_cleanup_complete = !escalation_complete - && !termination_started - && pty_drained - && process.kill_session_descendants_before_reap( - Instant::now() + NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, - ); - if escalation_complete || normal_cleanup_complete { - process.group_escalation_complete.store(true, Ordering::Release); + crate::process_session::reap_reserved_session_leader( + crate::process_session::ReservedChildReap { + changed: &process.exited, + signal_lock: &process.child_signal_lock, + pty_drained: &process.pty_drained, + termination_started: &process.termination_started, + cleanup_complete: &process.group_escalation_complete, + child_reaped: &process.child_reaped, + }, + NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, + |deadline| process.kill_session_descendants_before_reap(deadline), + || { let _ = child.wait(); - process.child_reaped.store(true, Ordering::Release); - drop(signal); - break; - } - drop(signal); - let state = process.exited.0.lock().unwrap(); - let _state = if !termination_started && pty_drained { - process - .exited - .1 - .wait_while(state, |_| { - !process.termination_started.load(Ordering::Acquire) - && !process - .group_escalation_complete - .load(Ordering::Acquire) - }) - .unwrap() - } else { - process - .exited - .1 - .wait_while(state, |_| { - !process.group_escalation_complete.load(Ordering::Acquire) - && (process.termination_started.load(Ordering::Acquire) - || !process.pty_drained.load(Ordering::Acquire)) - }) - .unwrap() - }; - } + }, + ); } else { let _ = child.wait(); process.child_reaped.store(true, Ordering::Release); @@ -942,6 +912,12 @@ impl LocalPtyProcess { *exited } + #[cfg(unix)] + fn abandon_termination(&self) { + self.termination_started.store(false, Ordering::Release); + self.exited.1.notify_all(); + } + #[cfg(unix)] fn wait_for_child_waitable_until(&self, deadline: Instant) { if self.child_waitable.load(Ordering::Acquire) { @@ -1072,14 +1048,20 @@ impl LocalPtyProcess { self.exited.1.notify_all(); if child_reaped { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + self.abandon_termination(); return false; }; - return matches!( + let empty = matches!( crate::process_session::session_is_empty_until(session, deadline), Ok(true) ); + if !empty { + self.abandon_termination(); + } + return empty; } if self.signal_terminal_process_session(libc::SIGHUP, deadline, snapshot).is_err() { + self.abandon_termination(); return false; } let _ = self.request_hangup(); @@ -1091,6 +1073,7 @@ impl LocalPtyProcess { // reserved with WNOWAIT, and do not acknowledge shutdown until no // background group remains. if !matches!(self.kill_terminal_process_session_until(deadline, snapshot), Ok(true)) { + self.abandon_termination(); return false; } self.group_escalation_complete.store(true, Ordering::Release); @@ -3353,7 +3336,7 @@ mod tests { process.normal_cleanup_failures.store(1, Ordering::Release); std::fs::write(&release_path, b"ready").unwrap(); - let retry_deadline = Instant::now() + Duration::from_secs(1); + let retry_deadline = Instant::now() + Duration::from_secs(3); while (!process.child_reaped.load(Ordering::Acquire) || process.normal_cleanup_attempts.load(Ordering::Acquire) < 2) && Instant::now() < retry_deadline diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 02afc5aa716f..3b2309f1eecc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2478,6 +2478,11 @@ mod unix { self.child_exit.1.notify_all(); } + fn abandon_group_escalation(&self) { + self.termination_started.store(false, Ordering::Release); + self.child_exit.1.notify_all(); + } + fn publish_exit_if_drained(&self) { if claim_host_exit_after_drain( &self.child_exit.0, @@ -2516,7 +2521,7 @@ mod unix { self.child_exit.1.notify_all(); self.publish_exit_if_drained(); } else { - self.termination_started.store(false, Ordering::Release); + self.abandon_group_escalation(); } return; } @@ -2530,7 +2535,7 @@ mod unix { ) .is_err() { - self.termination_started.store(false, Ordering::Release); + self.abandon_group_escalation(); return; } if !self.child_waitable.load(Ordering::Acquire) { @@ -2545,7 +2550,7 @@ mod unix { // gone. let kill_deadline = Instant::now() + HOST_KILL_WAIT; if !matches!(self.kill_terminal_process_session_until(kill_deadline), Ok(true)) { - self.termination_started.store(false, Ordering::Release); + self.abandon_group_escalation(); return; } self.finish_group_escalation(); @@ -3019,48 +3024,21 @@ mod unix { if observed_without_reaping { child_host.child_waitable.store(true, Ordering::Release); child_host.child_exit.1.notify_all(); - loop { - let signal = child_host.child_signal_lock.lock().unwrap(); - let escalation_complete = - child_host.group_escalation_complete.load(Ordering::Acquire); - let termination_started = - child_host.termination_started.load(Ordering::Acquire); - let pty_drained = child_host.pty_drained.load(Ordering::Acquire); - let normal_cleanup_complete = !escalation_complete - && !termination_started - && pty_drained - && child_host - .kill_session_descendants_before_reap(Instant::now() + HOST_KILL_WAIT); - if escalation_complete || normal_cleanup_complete { - child_host.group_escalation_complete.store(true, Ordering::Release); + crate::process_session::reap_reserved_session_leader( + crate::process_session::ReservedChildReap { + changed: &child_host.child_exit, + signal_lock: &child_host.child_signal_lock, + pty_drained: &child_host.pty_drained, + termination_started: &child_host.termination_started, + cleanup_complete: &child_host.group_escalation_complete, + child_reaped: &child_host.child_reaped, + }, + HOST_KILL_WAIT, + |deadline| child_host.kill_session_descendants_before_reap(deadline), + || { let _ = child.wait(); - child_host.child_reaped.store(true, Ordering::Release); - drop(signal); - break; - } - drop(signal); - let state = child_host.child_exit.0.lock().unwrap(); - let _state = if !termination_started && pty_drained { - child_host - .child_exit - .1 - .wait_while(state, |_| { - !child_host.termination_started.load(Ordering::Acquire) - && !child_host.group_escalation_complete.load(Ordering::Acquire) - }) - .unwrap() - } else { - child_host - .child_exit - .1 - .wait_while(state, |_| { - !child_host.group_escalation_complete.load(Ordering::Acquire) - && (child_host.termination_started.load(Ordering::Acquire) - || !child_host.pty_drained.load(Ordering::Acquire)) - }) - .unwrap() - }; - } + }, + ); let mut exited = child_host.child_exit.0.lock().unwrap(); *exited = true; drop(exited); @@ -3841,7 +3819,7 @@ mod unix { }; let host = spawn_host_runtime(&launch, &bootstrapped).unwrap(); - let retry_deadline = Instant::now() + Duration::from_secs(1); + let retry_deadline = Instant::now() + Duration::from_secs(3); while (!host.child_reaped.load(Ordering::Acquire) || host.normal_cleanup_attempts.load(Ordering::Acquire) < 2) && Instant::now() < retry_deadline @@ -3853,7 +3831,11 @@ mod unix { host.request_termination(); let _ = host.wait_for_child_exit(Duration::from_secs(1)); - assert!(retried, "hosted PTY cleanup was not retried after a transient failure"); + assert!( + retried, + "hosted PTY cleanup was not retried after a transient failure (attempts={})", + host.normal_cleanup_attempts.load(Ordering::Acquire) + ); assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 40bff13d7291..a82043e6698c 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -35,6 +35,7 @@ const LEGACY_STABLE_EMPTY_SCANS: usize = 2; #[cfg(unix)] const LEGACY_MAX_SCAN_ROUNDS: usize = 64; const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_LIFECYCLE_RESPONSE_BYTES: usize = 16 * 1024 * 1024; const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); const SHUTDOWN_RESPONSE_TIMEOUT: Duration = SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); @@ -825,25 +826,20 @@ fn read_matching_response_until( accept_unidentified_error: bool, deadline: Instant, ) -> anyhow::Result { - let mut line = String::new(); + let mut line = Vec::new(); loop { - set_transport_deadline(reader.get_mut().as_ref(), deadline)?; - match reader.read_line(&mut line) { - Ok(0) => anyhow::bail!(crate::localization::catalog().server.response_closed), - Ok(_) => {} - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) && Instant::now() < deadline => - { - continue; + match read_bounded_lifecycle_line_until(reader, &mut line, deadline) { + Ok(false) => anyhow::bail!(crate::localization::catalog().server.response_closed), + Ok(true) => {} + Err(LifecycleLineError::Invalid) => { + anyhow::bail!(crate::localization::catalog().server.response_invalid) + } + Err(LifecycleLineError::Deadline | LifecycleLineError::Transport) => { + anyhow::bail!(crate::localization::catalog().server.transport_failed) } - Err(_) => anyhow::bail!(crate::localization::catalog().server.transport_failed), } - let value: Value = serde_json::from_str(&line) + let value: Value = serde_json::from_slice(&line) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.response_invalid))?; - line.clear(); if value.get("event").is_some() { continue; } @@ -858,6 +854,62 @@ fn read_matching_response_until( } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LifecycleLineError { + Deadline, + Invalid, + Transport, +} + +fn read_bounded_lifecycle_line_until( + reader: &mut TransportReader, + line: &mut Vec, + deadline: Instant, +) -> Result { + line.clear(); + loop { + if Instant::now() >= deadline { + return Err(LifecycleLineError::Deadline); + } + set_transport_deadline(reader.get_mut().as_ref(), deadline).map_err(|_| { + if Instant::now() >= deadline { + LifecycleLineError::Deadline + } else { + LifecycleLineError::Transport + } + })?; + let available = match reader.fill_buf() { + Ok(available) => available, + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => + { + if Instant::now() >= deadline { + return Err(LifecycleLineError::Deadline); + } + continue; + } + Err(_) => return Err(LifecycleLineError::Transport), + }; + if available.is_empty() { + return Ok(!line.is_empty()); + } + let newline = available.iter().position(|byte| *byte == b'\n'); + let payload_bytes = newline.unwrap_or(available.len()); + if payload_bytes > MAX_LIFECYCLE_RESPONSE_BYTES.saturating_sub(line.len()) { + return Err(LifecycleLineError::Invalid); + } + line.extend_from_slice(&available[..payload_bytes]); + let consumed = payload_bytes + usize::from(newline.is_some()); + reader.consume(consumed); + if newline.is_some() { + return Ok(true); + } + } +} + #[cfg(unix)] fn response_data(response: &Value) -> anyhow::Result<&Value> { if response.get("ok").and_then(Value::as_bool) != Some(true) { @@ -921,34 +973,20 @@ fn wait_for_disconnect_until( path: &Path, deadline: Instant, ) -> anyhow::Result<()> { - let mut line = String::new(); + let mut line = Vec::new(); loop { - if set_transport_deadline(reader.get_mut().as_ref(), deadline).is_err() { - if connection_is_gone(path) { - return Ok(()); - } - anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out); - } - match reader.read_line(&mut line) { - Ok(0) => return Ok(()), - Ok(_) => line.clear(), - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) && Instant::now() < deadline => {} - Err(error) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => - { + match read_bounded_lifecycle_line_until(reader, &mut line, deadline) { + Ok(false) => return Ok(()), + Ok(true) => {} + Err(LifecycleLineError::Deadline) => { if connection_is_gone(path) { return Ok(()); } anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out); } - Err(_) => return shutdown_read_error(path), + Err(LifecycleLineError::Invalid | LifecycleLineError::Transport) => { + return shutdown_read_error(path); + } } } } From eb54fcb75fb49a0e58efc8f66ace18092e8881b7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 09:11:41 -0700 Subject: [PATCH 080/521] test(tui): expose reusable pid signaling --- .../cmux-tui-core/src/process_session.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 1746a756c2b1..ad3e447ad90c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -97,6 +97,7 @@ pub(crate) fn reap_reserved_session_leader( #[cfg(test)] thread_local! { static PROCESS_TABLE_SCAN_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } pub(crate) struct SessionProcessSnapshot { @@ -403,6 +404,8 @@ fn signal_if_still_member( } // SAFETY: membership was revalidated above and `signal` is a platform // signal constant supplied by this module's callers. + #[cfg(test)] + RAW_PID_SIGNAL_COUNT.set(RAW_PID_SIGNAL_COUNT.get() + 1); if unsafe { libc::kill(pid, signal) } == 0 { return Ok(()); } @@ -599,6 +602,24 @@ mod tests { child.wait().unwrap(); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn session_members_are_never_signaled_through_a_reusable_pid() { + let mut child = spawn_session_child(); + let session = libc::pid_t::try_from(child.id()).unwrap(); + RAW_PID_SIGNAL_COUNT.set(0); + + signal_until(session, libc::SIGCONT, Instant::now() + Duration::from_secs(1)).unwrap(); + let raw_signals = RAW_PID_SIGNAL_COUNT.get(); + + child.kill().unwrap(); + child.wait().unwrap(); + assert_eq!( + raw_signals, 0, + "session cleanup used a reusable PID instead of a stable process identity" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn shutdown_batch_reuses_one_process_table_snapshot() { From edcc294d6eb660fdeea90d454ec6144bef9d3a7c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 09:21:17 -0700 Subject: [PATCH 081/521] fix(tui): signal exact session processes --- .../cmux-tui-core/src/process_session.rs | 248 ++++++++++++++++-- 1 file changed, 233 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index ad3e447ad90c..a382fda7c896 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -7,6 +7,8 @@ use std::collections::{HashMap, HashSet}; use std::io; +#[cfg(target_os = "macos")] +use std::mem::size_of; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Condvar, Mutex}; use std::time::{Duration, Instant}; @@ -98,6 +100,7 @@ pub(crate) fn reap_reserved_session_leader( thread_local! { static PROCESS_TABLE_SCAN_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static STABLE_PROCESS_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } pub(crate) struct SessionProcessSnapshot { @@ -397,20 +400,10 @@ fn signal_if_still_member( session: libc::pid_t, signal: libc::c_int, ) -> io::Result<()> { - // Revalidate membership immediately before signaling. The reserved - // session leader prevents the session id itself from being recycled. - if !still_member(pid, session)? { - return Ok(()); - } - // SAFETY: membership was revalidated above and `signal` is a platform - // signal constant supplied by this module's callers. - #[cfg(test)] - RAW_PID_SIGNAL_COUNT.set(RAW_PID_SIGNAL_COUNT.get() + 1); - if unsafe { libc::kill(pid, signal) } == 0 { + let Some(process) = stable_process_in_session(pid, session)? else { return Ok(()); - } - let error = io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::ESRCH) { Ok(()) } else { Err(error) } + }; + process.signal(signal) } fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { @@ -423,7 +416,230 @@ fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) } } -#[cfg(test)] +fn stable_process_in_session( + pid: libc::pid_t, + session: libc::pid_t, +) -> io::Result> { + let Some(process) = StableProcess::capture(pid)? else { return Ok(None) }; + // Bracket the PID-based session query with a stable process identity. If + // the PID is recycled during getsid(2), the final identity check fails + // closed and no signal is sent. + let current_session = unsafe { libc::getsid(pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; + } + if current_session != session || !process.matches_current()? { + return Ok(None); + } + Ok(Some(process)) +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct StableProcess { + pid: libc::pid_t, + audit_token: MacAuditToken, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct MacAuditToken { + values: [u32; 8], +} + +#[cfg(target_os = "macos")] +impl StableProcess { + fn capture(pid: libc::pid_t) -> io::Result> { + const TASK_AUDIT_TOKEN: libc::task_flavor_t = 15; + let mut task = 0; + // SAFETY: mach_task_self returns the calling process's send right. + #[allow(deprecated)] + let current_task = unsafe { libc::mach_task_self() }; + // SAFETY: `task` points to writable storage for one task-name right. + let name_result = unsafe { task_name_for_pid(current_task, pid, &mut task) }; + if name_result != libc::KERN_SUCCESS { + return if process_is_gone(pid)? { + Ok(None) + } else { + Err(io::Error::other(format!( + "cannot acquire stable process identity for PID {pid}: Mach error {name_result}" + ))) + }; + } + + let mut audit_token = MacAuditToken { values: [0; 8] }; + let mut count = u32::try_from(size_of::() / size_of::()) + .expect("audit token count fits mach_msg_type_number_t"); + // SAFETY: the token buffer contains `count` natural_t values and the + // returned task-name right remains owned until the call completes. + let info_result = unsafe { + libc::task_info( + task, + TASK_AUDIT_TOKEN, + audit_token.values.as_mut_ptr().cast::(), + &mut count, + ) + }; + // SAFETY: `task` is the send right returned by task_name_for_pid. + let _ = unsafe { mach_port_deallocate(current_task, task) }; + if info_result != libc::KERN_SUCCESS { + return if process_is_gone(pid)? { + Ok(None) + } else { + Err(io::Error::other(format!( + "cannot read stable process identity for PID {pid}: Mach error {info_result}" + ))) + }; + } + let expected_pid = u32::try_from(pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; + if count != 8 || audit_token.values[5] != expected_pid { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "stable process identity returned a mismatched PID", + )); + } + Ok(Some(Self { pid, audit_token })) + } + + fn matches_current(&self) -> io::Result { + Ok(Self::capture(self.pid)?.is_some_and(|current| current == *self)) + } + + fn signal(&self, signal: libc::c_int) -> io::Result<()> { + let mut token = self.audit_token; + loop { + // SAFETY: the audit token came from TASK_AUDIT_TOKEN for this + // process instance; libproc validates its PID generation. + let result = unsafe { proc_signal_with_audittoken(&mut token, signal) }; + if result == 0 || result == libc::ESRCH { + #[cfg(test)] + STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); + return Ok(()); + } + if result != libc::EINTR { + return Err(io::Error::from_raw_os_error(result)); + } + } + } +} + +#[cfg(target_os = "macos")] +fn process_is_gone(pid: libc::pid_t) -> io::Result { + // SAFETY: signal zero performs a non-mutating liveness/permission probe. + if unsafe { libc::kill(pid, 0) } == 0 { + return Ok(false); + } + let error = io::Error::last_os_error(); + match error.raw_os_error() { + Some(libc::ESRCH) => Ok(true), + Some(libc::EPERM) => Ok(false), + _ => Err(error), + } +} + +#[cfg(target_os = "macos")] +unsafe extern "C" { + fn task_name_for_pid( + target_task: libc::mach_port_t, + pid: libc::pid_t, + task: *mut libc::mach_port_t, + ) -> libc::kern_return_t; + fn mach_port_deallocate( + task: libc::mach_port_t, + name: libc::mach_port_t, + ) -> libc::kern_return_t; +} + +#[cfg(target_os = "macos")] +#[link(name = "proc")] +unsafe extern "C" { + fn proc_signal_with_audittoken(token: *mut MacAuditToken, signal: libc::c_int) -> libc::c_int; +} + +#[cfg(target_os = "linux")] +#[derive(Debug)] +struct StableProcess { + pidfd: std::os::fd::OwnedFd, +} + +#[cfg(target_os = "linux")] +impl StableProcess { + fn capture(pid: libc::pid_t) -> io::Result> { + use std::os::fd::FromRawFd; + + // SAFETY: pidfd_open receives a validated integer PID and zero flags. + let descriptor = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + if descriptor < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; + } + let descriptor = i32::try_from(descriptor) + .map_err(|_| io::Error::other("pidfd descriptor is out of range"))?; + // SAFETY: a successful pidfd_open returns one newly owned descriptor. + Ok(Some(Self { pidfd: unsafe { std::os::fd::OwnedFd::from_raw_fd(descriptor) } })) + } + + fn matches_current(&self) -> io::Result { + self.signal_result(0) + } + + fn signal(&self, signal: libc::c_int) -> io::Result<()> { + let _ = self.signal_result(signal)?; + #[cfg(test)] + STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); + Ok(()) + } + + fn signal_result(&self, signal: libc::c_int) -> io::Result { + use std::os::fd::AsRawFd; + + // SAFETY: the descriptor is a live pidfd, the siginfo pointer is + // intentionally null, and flags must be zero. + let result = unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + self.pidfd.as_raw_fd(), + signal, + std::ptr::null::(), + 0, + ) + }; + if result == 0 { + return Ok(true); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) } + } +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +struct StableProcess; + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +impl StableProcess { + fn capture(_pid: libc::pid_t) -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "stable process signaling is unavailable on this platform", + )) + } + + fn matches_current(&self) -> io::Result { + Ok(false) + } + + fn signal(&self, _signal: libc::c_int) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "stable process signaling is unavailable on this platform", + )) + } +} + +#[cfg(all(target_os = "macos", test))] fn members(session: libc::pid_t) -> io::Result> { let sessions = HashSet::from([session]); Ok(scan_sessions(&sessions, None)?.remove(&session).unwrap_or_default()) @@ -498,7 +714,6 @@ fn macos_session_process_ids(session: libc::pid_t) -> io::Result) -> io::Result> { use std::ffi::c_void; - use std::mem::size_of; // Callers retain this snapshot and poll its members directly. A new full // process-table scan happens only after that generation has drained. @@ -608,9 +823,11 @@ mod tests { let mut child = spawn_session_child(); let session = libc::pid_t::try_from(child.id()).unwrap(); RAW_PID_SIGNAL_COUNT.set(0); + STABLE_PROCESS_SIGNAL_COUNT.set(0); signal_until(session, libc::SIGCONT, Instant::now() + Duration::from_secs(1)).unwrap(); let raw_signals = RAW_PID_SIGNAL_COUNT.get(); + let stable_signals = STABLE_PROCESS_SIGNAL_COUNT.get(); child.kill().unwrap(); child.wait().unwrap(); @@ -618,6 +835,7 @@ mod tests { raw_signals, 0, "session cleanup used a reusable PID instead of a stable process identity" ); + assert_eq!(stable_signals, 1, "session cleanup did not use the stable process handle"); } #[cfg(any(target_os = "macos", target_os = "linux"))] From dbb7d6b13a4aea001d6b2b77cf2e7889648cd038 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 09:39:22 -0700 Subject: [PATCH 082/521] test(tui): expose final lifecycle identity gaps --- .../crates/cmux-tui/src/server_lifecycle.rs | 9 +++++- .../src/server_lifecycle/legacy_process.rs | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a82043e6698c..a2f519bb1ed3 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1104,7 +1104,14 @@ mod tests { message.contains(&format!("{launcher} server stop --socket '/tmp/test socket'")), "{message}" ); - assert!(message.contains(&format!("then run `{launcher}` again")), "{message}"); + let replay_arguments = std::env::args_os().skip(1).collect::>(); + assert!(!replay_arguments.is_empty(), "test runner must provide an argument"); + let mut replay = launcher; + for argument in replay_arguments { + replay.push(' '); + replay.push_str(&shell_quote(Path::new(&argument))); + } + assert!(message.contains(&format!("then run `{replay}` again")), "{message}"); assert!(message.contains("Stopping exits pane processes.")); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 1144c41f2ffe..05e2f0ce753e 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -7,6 +7,11 @@ const PROCESS_TREE_RETRY_INTERVAL: Duration = Duration::from_millis(10); #[cfg(test)] const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); +#[cfg(test)] +thread_local! { + static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + fn ensure_helper_active() -> io::Result<()> { if super::legacy_helper_cancelled() { return Err(io::Error::new( @@ -49,6 +54,8 @@ impl ProcessIdentity { } // SAFETY: the PID was range-checked and its birth identity was // revalidated immediately before this signal. + #[cfg(test)] + RAW_PID_SIGNAL_COUNT.set(RAW_PID_SIGNAL_COUNT.get() + 1); if unsafe { libc::kill(self.pid, signal) } == 0 { return Ok(ExactSignalResult::Signaled); } @@ -786,6 +793,29 @@ mod tests { child.wait().unwrap(); } + #[test] + fn process_identity_never_signals_through_a_reusable_pid() { + let mut child = Command::new("sleep") + .arg("60") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let process = + ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); + RAW_PID_SIGNAL_COUNT.set(0); + + assert_eq!(process.signal(libc::SIGCONT).unwrap(), ExactSignalResult::Signaled); + let raw_signals = RAW_PID_SIGNAL_COUNT.get(); + + child.kill().unwrap(); + child.wait().unwrap(); + assert_eq!( + raw_signals, 0, + "legacy cleanup used a reusable PID instead of a stable process identity" + ); + } + #[test] fn termination_accepts_a_verified_process_that_already_exited() { let mut child = Command::new("sleep") From b374c5195debfb50af926175c8a23ea404f8bf5c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 09:44:05 -0700 Subject: [PATCH 083/521] fix(tui): preserve exact legacy lifecycle state --- .../cmux-tui-core/src/process_session.rs | 62 ++++++++++++------- .../crates/cmux-tui/src/server_lifecycle.rs | 35 +++++++++-- .../src/server_lifecycle/legacy_process.rs | 53 +++++++++------- 3 files changed, 102 insertions(+), 48 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index a382fda7c896..9c31496307c6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -403,7 +403,8 @@ fn signal_if_still_member( let Some(process) = stable_process_in_session(pid, session)? else { return Ok(()); }; - process.signal(signal) + let _ = process.signal(signal)?; + Ok(()) } fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { @@ -419,8 +420,8 @@ fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { fn stable_process_in_session( pid: libc::pid_t, session: libc::pid_t, -) -> io::Result> { - let Some(process) = StableProcess::capture(pid)? else { return Ok(None) }; +) -> io::Result> { + let Some(process) = StableProcessHandle::capture(pid)? else { return Ok(None) }; // Bracket the PID-based session query with a stable process identity. If // the PID is recycled during getsid(2), the final identity check fails // closed and no signal is sent. @@ -437,7 +438,8 @@ fn stable_process_in_session( #[cfg(target_os = "macos")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct StableProcess { +/// An OS-backed reference to one process instance that cannot retarget after PID reuse. +pub struct StableProcessHandle { pid: libc::pid_t, audit_token: MacAuditToken, } @@ -450,8 +452,9 @@ struct MacAuditToken { } #[cfg(target_os = "macos")] -impl StableProcess { - fn capture(pid: libc::pid_t) -> io::Result> { +impl StableProcessHandle { + /// Capture the process currently using `pid`, or return `None` if it is already gone. + pub fn capture(pid: libc::pid_t) -> io::Result> { const TASK_AUDIT_TOKEN: libc::task_flavor_t = 15; let mut task = 0; // SAFETY: mach_task_self returns the calling process's send right. @@ -504,20 +507,27 @@ impl StableProcess { Ok(Some(Self { pid, audit_token })) } - fn matches_current(&self) -> io::Result { + /// Return whether this exact process instance is still alive. + pub fn matches_current(&self) -> io::Result { Ok(Self::capture(self.pid)?.is_some_and(|current| current == *self)) } - fn signal(&self, signal: libc::c_int) -> io::Result<()> { + /// Signal this exact process instance, returning false if it is already gone. + pub fn signal(&self, signal: libc::c_int) -> io::Result { let mut token = self.audit_token; loop { // SAFETY: the audit token came from TASK_AUDIT_TOKEN for this // process instance; libproc validates its PID generation. let result = unsafe { proc_signal_with_audittoken(&mut token, signal) }; - if result == 0 || result == libc::ESRCH { + if result == 0 { + #[cfg(test)] + STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); + return Ok(true); + } + if result == libc::ESRCH { #[cfg(test)] STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); - return Ok(()); + return Ok(false); } if result != libc::EINTR { return Err(io::Error::from_raw_os_error(result)); @@ -561,13 +571,15 @@ unsafe extern "C" { #[cfg(target_os = "linux")] #[derive(Debug)] -struct StableProcess { +/// An OS-backed reference to one process instance that cannot retarget after PID reuse. +pub struct StableProcessHandle { pidfd: std::os::fd::OwnedFd, } #[cfg(target_os = "linux")] -impl StableProcess { - fn capture(pid: libc::pid_t) -> io::Result> { +impl StableProcessHandle { + /// Capture the process currently using `pid`, or return `None` if it is already gone. + pub fn capture(pid: libc::pid_t) -> io::Result> { use std::os::fd::FromRawFd; // SAFETY: pidfd_open receives a validated integer PID and zero flags. @@ -582,15 +594,17 @@ impl StableProcess { Ok(Some(Self { pidfd: unsafe { std::os::fd::OwnedFd::from_raw_fd(descriptor) } })) } - fn matches_current(&self) -> io::Result { + /// Return whether this exact process instance is still alive. + pub fn matches_current(&self) -> io::Result { self.signal_result(0) } - fn signal(&self, signal: libc::c_int) -> io::Result<()> { - let _ = self.signal_result(signal)?; + /// Signal this exact process instance, returning false if it is already gone. + pub fn signal(&self, signal: libc::c_int) -> io::Result { + let signaled = self.signal_result(signal)?; #[cfg(test)] STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); - Ok(()) + Ok(signaled) } fn signal_result(&self, signal: libc::c_int) -> io::Result { @@ -616,22 +630,26 @@ impl StableProcess { } #[cfg(not(any(target_os = "macos", target_os = "linux")))] -struct StableProcess; +/// Placeholder on platforms without stable process signaling. +pub struct StableProcessHandle; #[cfg(not(any(target_os = "macos", target_os = "linux")))] -impl StableProcess { - fn capture(_pid: libc::pid_t) -> io::Result> { +impl StableProcessHandle { + /// Report that stable process handles are unsupported on this platform. + pub fn capture(_pid: libc::pid_t) -> io::Result> { Err(io::Error::new( io::ErrorKind::Unsupported, "stable process signaling is unavailable on this platform", )) } - fn matches_current(&self) -> io::Result { + /// Return false because no stable process handle is available. + pub fn matches_current(&self) -> io::Result { Ok(false) } - fn signal(&self, _signal: libc::c_int) -> io::Result<()> { + /// Report that stable process signaling is unsupported on this platform. + pub fn signal(&self, _signal: libc::c_int) -> io::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "stable process signaling is unavailable on this platform", diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a2f519bb1ed3..be176b9f0a50 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -727,12 +727,13 @@ pub(crate) fn incompatible_server_message(identity: &ServerIdentity, path: &Path .join(messages.reason_separator); let launcher = current_launcher_command(); let command = format!("{launcher} server stop --socket {}", shell_quote(path)); + let replay = current_replay_command(); let restart = format!( "{}{}{}{}{}", messages.restart_before_command, command, messages.restart_between_commands, - launcher, + replay, messages.restart_after_command ); format!( @@ -760,6 +761,22 @@ fn current_launcher_command() -> String { launcher_command_from(override_command.as_deref(), argv0.as_deref()) } +fn current_replay_command() -> String { + replay_command_from(current_launcher_command(), std::env::args_os().skip(1)) +} + +fn replay_command_from(mut command: String, arguments: I) -> String +where + I: IntoIterator, + S: AsRef, +{ + for argument in arguments { + command.push(' '); + command.push_str(&shell_quote(Path::new(argument.as_ref()))); + } + command +} + fn launcher_command_from(override_command: Option<&OsStr>, argv0: Option<&OsStr>) -> String { if let Some(command) = override_command.and_then(OsStr::to_str).map(str::trim).filter(|command| { @@ -1004,9 +1021,11 @@ fn connection_is_gone(path: &Path) -> bool { fn shell_quote(path: &Path) -> String { let value = path.display().to_string(); - if value.chars().all(|character| { - character.is_ascii_alphanumeric() || matches!(character, '/' | '.' | '_' | '-') - }) { + if !value.is_empty() + && value.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '/' | '.' | '_' | '-') + }) + { value } else { format!("'{}'", value.replace('\'', "'\"'\"'")) @@ -1127,6 +1146,14 @@ mod tests { ); } + #[test] + fn replay_command_preserves_quoted_and_empty_arguments() { + assert_eq!( + replay_command_from("cmux-tui".to_string(), ["--session", "two words", "", "it's"]), + "cmux-tui --session 'two words' '' 'it'\"'\"'s'" + ); + } + #[cfg(unix)] #[test] fn modern_shutdown_requires_a_success_response_before_disconnect() { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 05e2f0ce753e..aa2b69883030 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -2,6 +2,8 @@ use std::collections::HashSet; use std::io; use std::time::{Duration, Instant}; +use cmux_tui_core::process_session::StableProcessHandle; + const PROCESS_TREE_MAX_ROUNDS: usize = 64; const PROCESS_TREE_RETRY_INTERVAL: Duration = Duration::from_millis(10); #[cfg(test)] @@ -46,24 +48,13 @@ impl ProcessIdentity { } fn signal(self, signal: libc::c_int) -> io::Result { - let Some(current) = Self::capture(self.pid)? else { + let Some(process) = self.stable_handle()? else { return Ok(ExactSignalResult::Gone); }; - if current != self { - return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); - } - // SAFETY: the PID was range-checked and its birth identity was - // revalidated immediately before this signal. - #[cfg(test)] - RAW_PID_SIGNAL_COUNT.set(RAW_PID_SIGNAL_COUNT.get() + 1); - if unsafe { libc::kill(self.pid, signal) } == 0 { - return Ok(ExactSignalResult::Signaled); - } - let error = io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::ESRCH) { - Ok(ExactSignalResult::Gone) + if process.signal(signal)? { + Ok(ExactSignalResult::Signaled) } else { - Err(error) + Ok(ExactSignalResult::Gone) } } @@ -72,13 +63,11 @@ impl ProcessIdentity { session: libc::pid_t, signal: libc::c_int, ) -> io::Result { - let Some(current) = Self::capture(self.pid)? else { + let Some(process) = self.stable_handle()? else { return Ok(ExactSignalResult::Gone); }; - if current != self { - return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); - } - // SAFETY: getsid only queries the exact process revalidated above. + // Bracket the PID-based session query with the stable handle. If the + // PID is recycled during getsid(2), no signal reaches its new owner. let current_session = unsafe { libc::getsid(self.pid) }; if current_session < 0 { let error = io::Error::last_os_error(); @@ -88,10 +77,30 @@ impl ProcessIdentity { Err(error) }; } - if current_session != session { + if current_session != session || !process.matches_current()? { return Ok(ExactSignalResult::Gone); } - self.signal(signal) + if process.signal(signal)? { + Ok(ExactSignalResult::Signaled) + } else { + Ok(ExactSignalResult::Gone) + } + } + + fn stable_handle(self) -> io::Result> { + let Some(process) = StableProcessHandle::capture(self.pid)? else { + return Ok(None); + }; + let Some(current) = Self::capture(self.pid)? else { + return Ok(None); + }; + if current != self { + return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); + } + if !process.matches_current()? { + return Ok(None); + } + Ok(Some(process)) } } From 993241e31ba2e94813b00d6888577cf68ba93355 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:23:33 -0700 Subject: [PATCH 084/521] test(tui): expose pidfd compatibility gap --- .../cmux-tui-core/src/process_session.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 9c31496307c6..2e3cd9f635e8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -856,6 +856,76 @@ mod tests { assert_eq!(stable_signals, 1, "session cleanup did not use the stable process handle"); } + #[cfg(target_os = "linux")] + #[test] + fn pidfd_open_denial_uses_an_exact_proc_handle() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_BLOCK_PIDFD_OPEN"; + if std::env::var_os(CHILD_ENV).is_some() { + let mut filter = [ + libc::sock_filter { + code: u16::try_from(libc::BPF_LD | libc::BPF_W | libc::BPF_ABS).unwrap(), + jt: 0, + jf: 0, + k: 0, + }, + libc::sock_filter { + code: u16::try_from(libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K).unwrap(), + jt: 0, + jf: 1, + k: u32::try_from(libc::SYS_pidfd_open).unwrap(), + }, + libc::sock_filter { + code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), + jt: 0, + jf: 0, + k: libc::SECCOMP_RET_ERRNO | u32::try_from(libc::ENOSYS).unwrap(), + }, + libc::sock_filter { + code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), + jt: 0, + jf: 0, + k: libc::SECCOMP_RET_ALLOW, + }, + ]; + let program = libc::sock_fprog { + len: u16::try_from(filter.len()).unwrap(), + filter: filter.as_mut_ptr(), + }; + // SAFETY: this one-test subprocess opts into a filter whose + // backing instructions remain live for the prctl call. + assert_eq!(unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }, 0); + // SAFETY: `program` describes the initialized filter above. + let installed = unsafe { + libc::prctl( + libc::PR_SET_SECCOMP, + libc::SECCOMP_MODE_FILTER, + std::ptr::from_ref(&program), + ) + }; + assert_eq!( + installed, + 0, + "could not install pidfd test filter: {}", + io::Error::last_os_error() + ); + + let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + let process = StableProcessHandle::capture(pid).unwrap().unwrap(); + assert!(process.matches_current().unwrap()); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "process_session::tests::pidfd_open_denial_uses_an_exact_proc_handle", + ]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "pidfd compatibility subprocess failed: {status}"); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn shutdown_batch_reuses_one_process_table_snapshot() { From 0baf46d4a03d058ba6e3968f791dd230b12919ac Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:29:38 -0700 Subject: [PATCH 085/521] fix(tui): fall back to exact proc handles --- cmux-tui/README.md | 4 + .../cmux-tui-core/src/process_session.rs | 207 ++++++++++++------ cmux-tui/crates/cmux-tui-core/src/server.rs | 4 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 + cmux-tui/crates/cmux-tui/src/localization.rs | 3 + cmux-tui/crates/cmux-tui/src/main.rs | 4 + cmux-tui/dist/RELEASING-TUI.md | 2 + 7 files changed, 155 insertions(+), 71 deletions(-) diff --git a/cmux-tui/README.md b/cmux-tui/README.md index 280a22a5bdca..8904fb07c016 100644 --- a/cmux-tui/README.md +++ b/cmux-tui/README.md @@ -39,6 +39,10 @@ cargo run -p cmux-tui -- attach --session agents --surface The default session is `main`. Default sockets live at `$TMPDIR/cmux-tui-/.sock`; use `--socket ` for an explicit path. Detach from an attached TUI with prefix `d`, which is `Ctrl-b d` by default. +Linux requires kernel 5.1 or newer, a mounted `/proc`, and permission to call +`pidfd_send_signal`. Linux 5.3 and newer use `pidfd_open`; Linux 5.1 and 5.2 +use an exact `/proc/` process handle. + `attach --surface ` attaches one PTY terminal by numeric or short surface id. It uses the full host terminal without the sidebar, status bar, pane border, or other tabs. Packaged builds can run as `npx cmux`. The optional machine rail lets that local client switch among the current session, other Unix sockets, and sessions reached through SSH. It is disabled by default and activates when `machine_sidebar.enabled` is true or `machines` contains a valid entry in `cmux-tui.json`. `npx cmux --cloud` composes those local targets with the Cloud catalog and enables temporary machine connections without sending local SSH details to Cloud. The client uses noninteractive SSH with strict host-key checking and the remote `cmux-tui relay --session ` transport primitive, so the remote headless session, trusted host key, authentication key, and binary must already exist. See [Machines and remote sessions](docs/machines.md). diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2e3cd9f635e8..099958680b44 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -573,7 +573,7 @@ unsafe extern "C" { #[derive(Debug)] /// An OS-backed reference to one process instance that cannot retarget after PID reuse. pub struct StableProcessHandle { - pidfd: std::os::fd::OwnedFd, + process_fd: std::os::fd::OwnedFd, } #[cfg(target_os = "linux")] @@ -584,14 +584,32 @@ impl StableProcessHandle { // SAFETY: pidfd_open receives a validated integer PID and zero flags. let descriptor = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; - if descriptor < 0 { - let error = io::Error::last_os_error(); - return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; + if descriptor >= 0 { + let descriptor = i32::try_from(descriptor) + .map_err(|_| io::Error::other("pidfd descriptor is out of range"))?; + // SAFETY: a successful pidfd_open returns one newly owned descriptor. + return Ok(Some(Self { + process_fd: unsafe { std::os::fd::OwnedFd::from_raw_fd(descriptor) }, + })); + } + let pidfd_error = io::Error::last_os_error(); + if pidfd_error.raw_os_error() == Some(libc::ESRCH) { + return Ok(None); + } + + // Linux 5.1 and 5.2 predate pidfd_open but pidfd_send_signal accepts + // an open /proc/PID directory as the same exact process reference. + match std::fs::File::open(format!("/proc/{pid}")) { + Ok(process_directory) => Ok(Some(Self { process_fd: process_directory.into() })), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(io::Error::new( + error.kind(), + format!( + "cannot acquire stable process identity: pidfd_open failed ({pidfd_error}); \ + /proc fallback failed ({error})" + ), + )), } - let descriptor = i32::try_from(descriptor) - .map_err(|_| io::Error::other("pidfd descriptor is out of range"))?; - // SAFETY: a successful pidfd_open returns one newly owned descriptor. - Ok(Some(Self { pidfd: unsafe { std::os::fd::OwnedFd::from_raw_fd(descriptor) } })) } /// Return whether this exact process instance is still alive. @@ -615,7 +633,7 @@ impl StableProcessHandle { let result = unsafe { libc::syscall( libc::SYS_pidfd_send_signal, - self.pidfd.as_raw_fd(), + self.process_fd.as_raw_fd(), signal, std::ptr::null::(), 0, @@ -629,6 +647,26 @@ impl StableProcessHandle { } } +/// Verify that this runtime can signal exact process instances before any PTY is spawned. +pub fn require_stable_process_signaling() -> io::Result<()> { + let pid = libc::pid_t::try_from(std::process::id()) + .map_err(|_| io::Error::new(io::ErrorKind::Unsupported, "invalid process id"))?; + let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::Unsupported, "current process identity is unavailable") + })?; + match process.matches_current() { + Ok(true) => Ok(()), + Ok(false) => Err(io::Error::new( + io::ErrorKind::Unsupported, + "stable process signaling did not retain the current process", + )), + Err(error) => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("stable process signaling is unavailable: {error}"), + )), + } +} + #[cfg(not(any(target_os = "macos", target_os = "linux")))] /// Placeholder on platforms without stable process signaling. pub struct StableProcessHandle; @@ -820,6 +858,74 @@ mod tests { command.spawn().unwrap() } + #[cfg(target_os = "linux")] + fn enter_syscall_blocked_subprocess( + child_env: &str, + test_name: &str, + syscalls: &[libc::c_long], + ) -> bool { + if std::env::var_os(child_env).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name]) + .env(child_env, "1") + .status() + .unwrap(); + assert!(status.success(), "pidfd compatibility subprocess failed: {status}"); + return false; + } + + let mut filter = vec![libc::sock_filter { + code: u16::try_from(libc::BPF_LD | libc::BPF_W | libc::BPF_ABS).unwrap(), + jt: 0, + jf: 0, + k: 0, + }]; + for syscall in syscalls { + filter.extend([ + libc::sock_filter { + code: u16::try_from(libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K).unwrap(), + jt: 0, + jf: 1, + k: u32::try_from(*syscall).unwrap(), + }, + libc::sock_filter { + code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), + jt: 0, + jf: 0, + k: libc::SECCOMP_RET_ERRNO | u32::try_from(libc::ENOSYS).unwrap(), + }, + ]); + } + filter.push(libc::sock_filter { + code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), + jt: 0, + jf: 0, + k: libc::SECCOMP_RET_ALLOW, + }); + let program = libc::sock_fprog { + len: u16::try_from(filter.len()).unwrap(), + filter: filter.as_mut_ptr(), + }; + // SAFETY: this one-test subprocess opts into a filter whose backing + // instructions remain live for the prctl call. + assert_eq!(unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }, 0); + // SAFETY: `program` describes the initialized filter above. + let installed = unsafe { + libc::prctl( + libc::PR_SET_SECCOMP, + libc::SECCOMP_MODE_FILTER, + std::ptr::from_ref(&program), + ) + }; + assert_eq!( + installed, + 0, + "could not install pidfd test filter: {}", + io::Error::last_os_error() + ); + true + } + #[cfg(target_os = "macos")] #[test] fn macos_session_query_is_scoped_to_the_owned_session() { @@ -860,70 +966,33 @@ mod tests { #[test] fn pidfd_open_denial_uses_an_exact_proc_handle() { const CHILD_ENV: &str = "CMUX_TUI_TEST_BLOCK_PIDFD_OPEN"; - if std::env::var_os(CHILD_ENV).is_some() { - let mut filter = [ - libc::sock_filter { - code: u16::try_from(libc::BPF_LD | libc::BPF_W | libc::BPF_ABS).unwrap(), - jt: 0, - jf: 0, - k: 0, - }, - libc::sock_filter { - code: u16::try_from(libc::BPF_JMP | libc::BPF_JEQ | libc::BPF_K).unwrap(), - jt: 0, - jf: 1, - k: u32::try_from(libc::SYS_pidfd_open).unwrap(), - }, - libc::sock_filter { - code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), - jt: 0, - jf: 0, - k: libc::SECCOMP_RET_ERRNO | u32::try_from(libc::ENOSYS).unwrap(), - }, - libc::sock_filter { - code: u16::try_from(libc::BPF_RET | libc::BPF_K).unwrap(), - jt: 0, - jf: 0, - k: libc::SECCOMP_RET_ALLOW, - }, - ]; - let program = libc::sock_fprog { - len: u16::try_from(filter.len()).unwrap(), - filter: filter.as_mut_ptr(), - }; - // SAFETY: this one-test subprocess opts into a filter whose - // backing instructions remain live for the prctl call. - assert_eq!(unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }, 0); - // SAFETY: `program` describes the initialized filter above. - let installed = unsafe { - libc::prctl( - libc::PR_SET_SECCOMP, - libc::SECCOMP_MODE_FILTER, - std::ptr::from_ref(&program), - ) - }; - assert_eq!( - installed, - 0, - "could not install pidfd test filter: {}", - io::Error::last_os_error() - ); + if !enter_syscall_blocked_subprocess( + CHILD_ENV, + "process_session::tests::pidfd_open_denial_uses_an_exact_proc_handle", + &[libc::SYS_pidfd_open], + ) { + return; + } - let pid = libc::pid_t::try_from(std::process::id()).unwrap(); - let process = StableProcessHandle::capture(pid).unwrap().unwrap(); - assert!(process.matches_current().unwrap()); + let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + let process = StableProcessHandle::capture(pid).unwrap().unwrap(); + assert!(process.matches_current().unwrap()); + } + + #[cfg(target_os = "linux")] + #[test] + fn missing_exact_signal_support_fails_the_runtime_preflight() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_BLOCK_PIDFD_SIGNALING"; + if !enter_syscall_blocked_subprocess( + CHILD_ENV, + "process_session::tests::missing_exact_signal_support_fails_the_runtime_preflight", + &[libc::SYS_pidfd_open, libc::SYS_pidfd_send_signal], + ) { return; } - let status = Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "process_session::tests::pidfd_open_denial_uses_an_exact_proc_handle", - ]) - .env(CHILD_ENV, "1") - .status() - .unwrap(); - assert!(status.success(), "pidfd compatibility subprocess failed: {status}"); + let error = require_stable_process_signaling().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::Unsupported); } #[cfg(any(target_os = "macos", target_os = "linux"))] diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 37fc64b1601a..236d9b4a36fa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -3092,9 +3092,9 @@ fn advertised_capabilities() -> Vec<&'static str> { PROVIDER_MANAGED_WORKSPACE_GUARD_CAPABILITY, ]; // Atomic shutdown is advertised only where PTY session descendants can - // be enumerated and confirmed dead. + // be enumerated and signaled through exact process handles. #[cfg(any(target_os = "macos", target_os = "linux"))] - { + if crate::process_session::require_stable_process_signaling().is_ok() { capabilities.push(SERVER_SHUTDOWN_CAPABILITY); } capabilities diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index a57bf6ad7c93..82fd72eb1fb2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1182,6 +1182,8 @@ impl Surface { mux: Weak, terminal_id: Option, ) -> anyhow::Result> { + #[cfg(unix)] + crate::process_session::require_stable_process_signaling()?; #[cfg(unix)] if let Some(root) = opts.terminal_host_root.clone() { let default_colors = mux.upgrade().map(|mux| mux.default_colors()).unwrap_or_default(); diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index bd9d11fc832c..ede08b9c666a 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -69,6 +69,7 @@ pub(crate) struct ServerMessages { pub shutdown_failed: &'static str, pub shutdown_cleanup_incomplete: &'static str, pub shutdown_unsupported: &'static str, + pub process_cleanup_unsupported: &'static str, pub shutdown_timed_out: &'static str, pub cleanup_label: &'static str, pub cleanup_pending_label: &'static str, @@ -386,6 +387,7 @@ static ENGLISH: Catalog = Catalog { shutdown_failed: "the server could not stop cleanly", shutdown_cleanup_incomplete: "the server could not finish pane cleanup before the shutdown deadline", shutdown_unsupported: "this server cannot be stopped by this client", + process_cleanup_unsupported: "this system does not support safe terminal process cleanup", shutdown_timed_out: "the server did not stop within 10 seconds", cleanup_label: "cleanup", cleanup_pending_label: "pending panes", @@ -587,6 +589,7 @@ static JAPANESE: Catalog = Catalog { shutdown_failed: "サーバーを正常に停止できませんでした", shutdown_cleanup_incomplete: "シャットダウン期限までにペインのクリーンアップを完了できませんでした", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", + process_cleanup_unsupported: "このシステムではターミナルプロセスを安全に終了できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", cleanup_label: "クリーンアップ", cleanup_pending_label: "保留中のペイン", diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index fe35798cae3a..74ea3a331710 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1053,6 +1053,10 @@ fn run_server( } } + #[cfg(unix)] + cmux_tui_core::process_session::require_stable_process_signaling() + .map_err(|_| anyhow::anyhow!(localization::catalog().server.process_cleanup_unsupported))?; + let mut surface_options = SurfaceOptions::default(); config::apply_browser_to_surface_options(&config, &mut surface_options); if let Some(term) = args.term { diff --git a/cmux-tui/dist/RELEASING-TUI.md b/cmux-tui/dist/RELEASING-TUI.md index f96c899c1197..61eea55221ba 100644 --- a/cmux-tui/dist/RELEASING-TUI.md +++ b/cmux-tui/dist/RELEASING-TUI.md @@ -31,6 +31,8 @@ canonical distribution identity. Linux packages contain static musl binaries that run on both glibc and musl distributions. PyPI publishes each Linux binary under matching manylinux and musllinux wheel tags so installers on both runtime families can resolve it. +The runtime requires Linux 5.1 or newer, a mounted `/proc`, and permission to +call `pidfd_send_signal`. ## One-time registry setup From a91654c877818eda8e5ac38d6e31a4962aaaba60 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:47:45 -0700 Subject: [PATCH 086/521] test(tui): expose bounded lifecycle ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 162 ++++++++++++++++++ .../cmux-tui-core/src/process_session.rs | 68 +++++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 61 +++++++ .../cmux-tui-core/src/workspace_registry.rs | 26 ++- 4 files changed, 314 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e68861a8789c..51dd0876738d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1393,6 +1393,8 @@ pub struct Mux { #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, #[cfg(test)] + browser_runtime_connect: Mutex>>, + #[cfg(test)] shutdown_owner_capacity: AtomicUsize, browser_runtime: Mutex>>, cell_pixels: Mutex<(u16, u16)>, @@ -1640,6 +1642,8 @@ impl Mux { #[cfg(test)] browser_bootstrap_before_runtime: Mutex::new(None), #[cfg(test)] + browser_runtime_connect: Mutex::new(None), + #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), browser_runtime: Mutex::new(None), cell_pixels: Mutex::new((8, 16)), @@ -4171,6 +4175,10 @@ impl Mux { fn browser_runtime(&self) -> anyhow::Result> { let mut runtime = self.browser_runtime.lock().unwrap(); + #[cfg(test)] + if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { + hook(); + } if self.is_shutting_down() { anyhow::bail!("server is shutting down"); } @@ -11621,6 +11629,102 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn terminal_adoption_overflow_refills_at_a_low_water_mark() { + let mux = Mux::new_for_test("adoption-low-water", SurfaceOptions::default()); + { + let mut tracked = mux.terminal_adoptions.lock().unwrap(); + tracked + .extend((0..TERMINAL_ADOPTION_QUEUE_CAPACITY).map(|index| format!("{index:032x}"))); + } + { + let mut state = mux.terminal_adoption_coordinator.state.lock().unwrap(); + state.rescan_required = true; + state.next_rescan = Some(Instant::now()); + } + mux.ensure_terminal_adoption_workers().unwrap(); + + mux.terminal_adoptions.lock().unwrap().remove(&format!("{:032x}", 0)); + mux.terminal_adoption_coordinator.wake.notify_all(); + std::thread::sleep(Duration::from_millis(200)); + let retained_after_one_slot = + mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required; + + { + let mut tracked = mux.terminal_adoptions.lock().unwrap(); + while tracked.len() > TERMINAL_ADOPTION_QUEUE_CAPACITY / 2 { + let terminal_id = tracked.iter().next().unwrap().clone(); + tracked.remove(&terminal_id); + } + } + mux.terminal_adoption_coordinator.wake.notify_all(); + let deadline = Instant::now() + Duration::from_secs(1); + loop { + if !mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required + || Instant::now() >= deadline + { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let refilled = !mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required; + mux.request_terminal_adoption_stop(); + assert!( + mux.wait_for_terminal_adoption_workers_until(Instant::now() + Duration::from_secs(2)) + ); + + assert!( + retained_after_one_slot, + "adoption overflow triggered a full rescan after one queue slot opened" + ); + assert!(refilled, "adoption overflow did not refill after reaching its low-water mark"); + } + + #[cfg(unix)] + #[test] + fn terminal_adoption_rescan_uses_one_registry_snapshot() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let root = std::env::temp_dir() + .join(format!("cmux-adoption-snapshot-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let uid = std::fs::metadata(&root).unwrap().uid(); + for _ in 0..3 { + let terminal_id = TerminalId::random().unwrap().to_hex(); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: terminal_id.clone(), + incarnation: crate::terminal_host::HostIncarnation::random().unwrap().to_hex(), + endpoint: format!("/tmp/cmux-th-{uid}/{terminal_id}.sock"), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: String::new(), + supports_set_defaults: false, + supports_terminate_only: false, + }; + let path = root.join(format!("{terminal_id}.json")); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + let mut permissions = std::fs::metadata(&path).unwrap().permissions(); + permissions.set_mode(0o600); + std::fs::set_permissions(path, permissions).unwrap(); + } + let mux = Mux::new_for_test("adoption-snapshot", SurfaceOptions::default()); + mux.update_surface_options(|options| options.terminal_host_root = Some(root.clone())); + mux.shutting_down.store(true, Ordering::Release); + mux.workspace_registry.lock().unwrap().reset_terminal_snapshot_count_for_test(); + + assert!(mux.rescan_terminal_adoptions()); + let snapshots = mux.workspace_registry.lock().unwrap().terminal_snapshot_count_for_test(); + let _ = std::fs::remove_dir_all(root); + + assert_eq!( + snapshots, 1, + "one adoption rescan queried the full terminal registry {snapshots} times" + ); + } + #[cfg(unix)] #[test] fn blocked_terminal_adoption_does_not_stall_an_unrelated_terminal() { @@ -12205,6 +12309,23 @@ mod tests { assert!(topology_retained); } + #[cfg(unix)] + #[test] + fn server_shutdown_preflights_process_control_before_removing_topology() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + crate::process_session::set_process_session_preflight_failure_for_test(true); + + let result = mux.close_all_surfaces_for_shutdown(); + + crate::process_session::set_process_session_preflight_failure_for_test(false); + assert!(result.is_err(), "server shutdown ignored an unavailable process-control runtime"); + assert!( + mux.surface(surface.id).is_some(), + "server shutdown removed topology before process-control preflight" + ); + } + #[test] fn server_shutdown_waits_for_async_browser_bootstrap() { let mux = test_mux(); @@ -12235,6 +12356,47 @@ mod tests { assert!(mux.browser_runtime.lock().unwrap().is_none()); } + #[test] + fn browser_runtime_connection_does_not_hold_the_shared_slot() { + let mux = Mux::new_for_test( + "browser-runtime-slot", + SurfaceOptions { + cdp_url: Some("ws://127.0.0.1:9/devtools/browser/unreachable".into()), + ..SurfaceOptions::default() + }, + ); + let (connect_started_tx, connect_started_rx) = std::sync::mpsc::sync_channel(1); + let (release_connect_tx, release_connect_rx) = std::sync::mpsc::sync_channel(1); + let release_connect_rx = Arc::new(Mutex::new(release_connect_rx)); + *mux.browser_runtime_connect.lock().unwrap() = Some(Arc::new({ + move || { + connect_started_tx.send(()).unwrap(); + release_connect_rx.lock().unwrap().recv().unwrap(); + } + })); + mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); + connect_started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + let slot_available = mux.browser_runtime.try_lock().is_ok(); + release_connect_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + loop { + if mux.async_surface_creations.inner.state.lock().unwrap().active == 0 + || Instant::now() >= deadline + { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + mux.request_daemon_shutdown(); + mux.shutdown(); + + assert!( + slot_available, + "browser connection setup held the shared runtime slot across blocking work" + ); + } + #[test] fn shutdown_fanout_runs_terminations_concurrently() { let gate = Arc::new((Mutex::new(false), std::sync::Condvar::new())); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 099958680b44..585be99354bc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -9,6 +9,8 @@ use std::collections::{HashMap, HashSet}; use std::io; #[cfg(target_os = "macos")] use std::mem::size_of; +#[cfg(test)] +use std::sync::atomic::AtomicUsize; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Condvar, Mutex}; use std::time::{Duration, Instant}; @@ -17,6 +19,43 @@ const SESSION_KILL_POLL: Duration = Duration::from_millis(5); const NATURAL_REAP_RETRY_INITIAL: Duration = Duration::from_millis(25); const NATURAL_REAP_RETRY_MAX: Duration = Duration::from_secs(1); +#[cfg(test)] +static DEDICATED_NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); + +#[cfg(test)] +thread_local! { + static FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +#[cfg(test)] +struct DedicatedNaturalReapWorker; + +#[cfg(test)] +impl DedicatedNaturalReapWorker { + fn enter() -> Self { + DEDICATED_NATURAL_REAP_WORKERS.fetch_add(1, Ordering::AcqRel); + Self + } +} + +#[cfg(test)] +impl Drop for DedicatedNaturalReapWorker { + fn drop(&mut self) { + DEDICATED_NATURAL_REAP_WORKERS.fetch_sub(1, Ordering::AcqRel); + } +} + +#[cfg(test)] +pub(crate) fn dedicated_natural_reap_workers_for_test() -> usize { + DEDICATED_NATURAL_REAP_WORKERS.load(Ordering::Acquire) +} + +#[cfg(test)] +pub(crate) fn set_process_session_preflight_failure_for_test(enabled: bool) { + FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.set(enabled); +} + /// Synchronization shared by a WNOWAIT child observer and explicit PTY /// shutdown. The session leader remains reserved until this state machine /// confirms that cleanup completed and performs the sole final reap. @@ -38,6 +77,8 @@ pub(crate) fn reap_reserved_session_leader( mut cleanup: impl FnMut(Instant) -> bool, reap: impl FnOnce(), ) { + #[cfg(test)] + let _dedicated_worker = DedicatedNaturalReapWorker::enter(); let mut reap = Some(reap); let mut retry_delay = NATURAL_REAP_RETRY_INITIAL; loop { @@ -108,7 +149,7 @@ pub(crate) struct SessionProcessSnapshot { state: Mutex, refreshed: Condvar, #[cfg(test)] - scan_count: std::sync::atomic::AtomicUsize, + scan_count: AtomicUsize, } struct SessionProcessSnapshotState { @@ -156,7 +197,7 @@ impl SessionProcessSnapshot { }), refreshed: Condvar::new(), #[cfg(test)] - scan_count: std::sync::atomic::AtomicUsize::new(usize::from(has_sessions)), + scan_count: AtomicUsize::new(usize::from(has_sessions)), }) } @@ -649,6 +690,13 @@ impl StableProcessHandle { /// Verify that this runtime can signal exact process instances before any PTY is spawned. pub fn require_stable_process_signaling() -> io::Result<()> { + #[cfg(test)] + if FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.get() { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "forced process-session preflight failure", + )); + } let pid = libc::pid_t::try_from(std::process::id()) .map_err(|_| io::Error::new(io::ErrorKind::Unsupported, "invalid process id"))?; let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { @@ -995,6 +1043,22 @@ mod tests { assert_eq!(error.kind(), io::ErrorKind::Unsupported); } + #[cfg(target_os = "linux")] + #[test] + fn missing_process_enumeration_fails_the_runtime_preflight() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_BLOCK_PROCESS_ENUMERATION"; + if !enter_syscall_blocked_subprocess( + CHILD_ENV, + "process_session::tests::missing_process_enumeration_fails_the_runtime_preflight", + &[libc::SYS_getdents64], + ) { + return; + } + + let error = require_stable_process_signaling().unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::Unsupported); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn shutdown_batch_reuses_one_process_table_snapshot() { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 82fd72eb1fb2..33bdffafdf06 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -3354,6 +3354,67 @@ mod tests { assert!(reaped, "the PTY leader remained unreaped after cleanup could succeed"); } + #[cfg(unix)] + #[test] + fn persistent_natural_cleanup_uses_one_shared_reaper() { + const CHILDREN: usize = 8; + + let root = std::env::temp_dir() + .join(format!("cmux-local-reap-bound-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let release_path = root.join("release"); + let mux = Mux::new_for_test("local-reap-bound", SurfaceOptions::default()); + let baseline = crate::process_session::dedicated_natural_reap_workers_for_test(); + let mut surfaces = Vec::new(); + let mut processes = Vec::new(); + for index in 0..CHILDREN { + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!("while [ ! -e {} ]; do sleep 0.01; done", release_path.display()), + ]), + ..SurfaceOptions::default() + }; + let surface = Surface::spawn( + SurfaceId::try_from(index + 1).unwrap(), + options, + Arc::downgrade(&mux), + ) + .unwrap(); + let process = + match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; + process.normal_cleanup_failures.store(usize::MAX, Ordering::Release); + surfaces.push(surface); + processes.push(process); + } + std::fs::write(&release_path, b"ready").unwrap(); + + let deadline = Instant::now() + Duration::from_secs(3); + while crate::process_session::dedicated_natural_reap_workers_for_test() + < baseline + CHILDREN + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let dedicated = crate::process_session::dedicated_natural_reap_workers_for_test() + .saturating_sub(baseline); + + for process in &processes { + let _ = process.terminate_and_wait(Instant::now() + Duration::from_secs(2)); + } + drop(surfaces); + let _ = std::fs::remove_dir_all(root); + + assert!( + dedicated <= 1, + "persistent natural cleanup retained {dedicated} dedicated retry workers" + ); + } + #[cfg(unix)] #[test] fn explicit_shutdown_deadline_does_not_wait_for_natural_cleanup_sweep() { diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 58bf15d73738..741ebce2fac0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -7,6 +7,8 @@ use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; use anyhow::Context; use fs4::FileExt; @@ -175,6 +177,8 @@ pub struct WorkspaceRegistry { generation: String, session_name: String, _lease: Option, + #[cfg(test)] + terminal_snapshot_count: AtomicUsize, } impl std::fmt::Debug for WorkspaceRegistry { @@ -295,7 +299,15 @@ impl WorkspaceRegistry { if quick_check != "ok" { anyhow::bail!("workspace registry integrity check failed: {quick_check}"); } - Ok(Self { connection, registry_id, generation: new_uuid_v4(), session_name, _lease: lease }) + Ok(Self { + connection, + registry_id, + generation: new_uuid_v4(), + session_name, + _lease: lease, + #[cfg(test)] + terminal_snapshot_count: AtomicUsize::new(0), + }) } pub fn snapshot(&self) -> anyhow::Result { @@ -348,6 +360,8 @@ impl WorkspaceRegistry { /// Returns the canonical, non-tombstoned terminal placement projection. /// Runtime surface ids and renderer process ids are intentionally absent. pub fn terminal_snapshot(&self) -> anyhow::Result { + #[cfg(test)] + self.terminal_snapshot_count.fetch_add(1, Ordering::Relaxed); let revision = current_terminal_revision(&self.connection)?; let mut statement = self.connection.prepare( "SELECT terminal_id, workspace_key, incarnation, lifecycle, @@ -376,6 +390,16 @@ impl WorkspaceRegistry { }) } + #[cfg(test)] + pub(crate) fn reset_terminal_snapshot_count_for_test(&self) { + self.terminal_snapshot_count.store(0, Ordering::Relaxed); + } + + #[cfg(test)] + pub(crate) fn terminal_snapshot_count_for_test(&self) -> usize { + self.terminal_snapshot_count.load(Ordering::Relaxed) + } + /// Includes tombstones and is intended for reconciliation and idempotent /// close handling, not frontend materialization. pub fn terminal_record(&self, terminal_id: &str) -> anyhow::Result> { From 2ea0dd979e1a0b84ba3f00e3354715c8959d753c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:08:07 -0700 Subject: [PATCH 087/521] fix(tui): give lifecycle cleanup bounded owners --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 396 ++++++++++---- .../cmux-tui-core/src/process_session.rs | 501 ++++++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 83 ++- .../src/terminal_host_runtime.rs | 82 ++- .../cmux-tui-core/src/workspace_registry.rs | 12 + 5 files changed, 840 insertions(+), 234 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 51dd0876738d..bd33d3455e29 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1,7 +1,7 @@ //! The multiplexer: owns the session [`State`] and every surface runtime, //! and broadcasts [`MuxEvent`]s to subscribed frontends. -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; @@ -57,6 +57,8 @@ const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; #[cfg(unix)] const TERMINAL_ADOPTION_QUEUE_CAPACITY: usize = SHUTDOWN_OWNER_CAPACITY; #[cfg(unix)] +const TERMINAL_ADOPTION_DEFERRED_CAPACITY: usize = TERMINAL_ADOPTION_QUEUE_CAPACITY; +#[cfg(unix)] const TERMINAL_ADOPTION_WORKERS: usize = 4; #[cfg(not(test))] const SHUTDOWN_RECONCILE_MAX_ATTEMPTS: usize = 8; @@ -75,6 +77,8 @@ struct TerminalAdoptionTask { #[cfg(unix)] struct TerminalAdoptionQueueState { tasks: Vec, + deferred: VecDeque, + in_flight: usize, rescan_required: bool, next_rescan: Option, rescan_delay: Duration, @@ -87,6 +91,8 @@ impl Default for TerminalAdoptionQueueState { fn default() -> Self { Self { tasks: Vec::new(), + deferred: VecDeque::new(), + in_flight: 0, rescan_required: false, next_rescan: None, rescan_delay: Duration::from_millis(100), @@ -96,6 +102,47 @@ impl Default for TerminalAdoptionQueueState { } } +#[cfg(unix)] +impl TerminalAdoptionQueueState { + fn active_len(&self) -> usize { + self.tasks.len() + self.in_flight + } + + fn enqueue(&mut self, task: TerminalAdoptionTask) -> bool { + if self.active_len() < TERMINAL_ADOPTION_QUEUE_CAPACITY { + self.tasks.push(task); + return true; + } + if self.deferred.len() < TERMINAL_ADOPTION_DEFERRED_CAPACITY { + self.deferred.push_back(task); + return true; + } + false + } + + fn promote_deferred(&mut self) { + while self.active_len() < TERMINAL_ADOPTION_QUEUE_CAPACITY { + let Some(task) = self.deferred.pop_front() else { break }; + self.tasks.push(task); + } + } + + fn requeue_retry(&mut self, task: TerminalAdoptionTask) { + if let Some(next) = self.deferred.pop_front() { + self.tasks.push(next); + self.deferred.push_back(task); + } else { + self.tasks.push(task); + } + } + + fn rescan_due(&self, now: Instant) -> bool { + self.rescan_required + && self.deferred.is_empty() + && self.next_rescan.is_none_or(|next_rescan| next_rescan <= now) + } +} + #[cfg(unix)] #[derive(Default)] struct TerminalAdoptionCoordinator { @@ -1160,6 +1207,87 @@ impl Drop for AsyncSurfaceCreationGuard { } } +#[derive(Default)] +enum BrowserRuntimeSlotState { + #[default] + Empty, + Connecting, + Ready(Arc), + Stopping(Option>), +} + +#[derive(Default)] +struct BrowserRuntimeSlot { + state: Mutex, + changed: std::sync::Condvar, +} + +impl BrowserRuntimeSlot { + fn stop(&self) { + let mut state = self.state.lock().unwrap(); + let current = std::mem::take(&mut *state); + *state = match current { + BrowserRuntimeSlotState::Ready(runtime) => { + BrowserRuntimeSlotState::Stopping(Some(runtime)) + } + BrowserRuntimeSlotState::Stopping(runtime) => { + BrowserRuntimeSlotState::Stopping(runtime) + } + BrowserRuntimeSlotState::Empty | BrowserRuntimeSlotState::Connecting => { + BrowserRuntimeSlotState::Stopping(None) + } + }; + self.changed.notify_all(); + } + + fn take_for_shutdown(&self) -> Option> { + self.stop(); + let mut state = self.state.lock().unwrap(); + let BrowserRuntimeSlotState::Stopping(runtime) = &mut *state else { + unreachable!("stopped browser runtime slot has a stopping state"); + }; + runtime.take() + } + + fn restore_for_shutdown(&self, runtime: Arc) { + let mut state = self.state.lock().unwrap(); + let BrowserRuntimeSlotState::Stopping(current) = &mut *state else { + unreachable!("browser runtime can only be restored while stopping"); + }; + debug_assert!(current.is_none(), "browser runtime restored twice"); + *current = Some(runtime); + } + + fn take_on_drop(&mut self) -> Option> { + let state = self.state.get_mut().unwrap_or_else(std::sync::PoisonError::into_inner); + match std::mem::take(state) { + BrowserRuntimeSlotState::Ready(runtime) => Some(runtime), + BrowserRuntimeSlotState::Stopping(runtime) => runtime, + BrowserRuntimeSlotState::Empty | BrowserRuntimeSlotState::Connecting => None, + } + } + + #[cfg(test)] + fn install_for_test(&self, runtime: Arc) { + *self.state.lock().unwrap() = BrowserRuntimeSlotState::Ready(runtime); + } + + #[cfg(test)] + fn has_runtime_for_test(&self) -> bool { + match &*self.state.lock().unwrap() { + BrowserRuntimeSlotState::Ready(_) | BrowserRuntimeSlotState::Stopping(Some(_)) => true, + BrowserRuntimeSlotState::Empty + | BrowserRuntimeSlotState::Connecting + | BrowserRuntimeSlotState::Stopping(None) => false, + } + } + + #[cfg(test)] + fn lock_available_for_test(&self) -> bool { + self.state.try_lock().is_ok() + } +} + enum SurfaceResizeRestore { Complete(bool), Pending(Receiver), @@ -1396,7 +1524,7 @@ pub struct Mux { browser_runtime_connect: Mutex>>, #[cfg(test)] shutdown_owner_capacity: AtomicUsize, - browser_runtime: Mutex>>, + browser_runtime: BrowserRuntimeSlot, cell_pixels: Mutex<(u16, u16)>, default_colors: Mutex, sidebar_plugin: Mutex, @@ -1645,7 +1773,7 @@ impl Mux { browser_runtime_connect: Mutex::new(None), #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), - browser_runtime: Mutex::new(None), + browser_runtime: BrowserRuntimeSlot::default(), cell_pixels: Mutex::new((8, 16)), default_colors: Mutex::new(DefaultColors::default()), sidebar_plugin: Mutex::new(SidebarPluginRuntime::default()), @@ -2183,28 +2311,32 @@ impl Mux { if tracked.contains(&terminal_id) { return; } - if tracked.len() >= TERMINAL_ADOPTION_QUEUE_CAPACITY { - let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); - state.rescan_required = true; - state.next_rescan.get_or_insert_with(Instant::now); - self.terminal_adoption_coordinator.wake.notify_one(); - return; - } tracked.insert(terminal_id.clone()); } - self.terminal_adoption_coordinator.state.lock().unwrap().tasks.push(TerminalAdoptionTask { + let task = TerminalAdoptionTask { options, record, record_path, next_attempt: Instant::now() + Duration::from_millis(100), delay: Duration::from_millis(100), - }); + }; + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + if !state.enqueue(task) { + self.terminal_adoptions.lock().unwrap().remove(&terminal_id); + state.rescan_required = true; + state.next_rescan.get_or_insert_with(Instant::now); + self.terminal_adoption_coordinator.wake.notify_one(); + return; + } + drop(state); self.terminal_adoption_coordinator.wake.notify_one(); if self.ensure_terminal_adoption_workers().is_err() { self.terminal_adoptions.lock().unwrap().remove(&terminal_id); let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); state.tasks.retain(|task| task.record.terminal_id != terminal_id); + state.deferred.retain(|task| task.record.terminal_id != terminal_id); + state.promote_deferred(); state.rescan_required = true; state.next_rescan.get_or_insert_with(Instant::now); } @@ -2217,12 +2349,29 @@ impl Mux { let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(root) else { return false; }; + let registered = { + let registry = self.workspace_registry.lock().unwrap(); + let Ok(terminals) = registry.terminal_ids_including_tombstones() else { + return false; + }; + terminals.into_iter().collect::>() + }; + let live_counts = { + let state = self.state.lock().unwrap(); + let mut counts = HashMap::::new(); + for terminal_id in state + .surfaces + .values() + .filter_map(|surface| surface.terminal_host_identity()) + .map(|identity| identity.terminal_id) + { + *counts.entry(terminal_id).or_default() += 1; + } + counts + }; for (record_path, record) in records { - let already_live = self - .resolve_terminal(&record.terminal_id) - .ok() - .flatten() - .is_some_and(|resolution| resolution.surface.is_some()); + let already_live = registered.contains(&record.terminal_id) + && live_counts.get(&record.terminal_id).copied() == Some(1); if already_live { continue; } @@ -2253,11 +2402,9 @@ impl Mux { return; } - let tracked = mux.terminal_adoptions.lock().unwrap().len(); let mut state = coordinator.state.lock().unwrap(); let now = Instant::now(); - let rescan_due = state.next_rescan.is_none_or(|next_rescan| next_rescan <= now); - if state.rescan_required && rescan_due && tracked < TERMINAL_ADOPTION_QUEUE_CAPACITY { + if state.rescan_due(now) { state.rescan_required = false; state.next_rescan = None; drop(state); @@ -2274,18 +2421,22 @@ impl Mux { if let Some(index) = state.tasks.iter().position(|task| task.next_attempt <= now) { let mut task = state.tasks.swap_remove(index); + state.in_flight += 1; drop(state); let complete = mux.try_terminal_adoption(&task); - if complete { - mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); - coordinator.wake.notify_one(); - } else if mux.shutting_down.load(Ordering::Acquire) { + let mut state = coordinator.state.lock().unwrap(); + state.in_flight = + state.in_flight.checked_sub(1).expect("adoption worker tracks in-flight tasks"); + if complete || mux.shutting_down.load(Ordering::Acquire) { mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); } else { task.delay = (task.delay * 2).min(Duration::from_secs(5)); task.next_attempt = Instant::now() + task.delay; - coordinator.state.lock().unwrap().tasks.push(task); + state.requeue_retry(task); } + state.promote_deferred(); + drop(state); + coordinator.wake.notify_one(); continue; } @@ -2314,6 +2465,7 @@ impl Mux { let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); state.stopping = true; state.tasks.clear(); + state.deferred.clear(); self.terminal_adoption_coordinator.wake.notify_all(); } @@ -4174,25 +4326,67 @@ impl Mux { } fn browser_runtime(&self) -> anyhow::Result> { - let mut runtime = self.browser_runtime.lock().unwrap(); + loop { + let mut state = self.browser_runtime.state.lock().unwrap(); + if self.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } + match &*state { + BrowserRuntimeSlotState::Ready(runtime) if !runtime.is_closed() => { + return Ok(runtime.clone()); + } + BrowserRuntimeSlotState::Ready(_) => { + *state = BrowserRuntimeSlotState::Empty; + } + BrowserRuntimeSlotState::Empty => { + *state = BrowserRuntimeSlotState::Connecting; + break; + } + BrowserRuntimeSlotState::Connecting => { + drop(self.browser_runtime.changed.wait(state).unwrap()); + } + BrowserRuntimeSlotState::Stopping(_) => { + anyhow::bail!("server is shutting down"); + } + } + } + + let opts = self.surface_options.lock().unwrap().clone(); #[cfg(test)] if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { hook(); } - if self.is_shutting_down() { - anyhow::bail!("server is shutting down"); - } - if let Some(existing) = runtime.as_ref().filter(|existing| !existing.is_closed()) { - return Ok(existing.clone()); - } - let opts = self.surface_options.lock().unwrap().clone(); - let created = BrowserRuntime::connect(&opts)?; - if self.is_shutting_down() { - created.shutdown(); - anyhow::bail!("server is shutting down"); + let connected = BrowserRuntime::connect(&opts); + let mut state = self.browser_runtime.state.lock().unwrap(); + match connected { + Ok(created) + if matches!(*state, BrowserRuntimeSlotState::Connecting) + && !self.is_shutting_down() => + { + *state = BrowserRuntimeSlotState::Ready(created.clone()); + self.browser_runtime.changed.notify_all(); + Ok(created) + } + Ok(created) => { + self.browser_runtime.changed.notify_all(); + drop(state); + created.shutdown(); + anyhow::bail!("server is shutting down"); + } + Err(error) => { + let stopping = self.is_shutting_down() + || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); + if matches!(*state, BrowserRuntimeSlotState::Connecting) { + *state = BrowserRuntimeSlotState::Empty; + } + self.browser_runtime.changed.notify_all(); + drop(state); + if stopping { + anyhow::bail!("server is shutting down"); + } + Err(error) + } } - *runtime = Some(created.clone()); - Ok(created) } fn start_browser_bootstrap( @@ -4777,6 +4971,7 @@ impl Mux { pub fn shutdown(&self) { self.shutting_down.store(true, Ordering::Release); + self.browser_runtime.stop(); #[cfg(unix)] self.request_terminal_adoption_stop(); let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; @@ -4790,7 +4985,7 @@ impl Mux { surface.disconnect_for_daemon_shutdown(); } self.terminate_staged_shutdown_owners_until(deadline); - if let Some(runtime) = self.browser_runtime.lock().unwrap().take() { + if let Some(runtime) = self.browser_runtime.take_for_shutdown() { let _ = runtime.shutdown_until(deadline); } } @@ -4841,8 +5036,12 @@ impl Mux { /// closed so a retry cannot race a new PTY into the session. pub fn close_all_surfaces_for_shutdown(&self) -> anyhow::Result { let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + #[cfg(unix)] + crate::process_session::require_stable_process_signaling_until(deadline) + .context("preflight process control for server shutdown")?; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; self.shutting_down.store(true, Ordering::Release); + self.browser_runtime.stop(); #[cfg(unix)] self.request_terminal_adoption_stop(); if !self.surface_creations.stop_and_wait_until(deadline) { @@ -4964,7 +5163,7 @@ impl Mux { } self.terminate_staged_shutdown_owners_until(deadline); let retained = self.shutdown_owners.snapshot(); - let current_browser_runtime = self.browser_runtime.lock().unwrap().take(); + let current_browser_runtime = self.browser_runtime.take_for_shutdown(); let mut browser_runtimes = Vec::new(); for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { @@ -4988,7 +5187,7 @@ impl Mux { }); if surface_failed && runtime.source() == BrowserSource::External { if is_current { - *self.browser_runtime.lock().unwrap() = Some(runtime); + self.browser_runtime.restore_for_shutdown(runtime); } continue; } @@ -4999,7 +5198,7 @@ impl Mux { } else { browser_runtime_failed = true; if is_current { - *self.browser_runtime.lock().unwrap() = Some(runtime); + self.browser_runtime.restore_for_shutdown(runtime); } } } @@ -5076,6 +5275,7 @@ impl Mux { self.shutting_down.store(true, Ordering::Release); self.surface_creations.stop(); self.async_surface_creations.stop(); + self.browser_runtime.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); #[cfg(unix)] self.request_terminal_adoption_stop(); @@ -8970,9 +9170,7 @@ impl Drop for Mux { surface.disconnect_for_daemon_shutdown(); } } - if let Ok(runtime) = self.browser_runtime.get_mut() - && let Some(runtime) = runtime.take() - { + if let Some(runtime) = self.browser_runtime.take_on_drop() { runtime.shutdown(); } } @@ -11631,54 +11829,74 @@ mod tests { #[cfg(unix)] #[test] - fn terminal_adoption_overflow_refills_at_a_low_water_mark() { - let mux = Mux::new_for_test("adoption-low-water", SurfaceOptions::default()); - { - let mut tracked = mux.terminal_adoptions.lock().unwrap(); - tracked - .extend((0..TERMINAL_ADOPTION_QUEUE_CAPACITY).map(|index| format!("{index:032x}"))); - } - { - let mut state = mux.terminal_adoption_coordinator.state.lock().unwrap(); - state.rescan_required = true; - state.next_rescan = Some(Instant::now()); + fn terminal_adoption_overflow_uses_a_bounded_deferred_tier() { + fn task(index: usize) -> TerminalAdoptionTask { + let terminal_id = format!("{index:032x}"); + TerminalAdoptionTask { + options: SurfaceOptions::default(), + record: crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: terminal_id.clone(), + incarnation: terminal_id, + endpoint: String::new(), + owner_token: String::new(), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: String::new(), + supports_set_defaults: false, + supports_terminate_only: false, + }, + record_path: std::path::PathBuf::new(), + next_attempt: Instant::now(), + delay: Duration::from_millis(100), + } } - mux.ensure_terminal_adoption_workers().unwrap(); - - mux.terminal_adoptions.lock().unwrap().remove(&format!("{:032x}", 0)); - mux.terminal_adoption_coordinator.wake.notify_all(); - std::thread::sleep(Duration::from_millis(200)); - let retained_after_one_slot = - mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required; - { - let mut tracked = mux.terminal_adoptions.lock().unwrap(); - while tracked.len() > TERMINAL_ADOPTION_QUEUE_CAPACITY / 2 { - let terminal_id = tracked.iter().next().unwrap().clone(); - tracked.remove(&terminal_id); - } + let mut state = TerminalAdoptionQueueState::default(); + for index in 0..TERMINAL_ADOPTION_QUEUE_CAPACITY { + assert!(state.enqueue(task(index))); } - mux.terminal_adoption_coordinator.wake.notify_all(); - let deadline = Instant::now() + Duration::from_secs(1); - loop { - if !mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required - || Instant::now() >= deadline - { - break; - } - std::thread::sleep(Duration::from_millis(10)); + for index in TERMINAL_ADOPTION_QUEUE_CAPACITY..TERMINAL_ADOPTION_QUEUE_CAPACITY * 2 { + assert!(state.enqueue(task(index))); } - let refilled = !mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required; - mux.request_terminal_adoption_stop(); assert!( - mux.wait_for_terminal_adoption_workers_until(Instant::now() + Duration::from_secs(2)) + !state.enqueue(task(TERMINAL_ADOPTION_QUEUE_CAPACITY * 2)), + "adoption overflow exceeded both bounded queue tiers" + ); + state.rescan_required = true; + state.next_rescan = Some(Instant::now()); + assert!( + !state.rescan_due(Instant::now()), + "adoption overflow rescanned while deferred records remained" ); + let retry = state.tasks.pop().unwrap(); + let retry_id = retry.record.terminal_id.clone(); + state.in_flight += 1; + state.in_flight -= 1; + state.requeue_retry(retry); + assert!( + state.tasks.iter().any(|task| { + task.record.terminal_id == format!("{TERMINAL_ADOPTION_QUEUE_CAPACITY:032x}") + }), + "a permanently retrying active record starved the oldest deferred record" + ); + assert_eq!( + state.deferred.back().map(|task| task.record.terminal_id.as_str()), + Some(retry_id.as_str()), + "a retry did not rotate behind deferred adoption work" + ); + + for _ in 0..TERMINAL_ADOPTION_DEFERRED_CAPACITY { + state.tasks.pop().unwrap(); + state.promote_deferred(); + } + assert_eq!(state.active_len(), TERMINAL_ADOPTION_QUEUE_CAPACITY); + assert!(state.deferred.is_empty()); assert!( - retained_after_one_slot, - "adoption overflow triggered a full rescan after one queue slot opened" + state.rescan_due(Instant::now()), + "adoption overflow did not request one new batch after draining deferred records" ); - assert!(refilled, "adoption overflow did not refill after reaching its low-water mark"); } #[cfg(unix)] @@ -12176,7 +12394,7 @@ mod tests { "session-1", ); insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface).unwrap(); - *mux.browser_runtime.lock().unwrap() = Some(runtime); + mux.browser_runtime.install_for_test(runtime); let result = mux.close_all_surfaces_for_shutdown(); @@ -12185,7 +12403,7 @@ mod tests { "owned launched browser was not used as the authoritative shutdown fallback: {result:?}" ); assert!(mux.shutdown_owners.is_empty()); - assert!(mux.browser_runtime.lock().unwrap().is_none()); + assert!(!mux.browser_runtime.has_runtime_for_test()); } #[test] @@ -12353,7 +12571,7 @@ mod tests { release_bootstrap_tx.send(()).unwrap(); assert_eq!(shutdown_done_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(), 1); shutdown.join().unwrap(); - assert!(mux.browser_runtime.lock().unwrap().is_none()); + assert!(!mux.browser_runtime.has_runtime_for_test()); } #[test] @@ -12377,7 +12595,7 @@ mod tests { mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); connect_started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); - let slot_available = mux.browser_runtime.try_lock().is_ok(); + let slot_available = mux.browser_runtime.lock_available_for_test(); release_connect_tx.send(()).unwrap(); let deadline = Instant::now() + Duration::from_secs(1); loop { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 585be99354bc..00046cfe4e06 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -9,18 +9,27 @@ use std::collections::{HashMap, HashSet}; use std::io; #[cfg(target_os = "macos")] use std::mem::size_of; -#[cfg(test)] -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Condvar, Mutex}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::time::{Duration, Instant}; const SESSION_KILL_POLL: Duration = Duration::from_millis(5); +const PROCESS_SESSION_PREFLIGHT_TIMEOUT: Duration = Duration::from_secs(1); const NATURAL_REAP_RETRY_INITIAL: Duration = Duration::from_millis(25); const NATURAL_REAP_RETRY_MAX: Duration = Duration::from_secs(1); +const NATURAL_REAP_DEGRADED_RETRY: Duration = Duration::from_secs(30); +const NATURAL_REAP_BATCH_WINDOW: Duration = Duration::from_millis(5); +const NATURAL_REAP_CAPACITY: usize = 4_096; +#[cfg(not(test))] +const NATURAL_REAP_MAX_ATTEMPTS: usize = 8; +#[cfg(test)] +const NATURAL_REAP_MAX_ATTEMPTS: usize = 3; #[cfg(test)] -static DEDICATED_NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); +static NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); + +static NATURAL_REAPER: OnceLock>> = OnceLock::new(); #[cfg(test)] thread_local! { @@ -28,27 +37,9 @@ thread_local! { const { std::cell::Cell::new(false) }; } -#[cfg(test)] -struct DedicatedNaturalReapWorker; - -#[cfg(test)] -impl DedicatedNaturalReapWorker { - fn enter() -> Self { - DEDICATED_NATURAL_REAP_WORKERS.fetch_add(1, Ordering::AcqRel); - Self - } -} - -#[cfg(test)] -impl Drop for DedicatedNaturalReapWorker { - fn drop(&mut self) { - DEDICATED_NATURAL_REAP_WORKERS.fetch_sub(1, Ordering::AcqRel); - } -} - #[cfg(test)] pub(crate) fn dedicated_natural_reap_workers_for_test() -> usize { - DEDICATED_NATURAL_REAP_WORKERS.load(Ordering::Acquire) + NATURAL_REAP_WORKERS.load(Ordering::Acquire) } #[cfg(test)] @@ -60,7 +51,6 @@ pub(crate) fn set_process_session_preflight_failure_for_test(enabled: bool) { /// shutdown. The session leader remains reserved until this state machine /// confirms that cleanup completed and performs the sole final reap. pub(crate) struct ReservedChildReap<'a> { - pub(crate) changed: &'a (Mutex, Condvar), pub(crate) signal_lock: &'a Mutex<()>, pub(crate) pty_drained: &'a AtomicBool, pub(crate) termination_started: &'a AtomicBool, @@ -68,71 +58,360 @@ pub(crate) struct ReservedChildReap<'a> { pub(crate) child_reaped: &'a AtomicBool, } -/// Reap a waitable PTY session leader without holding its signal lock during -/// process-table scans. A transient natural-cleanup failure is retried even -/// when no other lifecycle event arrives. -pub(crate) fn reap_reserved_session_leader( - sync: ReservedChildReap<'_>, +enum NaturalReaperCommand { + Add(NaturalReapRequest), + Wake, +} + +struct NaturalReapRequest { + session: libc::pid_t, cleanup_timeout: Duration, - mut cleanup: impl FnMut(Instant) -> bool, + needs_cleanup: Box bool + Send>, + prepare_cleanup: Box bool + Send>, + finish: Box bool + Send>, + _lease: ReservedChildReaperLease, + next_attempt: Instant, + retry_delay: Duration, + attempts: usize, + degraded: bool, +} + +struct NaturalReaper { + sender: mpsc::Sender, + active: Arc, + degraded: Arc, + wake_pending: Arc, + _worker: std::thread::JoinHandle<()>, +} + +pub(crate) struct ReservedChildReaperLease { + sender: mpsc::Sender, + active: Arc, + degraded: Arc, +} + +impl Drop for ReservedChildReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +impl NaturalReaper { + fn start() -> io::Result { + let (sender, receiver) = mpsc::channel(); + let active = Arc::new(AtomicUsize::new(0)); + let degraded = Arc::new(AtomicUsize::new(0)); + let wake_pending = Arc::new(AtomicBool::new(false)); + let worker_wake_pending = wake_pending.clone(); + let worker = std::thread::Builder::new().name("cmux-pty-session-reaper".into()).spawn( + move || { + #[cfg(test)] + NATURAL_REAP_WORKERS.fetch_add(1, Ordering::AcqRel); + run_natural_reaper(receiver, worker_wake_pending); + #[cfg(test)] + NATURAL_REAP_WORKERS.fetch_sub(1, Ordering::AcqRel); + }, + )?; + Ok(Self { sender, active, degraded, wake_pending, _worker: worker }) + } + + fn lease(&self) -> io::Result { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < NATURAL_REAP_CAPACITY).then_some(active + 1) + }) + .map_err(|_| { + io::Error::new(io::ErrorKind::WouldBlock, "PTY session reaper capacity exhausted") + })?; + Ok(ReservedChildReaperLease { + sender: self.sender.clone(), + active: self.active.clone(), + degraded: self.degraded.clone(), + }) + } +} + +pub(crate) fn reserve_child_reaper() -> io::Result { + let mut slot = NATURAL_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); + if slot.is_none() { + *slot = Some(NaturalReaper::start()?); + } + slot.as_ref().expect("natural PTY reaper initialized").lease() +} + +pub(crate) fn wake_child_reaper() { + let Some(slot) = NATURAL_REAPER.get() else { return }; + let slot = slot.lock().unwrap(); + let Some(reaper) = slot.as_ref() else { return }; + if !reaper.wake_pending.swap(true, Ordering::AcqRel) { + let _ = reaper.sender.send(NaturalReaperCommand::Wake); + } +} + +pub(crate) fn enqueue_reserved_session_leader( + lease: ReservedChildReaperLease, + session: libc::pid_t, + cleanup_timeout: Duration, + needs_cleanup: impl Fn() -> bool + Send + 'static, + prepare_cleanup: impl FnMut() -> bool + Send + 'static, + finish: impl FnMut(bool) -> bool + Send + 'static, +) { + let sender = lease.sender.clone(); + sender + .send(NaturalReaperCommand::Add(NaturalReapRequest { + session, + cleanup_timeout, + needs_cleanup: Box::new(needs_cleanup), + prepare_cleanup: Box::new(prepare_cleanup), + finish: Box::new(finish), + _lease: lease, + next_attempt: Instant::now(), + retry_delay: NATURAL_REAP_RETRY_INITIAL, + attempts: 0, + degraded: false, + })) + .expect("natural PTY reaper remains alive while its service is registered"); +} + +pub(crate) fn reserved_child_needs_cleanup(sync: ReservedChildReap<'_>) -> bool { + let _signal = sync.signal_lock.lock().unwrap(); + !sync.cleanup_complete.load(Ordering::Acquire) + && !sync.termination_started.load(Ordering::Acquire) + && sync.pty_drained.load(Ordering::Acquire) +} + +pub(crate) fn poll_reserved_session_leader( + sync: ReservedChildReap<'_>, + cleanup_succeeded: bool, reap: impl FnOnce(), +) -> bool { + let should_attempt_cleanup = { + let _signal = sync.signal_lock.lock().unwrap(); + if sync.cleanup_complete.load(Ordering::Acquire) { + reap(); + sync.child_reaped.store(true, Ordering::Release); + return true; + } + !sync.termination_started.load(Ordering::Acquire) + && sync.pty_drained.load(Ordering::Acquire) + }; + + if should_attempt_cleanup { + let _signal = sync.signal_lock.lock().unwrap(); + let cleanup_claimed = sync.cleanup_complete.load(Ordering::Acquire); + let termination_started = sync.termination_started.load(Ordering::Acquire); + if cleanup_claimed || (cleanup_succeeded && !termination_started) { + sync.cleanup_complete.store(true, Ordering::Release); + reap(); + sync.child_reaped.store(true, Ordering::Release); + return true; + } + } + false +} + +fn run_natural_reaper( + receiver: mpsc::Receiver, + wake_pending: Arc, ) { - #[cfg(test)] - let _dedicated_worker = DedicatedNaturalReapWorker::enter(); - let mut reap = Some(reap); - let mut retry_delay = NATURAL_REAP_RETRY_INITIAL; + let mut pending = Vec::::new(); loop { - let should_attempt_cleanup = { - let _signal = sync.signal_lock.lock().unwrap(); - if sync.cleanup_complete.load(Ordering::Acquire) { - reap.take().expect("reserved child is reaped once")(); - sync.child_reaped.store(true, Ordering::Release); - return; - } - !sync.termination_started.load(Ordering::Acquire) - && sync.pty_drained.load(Ordering::Acquire) + let received = if pending.is_empty() { + receiver.recv().map_err(|_| RecvTimeoutError::Disconnected) + } else { + let now = Instant::now(); + let wait = pending + .iter() + .map(|request| request.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(NATURAL_REAP_RETRY_MAX); + receiver.recv_timeout(wait) }; + match received { + Ok(command) => { + accept_natural_reaper_command(command, &mut pending, wake_pending.as_ref()); + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) if pending.is_empty() => return, + Err(RecvTimeoutError::Disconnected) => { + std::thread::sleep(NATURAL_REAP_RETRY_MAX); + } + } - let cleanup_succeeded = should_attempt_cleanup && cleanup(Instant::now() + cleanup_timeout); - if should_attempt_cleanup { - let _signal = sync.signal_lock.lock().unwrap(); - let cleanup_claimed = sync.cleanup_complete.load(Ordering::Acquire); - let termination_started = sync.termination_started.load(Ordering::Acquire); - if cleanup_claimed || (cleanup_succeeded && !termination_started) { - sync.cleanup_complete.store(true, Ordering::Release); - reap.take().expect("reserved child is reaped once")(); - sync.child_reaped.store(true, Ordering::Release); - return; + let batch_deadline = Instant::now() + NATURAL_REAP_BATCH_WINDOW; + while let Some(remaining) = batch_deadline.checked_duration_since(Instant::now()) { + match receiver.recv_timeout(remaining) { + Ok(command) => { + accept_natural_reaper_command(command, &mut pending, wake_pending.as_ref()); + } + Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => break, } } - let state = sync.changed.0.lock().unwrap(); - let retry_natural_cleanup = should_attempt_cleanup - && !cleanup_succeeded - && !sync.termination_started.load(Ordering::Acquire) - && !sync.cleanup_complete.load(Ordering::Acquire); - if retry_natural_cleanup { - let (_state, _) = sync - .changed - .1 - .wait_timeout_while(state, retry_delay, |_| { - !sync.termination_started.load(Ordering::Acquire) - && !sync.cleanup_complete.load(Ordering::Acquire) - && sync.pty_drained.load(Ordering::Acquire) - }) - .unwrap(); - retry_delay = (retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + let now = Instant::now(); + let mut due = Vec::new(); + let mut index = 0; + while index < pending.len() { + if pending[index].next_attempt <= now { + due.push(pending.swap_remove(index)); + } else { + index += 1; + } + } + if due.is_empty() { + continue; + } + + let needs_cleanup = due.iter().map(|request| (request.needs_cleanup)()).collect::>(); + let prepared = due + .iter_mut() + .zip(&needs_cleanup) + .map(|(request, needed)| !needed || (request.prepare_cleanup)()) + .collect::>(); + let sessions = due + .iter() + .zip(needs_cleanup.iter().zip(&prepared)) + .filter_map(|(request, (needed, prepared))| { + (*needed && *prepared).then_some(request.session) + }) + .collect::>(); + let cleanup_timeout = due + .iter() + .zip(&needs_cleanup) + .filter_map(|(request, needed)| needed.then_some(request.cleanup_timeout)) + .max() + .unwrap_or_default(); + let cleanup_deadline = Instant::now() + cleanup_timeout; + let cleaned = if sessions.is_empty() { + HashSet::new() } else { - let _state = sync - .changed - .1 - .wait_while(state, |_| { - !sync.cleanup_complete.load(Ordering::Acquire) - && (sync.termination_started.load(Ordering::Acquire) - || !sync.pty_drained.load(Ordering::Acquire)) + SessionProcessSnapshot::capture(sessions.iter().copied(), cleanup_deadline) + .map(|snapshot| { + kill_sessions_until_only_leaders_from_snapshot( + &snapshot, + &sessions, + cleanup_deadline, + ) }) - .unwrap(); - retry_delay = NATURAL_REAP_RETRY_INITIAL; + .unwrap_or_default() + }; + + for ((mut request, needed), prepared) in due.into_iter().zip(needs_cleanup).zip(prepared) { + let cleanup_succeeded = needed && prepared && cleaned.contains(&request.session); + let done = (request.finish)(cleanup_succeeded); + if done { + if request.degraded { + request._lease.degraded.fetch_sub(1, Ordering::AcqRel); + } + continue; + } + if needed { + request.attempts = request.attempts.saturating_add(1); + } + if request.attempts >= NATURAL_REAP_MAX_ATTEMPTS { + if !request.degraded { + request.degraded = true; + request._lease.degraded.fetch_add(1, Ordering::AcqRel); + } + request.next_attempt = Instant::now() + NATURAL_REAP_DEGRADED_RETRY; + } else { + request.next_attempt = Instant::now() + request.retry_delay; + request.retry_delay = (request.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + } + pending.push(request); + } + } +} + +fn kill_sessions_until_only_leaders_from_snapshot( + snapshot: &SessionProcessSnapshot, + sessions: &HashSet, + deadline: Instant, +) -> HashSet { + let mut states = HashMap::)>::new(); + let mut failed = HashSet::new(); + for session in sessions { + match snapshot.members(*session) { + Ok((generation, members)) => { + if signal_members(&members, *session, None).is_ok() { + states.insert(*session, (generation, members)); + } else { + failed.insert(*session); + } + } + Err(_) => { + failed.insert(*session); + } + } + } + + let mut complete = HashSet::new(); + while complete.len() + failed.len() < sessions.len() && Instant::now() < deadline { + let mut refresh = Vec::new(); + for (session, (_, members)) in &states { + if complete.contains(session) || failed.contains(session) { + continue; + } + let drained = members + .iter() + .copied() + .filter(|pid| pid != session) + .map(|pid| still_member(pid, *session)) + .collect::>>() + .map(|members| members.into_iter().all(|alive| !alive)); + match drained { + Ok(true) => refresh.push(*session), + Ok(false) => {} + Err(_) => { + failed.insert(*session); + } + } + } + + if let Some(first) = refresh.first().copied() { + let observed_generation = states[&first].0; + if snapshot.refresh_members(first, observed_generation, deadline).is_err() { + break; + } + for session in refresh { + let Ok((generation, members)) = snapshot.members(session) else { + failed.insert(session); + continue; + }; + if members.iter().all(|pid| *pid == session) { + complete.insert(session); + continue; + } + if signal_members(&members, session, None).is_err() { + failed.insert(session); + continue; + } + states.insert(session, (generation, members)); + } + } + + if complete.len() + failed.len() < sessions.len() { + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(SESSION_KILL_POLL), + ); + } + } + complete +} + +fn accept_natural_reaper_command( + command: NaturalReaperCommand, + pending: &mut Vec, + wake_pending: &AtomicBool, +) { + match command { + NaturalReaperCommand::Add(request) => pending.push(request), + NaturalReaperCommand::Wake => { + wake_pending.store(false, Ordering::Release); + let now = Instant::now(); + for request in pending { + request.next_attempt = now; + } } } } @@ -688,8 +967,13 @@ impl StableProcessHandle { } } -/// Verify that this runtime can signal exact process instances before any PTY is spawned. +/// Verify that this runtime can enumerate PTY session members and signal exact +/// process instances before any PTY is spawned or shutdown topology mutates. pub fn require_stable_process_signaling() -> io::Result<()> { + require_stable_process_signaling_until(Instant::now() + PROCESS_SESSION_PREFLIGHT_TIMEOUT) +} + +pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::Result<()> { #[cfg(test)] if FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.get() { return Err(io::Error::new( @@ -703,16 +987,26 @@ pub fn require_stable_process_signaling() -> io::Result<()> { io::Error::new(io::ErrorKind::Unsupported, "current process identity is unavailable") })?; match process.matches_current() { - Ok(true) => Ok(()), - Ok(false) => Err(io::Error::new( - io::ErrorKind::Unsupported, - "stable process signaling did not retain the current process", - )), - Err(error) => Err(io::Error::new( - io::ErrorKind::Unsupported, - format!("stable process signaling is unavailable: {error}"), - )), + Ok(true) => {} + Ok(false) => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "stable process signaling did not retain the current process", + )); + } + Err(error) => { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("stable process signaling is unavailable: {error}"), + )); + } } + all_process_ids(Some(deadline)).map(|_| ()).map_err(|error| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("PTY session enumeration is unavailable: {error}"), + ) + }) } #[cfg(not(any(target_os = "macos", target_os = "linux")))] @@ -1096,10 +1390,9 @@ mod tests { .map(|child| libc::pid_t::try_from(child.id()).unwrap()) .collect::>(); let deadline = Instant::now() + Duration::from_secs(1); - let snapshot = std::sync::Arc::new( - SessionProcessSnapshot::capture(sessions.clone(), deadline).unwrap(), - ); - let barrier = std::sync::Arc::new(std::sync::Barrier::new(sessions.len())); + let snapshot = + Arc::new(SessionProcessSnapshot::capture(sessions.clone(), deadline).unwrap()); + let barrier = Arc::new(std::sync::Barrier::new(sessions.len())); std::thread::scope(|scope| { for session in &sessions { @@ -1123,4 +1416,26 @@ mod tests { "concurrent refreshes did not coalesce onto one new process-table snapshot" ); } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn natural_reap_batch_refreshes_all_sessions_with_one_scan() { + let mut children = [spawn_session_child(), spawn_session_child()]; + let sessions = children + .iter() + .map(|child| libc::pid_t::try_from(child.id()).unwrap()) + .collect::>(); + let deadline = Instant::now() + Duration::from_secs(1); + let snapshot = SessionProcessSnapshot::capture(sessions.iter().copied(), deadline).unwrap(); + + let completed = + kill_sessions_until_only_leaders_from_snapshot(&snapshot, &sessions, deadline); + let scan_count = snapshot.scan_count(); + + for child in &mut children { + child.wait().unwrap(); + } + assert_eq!(completed, sessions); + assert_eq!(scan_count, 2, "one natural-reap batch did not share its reconciliation scan"); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 33bdffafdf06..00e5c9ff8f94 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -17,6 +17,7 @@ use std::sync::mpsc::{ use std::sync::{Arc, Condvar, Mutex, TryLockError, Weak}; use std::time::{Duration, Instant}; +use anyhow::Context; use ghostty_vt::{ Callbacks, CursorShape, MouseEncoders, MouseInput, RenderFrame, RenderState, Rgb, Terminal, TerminalColorOverrides, @@ -835,6 +836,7 @@ impl LocalPtyProcess { fn spawn_reaper( self: &Arc, mut child: Box, + reaper: crate::process_session::ReservedChildReaperLease, ) -> std::io::Result<()> { let process = self.clone(); std::thread::Builder::new().name("surface-child".into()).spawn(move || { @@ -845,23 +847,55 @@ impl LocalPtyProcess { .and_then(|pid| libc::pid_t::try_from(pid).ok()) .is_some_and(|pid| wait_for_local_child_without_reaping(pid).is_ok()); if observed_without_reaping { + let session = + libc::pid_t::try_from(process.pid.expect("observed child has a PID")) + .expect("portable-pty child PID fits pid_t"); process.child_waitable.store(true, Ordering::Release); process.exited.1.notify_all(); - crate::process_session::reap_reserved_session_leader( - crate::process_session::ReservedChildReap { - changed: &process.exited, - signal_lock: &process.child_signal_lock, - pty_drained: &process.pty_drained, - termination_started: &process.termination_started, - cleanup_complete: &process.group_escalation_complete, - child_reaped: &process.child_reaped, - }, + let cleanup_process = process.clone(); + let prepare_process = process.clone(); + let reap_process = process.clone(); + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, - |deadline| process.kill_session_descendants_before_reap(deadline), - || { - let _ = child.wait(); + move || { + crate::process_session::reserved_child_needs_cleanup( + crate::process_session::ReservedChildReap { + signal_lock: &cleanup_process.child_signal_lock, + pty_drained: &cleanup_process.pty_drained, + termination_started: &cleanup_process.termination_started, + cleanup_complete: &cleanup_process.group_escalation_complete, + child_reaped: &cleanup_process.child_reaped, + }, + ) + }, + move || prepare_process.prepare_natural_cleanup(), + move |cleanup_succeeded| { + let done = crate::process_session::poll_reserved_session_leader( + crate::process_session::ReservedChildReap { + signal_lock: &reap_process.child_signal_lock, + pty_drained: &reap_process.pty_drained, + termination_started: &reap_process.termination_started, + cleanup_complete: &reap_process.group_escalation_complete, + child_reaped: &reap_process.child_reaped, + }, + cleanup_succeeded, + || { + let mut child = + child.take().expect("reserved child is reaped once"); + let _ = child.wait(); + }, + ); + if done { + *reap_process.exited.0.lock().unwrap() = true; + reap_process.exited.1.notify_all(); + } + done }, ); + return; } else { let _ = child.wait(); process.child_reaped.store(true, Ordering::Release); @@ -896,6 +930,7 @@ impl LocalPtyProcess { { self.pty_drained.store(true, Ordering::Release); self.exited.1.notify_all(); + crate::process_session::wake_child_reaper(); } } @@ -916,6 +951,7 @@ impl LocalPtyProcess { fn abandon_termination(&self) { self.termination_started.store(false, Ordering::Release); self.exited.1.notify_all(); + crate::process_session::wake_child_reaper(); } #[cfg(unix)] @@ -990,7 +1026,7 @@ impl LocalPtyProcess { } #[cfg(unix)] - fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + fn prepare_natural_cleanup(&self) -> bool { #[cfg(test)] { self.normal_cleanup_attempts.fetch_add(1, Ordering::AcqRel); @@ -1009,15 +1045,7 @@ impl LocalPtyProcess { return false; } } - let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { - return false; - }; - matches!( - crate::process_session::kill_until_only_leader(leader, leader, deadline, || { - self.child_waitable.load(Ordering::Acquire) - }), - Ok(true) - ) + true } fn terminate_and_wait(&self, deadline: Instant) -> bool { @@ -1046,6 +1074,7 @@ impl LocalPtyProcess { self.child_reaped.load(Ordering::Acquire) }; self.exited.1.notify_all(); + crate::process_session::wake_child_reaper(); if child_reaped { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { self.abandon_termination(); @@ -1078,6 +1107,7 @@ impl LocalPtyProcess { } self.group_escalation_complete.store(true, Ordering::Release); self.exited.1.notify_all(); + crate::process_session::wake_child_reaper(); } #[cfg(not(unix))] { @@ -1208,6 +1238,8 @@ impl Surface { return Self::spawn_hosted(id, opts, mux, attachment, true); } let _ = terminal_id; + let reaper = crate::process_session::reserve_child_reaper() + .context("reserve bounded PTY session cleanup")?; let pty = native_pty_system().openpty(PtySize { rows: opts.rows, cols: opts.cols, @@ -1314,7 +1346,7 @@ impl Surface { frame_requests, })); - process.spawn_reaper(child)?; + process.spawn_reaper(child, reaper)?; if let Err(error) = spawn_frame_producer(&surface, frame_rx) { let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); return Err(error); @@ -3394,8 +3426,9 @@ mod tests { std::fs::write(&release_path, b"ready").unwrap(); let deadline = Instant::now() + Duration::from_secs(3); - while crate::process_session::dedicated_natural_reap_workers_for_test() - < baseline + CHILDREN + while processes + .iter() + .any(|process| process.normal_cleanup_attempts.load(Ordering::Acquire) == 0) && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 3b2309f1eecc..7536a7673ae8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2417,7 +2417,7 @@ mod unix { }) } - fn kill_session_descendants_before_reap(&self, deadline: Instant) -> bool { + fn prepare_natural_cleanup(&self) -> bool { #[cfg(test)] { self.normal_cleanup_attempts.fetch_add(1, Ordering::AcqRel); @@ -2431,15 +2431,7 @@ mod unix { return false; } } - let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { - return false; - }; - matches!( - crate::process_session::kill_until_only_leader(leader, leader, deadline, || { - self.child_waitable.load(Ordering::Acquire) - }), - Ok(true) - ) + true } fn request_forced_pty_drain(&self) { @@ -2458,6 +2450,7 @@ mod unix { self.termination_started.swap(true, Ordering::AcqRel) }; self.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); if already_started { return; } @@ -2476,11 +2469,13 @@ mod unix { fn finish_group_escalation(&self) { self.group_escalation_complete.store(true, Ordering::Release); self.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); } fn abandon_group_escalation(&self) { self.termination_started.store(false, Ordering::Release); self.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); } fn publish_exit_if_drained(&self) { @@ -2501,6 +2496,7 @@ mod unix { self.child_reaped.load(Ordering::Acquire) }; self.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); if child_reaped { let session_empty = self .pid @@ -2860,6 +2856,8 @@ mod unix { launch: &HostLaunch, bootstrapped: &crate::terminal_host::BootstrappedHost, ) -> anyhow::Result> { + let reaper = crate::process_session::reserve_child_reaper() + .context("reserve bounded PTY session cleanup")?; let pty = native_pty_system().openpty(PtySize { rows: launch.rows, cols: launch.cols, @@ -3013,6 +3011,7 @@ mod unix { // child wait rendezvous, so clients can safely stop at Exit. reader_host.pty_drained.store(true, Ordering::Release); reader_host.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); reader_host.publish_exit_if_drained(); })?; let child_host = shared.clone(); @@ -3022,28 +3021,57 @@ mod unix { .and_then(|pid| libc::pid_t::try_from(pid).ok()) .is_some_and(|pid| wait_for_child_exit_without_reaping(pid).is_ok()); if observed_without_reaping { + let session = + libc::pid_t::try_from(child_host.pid.expect("observed child has a PID")) + .expect("portable-pty child PID fits pid_t"); child_host.child_waitable.store(true, Ordering::Release); child_host.child_exit.1.notify_all(); - crate::process_session::reap_reserved_session_leader( - crate::process_session::ReservedChildReap { - changed: &child_host.child_exit, - signal_lock: &child_host.child_signal_lock, - pty_drained: &child_host.pty_drained, - termination_started: &child_host.termination_started, - cleanup_complete: &child_host.group_escalation_complete, - child_reaped: &child_host.child_reaped, - }, + let cleanup_host = child_host.clone(); + let prepare_host = child_host.clone(); + let reap_host = child_host.clone(); + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, HOST_KILL_WAIT, - |deadline| child_host.kill_session_descendants_before_reap(deadline), - || { - let _ = child.wait(); + move || { + crate::process_session::reserved_child_needs_cleanup( + crate::process_session::ReservedChildReap { + signal_lock: &cleanup_host.child_signal_lock, + pty_drained: &cleanup_host.pty_drained, + termination_started: &cleanup_host.termination_started, + cleanup_complete: &cleanup_host.group_escalation_complete, + child_reaped: &cleanup_host.child_reaped, + }, + ) + }, + move || prepare_host.prepare_natural_cleanup(), + move |cleanup_succeeded| { + let done = crate::process_session::poll_reserved_session_leader( + crate::process_session::ReservedChildReap { + signal_lock: &reap_host.child_signal_lock, + pty_drained: &reap_host.pty_drained, + termination_started: &reap_host.termination_started, + cleanup_complete: &reap_host.group_escalation_complete, + child_reaped: &reap_host.child_reaped, + }, + cleanup_succeeded, + || { + let mut child = + child.take().expect("reserved child is reaped once"); + let _ = child.wait(); + }, + ); + if done { + let mut exited = reap_host.child_exit.0.lock().unwrap(); + *exited = true; + drop(exited); + reap_host.child_exit.1.notify_all(); + reap_host.publish_exit_if_drained(); + } + done }, ); - let mut exited = child_host.child_exit.0.lock().unwrap(); - *exited = true; - drop(exited); - child_host.child_exit.1.notify_all(); - child_host.publish_exit_if_drained(); } else { // Native Unix PTYs always expose a PID and support waitid; // retain a conservative fallback for alternate backends. diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 741ebce2fac0..bdda06273783 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -407,6 +407,18 @@ impl WorkspaceRegistry { read_terminal(&self.connection, terminal_id) } + pub(crate) fn terminal_ids_including_tombstones(&self) -> anyhow::Result> { + #[cfg(test)] + self.terminal_snapshot_count.fetch_add(1, Ordering::Relaxed); + let mut statement = self + .connection + .prepare("SELECT terminal_id FROM terminal_placements ORDER BY terminal_id ASC")?; + statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>() + .map_err(Into::into) + } + pub fn replay_terminal( &self, mutation: &WorkspaceMutation, From 65df52b3076e6aa1e46476405d67b68fbcddc730 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:46:26 -0700 Subject: [PATCH 088/521] fix(ci): route TUI package verification runner --- .github/workflows/cmux-tui-build-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 0bd5302d378f..451dd1448657 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -418,7 +418,7 @@ jobs: name: verify Linux package entrypoints (${{ matrix.architecture }}) needs: package if: ${{ inputs.package_npm || inputs.package_pypi }} - runs-on: ubuntu-latest + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 30 permissions: contents: read From a514152d0a02b9b69b6219679f394bfedcd606ef Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:25:30 -0700 Subject: [PATCH 089/521] test(tui): expose final bounded lifecycle gaps --- .../src/terminal_host_runtime.rs | 75 +++++++++++++++++-- .../crates/cmux-tui/src/server_lifecycle.rs | 38 +++++++++- 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 7536a7673ae8..1408c9a2d4cc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -306,12 +306,19 @@ mod unix { static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); #[cfg(test)] static NEXT_HOST_NORMAL_CLEANUP_FAILURES: AtomicUsize = AtomicUsize::new(0); + #[cfg(test)] + static NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES: AtomicUsize = AtomicUsize::new(0); struct SpawnedHostProcess { child: Option, } impl SpawnedHostProcess { + #[cfg(test)] + fn new_for_test(child: std::process::Child) -> Self { + Self { child: Some(child) } + } + fn child_mut(&mut self) -> &mut std::process::Child { self.child.as_mut().expect("terminal-host child is present") } @@ -338,15 +345,25 @@ mod unix { } fn detach_reaper(mut self) { - let Some(mut child) = self.child.take() else { return }; - let _ = thread::Builder::new().name("terminal-host-detached-reaper".into()).spawn( - move || { - let _ = child.wait(); - }, - ); + let Some(child) = self.child.take() else { return }; + let _ = spawn_host_process_reaper(child); } } + fn spawn_host_process_reaper(mut child: std::process::Child) -> std::io::Result<()> { + #[cfg(test)] + if NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1)) + .is_ok() + { + return Err(std::io::Error::other("forced terminal-host reaper spawn failure")); + } + thread::Builder::new().name("terminal-host-detached-reaper".into()).spawn(move || { + let _ = child.wait(); + })?; + Ok(()) + } + impl Drop for SpawnedHostProcess { fn drop(&mut self) { if let Some(child) = self.child.as_mut() { @@ -3867,6 +3884,52 @@ mod unix { assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); } + #[test] + fn detached_host_process_retains_reap_ownership_when_worker_spawn_fails() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + let child = + Command::new("/bin/sh").arg("-c").arg("exit 0").spawn().expect("spawn test child"); + let pid = libc::pid_t::try_from(child.id()).expect("child PID fits pid_t"); + let process = SpawnedHostProcess::new_for_test(child); + NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES.store(1, Ordering::Release); + + process.detach_reaper(); + + let deadline = Instant::now() + Duration::from_secs(1); + let reaped_by_owner = loop { + let mut status = 0; + // SAFETY: pid names the test-owned child and status is writable + // for the duration of this nonblocking wait. + let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; + if result == -1 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ECHILD) { + break true; + } + panic!("waitpid failed: {error}"); + } + if result == pid { + break false; + } + if Instant::now() >= deadline { + // SAFETY: pid remains the exact test-owned child until it + // is reaped, and this branch is only timeout cleanup. + unsafe { + libc::kill(pid, libc::SIGKILL); + libc::waitpid(pid, &raw mut status, 0); + } + break false; + } + thread::sleep(Duration::from_millis(10)); + }; + NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES.store(0, Ordering::Release); + + assert!( + reaped_by_owner, + "detached terminal-host cleanup abandoned the child after thread spawn failed" + ); + } + #[test] fn stalled_host_handshake_is_time_bounded() { let (record_path, record, lease) = record_fixture("handshake-timeout"); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index be176b9f0a50..b7160766771c 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -4,6 +4,8 @@ use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; #[cfg(unix)] use std::process::{Command, Stdio}; +#[cfg(test)] +use std::sync::atomic::AtomicUsize; #[cfg(unix)] use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; @@ -54,6 +56,8 @@ type TransportReader = BufReader>; #[cfg(unix)] static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); +#[cfg(test)] +static NEXT_BLOCKING_CONNECT_DELAY_MS: AtomicUsize = AtomicUsize::new(0); #[cfg(unix)] extern "C" fn handle_legacy_helper_signal(_: libc::c_int) { @@ -273,7 +277,7 @@ impl ServerProbe { path: &Path, deadline: Instant, ) -> anyhow::Result<(Self, TransportReader, Option)> { - let stream = transport::connect(path) + let stream = blocking_transport_connect(path) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.connect_failed))?; let peer_process_id = stream.peer_process_id().ok().flatten(); set_transport_deadline(stream.as_ref(), deadline)?; @@ -299,6 +303,17 @@ impl ServerProbe { } } +fn blocking_transport_connect(path: &Path) -> std::io::Result> { + #[cfg(test)] + { + let delay_ms = NEXT_BLOCKING_CONNECT_DELAY_MS.swap(0, Ordering::AcqRel); + if delay_ms > 0 { + std::thread::sleep(Duration::from_millis(delay_ms as u64)); + } + } + transport::connect(path) +} + #[derive(Debug)] pub(crate) struct IncompatibleLocalServer { message: String, @@ -1154,6 +1169,27 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn lifecycle_connect_deadline_does_not_enter_the_blocking_transport_primitive() { + let path = PathBuf::from("/tmp").join(format!( + "cmux-tui-connect-deadline-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + NEXT_BLOCKING_CONNECT_DELAY_MS.store(300, Ordering::Release); + let started = Instant::now(); + let result = ServerProbe::connect_until(&path, started + Duration::from_millis(50)); + let elapsed = started.elapsed(); + NEXT_BLOCKING_CONNECT_DELAY_MS.store(0, Ordering::Release); + + assert!(result.is_err(), "missing lifecycle socket unexpectedly connected"); + assert!( + elapsed < Duration::from_millis(250), + "lifecycle path entered an unbounded socket connect: {elapsed:?}" + ); + } + #[cfg(unix)] #[test] fn modern_shutdown_requires_a_success_response_before_disconnect() { From f02fd78b661873a12602a8590fe924b0000bd7c6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:21:16 -0700 Subject: [PATCH 090/521] fix(tui): bound final lifecycle ownership --- cmux-tui/Cargo.lock | 1 + cmux-tui/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-core/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-core/src/platform.rs | 495 +++++++++++++++++- .../src/terminal_host_runtime.rs | 248 ++++++--- .../crates/cmux-tui/src/server_lifecycle.rs | 51 +- 6 files changed, 699 insertions(+), 98 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 1369754a9648..c4f22b202521 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -252,6 +252,7 @@ dependencies = [ "sha2", "tungstenite", "uds_windows", + "windows-sys", "zeroize", ] diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index eb0284e66e39..7095bd92ca0e 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -47,6 +47,7 @@ base64 = "0.22" libc = "0.2" tungstenite = { version = "0.29", default-features = false, features = ["handshake"] } uds_windows = "1.2" +windows-sys = { version = "0.61", features = ["Win32_Networking_WinSock"] } regex = "1" bindgen = "0.72" smallvec = "1" diff --git a/cmux-tui/crates/cmux-tui-core/Cargo.toml b/cmux-tui/crates/cmux-tui-core/Cargo.toml index 93d3f28123e7..a382992a8e7d 100644 --- a/cmux-tui/crates/cmux-tui-core/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-core/Cargo.toml @@ -29,3 +29,4 @@ zeroize.workspace = true [target.'cfg(windows)'.dependencies] uds_windows.workspace = true +windows-sys.workspace = true diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 3dd0faf2a911..3688d4a178e1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -6,7 +6,7 @@ pub mod transport { use std::io::{self, Read, Write}; use std::net::Shutdown; use std::path::Path; - use std::time::Duration; + use std::time::{Duration, Instant}; pub trait Stream: Read + Write + Send + Sync { fn try_clone_box(&self) -> io::Result>; @@ -32,6 +32,10 @@ pub mod transport { imp::connect(path) } + pub fn connect_until(path: &Path, deadline: Instant) -> io::Result> { + imp::connect_until(path, deadline) + } + impl Listener { pub fn accept(&self) -> io::Result> { self.inner.accept() @@ -41,9 +45,12 @@ pub mod transport { #[cfg(unix)] mod imp { use std::io; + use std::mem::{offset_of, size_of}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + use std::os::unix::ffi::OsStrExt; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::Path; - use std::time::Duration; + use std::time::{Duration, Instant}; use super::Stream; @@ -59,6 +66,200 @@ pub mod transport { Ok(Box::new(UnixStream::connect(path)?)) } + pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { + ensure_connect_time_remaining(deadline)?; + let (address, address_len) = unix_socket_address(path)?; + // SAFETY: socket has no pointer arguments and returns a new owned + // descriptor on success. + let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; + if descriptor < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: descriptor is a fresh successful socket result and this + // OwnedFd takes its sole ownership. + let descriptor = unsafe { OwnedFd::from_raw_fd(descriptor) }; + // SAFETY: F_GETFD only reads flags from this valid descriptor. + let descriptor_flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFD) }; + if descriptor_flags < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: F_SETFD updates flags on this valid descriptor. + if unsafe { + libc::fcntl( + descriptor.as_raw_fd(), + libc::F_SETFD, + descriptor_flags | libc::FD_CLOEXEC, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + let stream = UnixStream::from(descriptor); + stream.set_nonblocking(true)?; + loop { + ensure_connect_time_remaining(deadline)?; + // SAFETY: address is an initialized sockaddr_un with its + // exact kernel-visible length, and stream owns a valid + // AF_UNIX socket. + let connected = unsafe { + libc::connect( + stream.as_raw_fd(), + (&raw const address).cast::(), + address_len, + ) + }; + if connected == 0 { + break; + } + let error = io::Error::last_os_error(); + let code = error.raw_os_error(); + if retry_connect_after_would_block(code, deadline)? { + continue; + } + let pending = [ + Some(libc::EAGAIN), + Some(libc::EINPROGRESS), + Some(libc::EWOULDBLOCK), + Some(libc::EALREADY), + Some(libc::EINTR), + ] + .contains(&code); + if !pending { + return Err(error); + } + wait_for_connect(&stream, deadline)?; + break; + } + stream.set_nonblocking(false)?; + Ok(Box::new(stream)) + } + + fn retry_connect_after_would_block( + code: Option, + deadline: Instant, + ) -> io::Result { + #[cfg(target_os = "linux")] + if code == Some(libc::EAGAIN) || code == Some(libc::EWOULDBLOCK) { + // Linux leaves a nonblocking AF_UNIX socket unconnected when + // the listener queue is full. The socket still polls writable + // with SO_ERROR zero, so retry connect instead of treating + // writability as completion. + let remaining = ensure_connect_time_remaining(deadline)?; + std::thread::sleep(remaining.min(Duration::from_millis(5))); + return Ok(true); + } + let _ = (code, deadline); + Ok(false) + } + + fn ensure_connect_time_remaining(deadline: Instant) -> io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::TimedOut, + "transport connection exceeded its deadline", + ) + }) + } + + fn wait_for_connect(stream: &UnixStream, deadline: Instant) -> io::Result<()> { + loop { + let remaining = ensure_connect_time_remaining(deadline)?; + let timeout_ms = remaining.as_millis().max(1).min(i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: stream.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for the + // duration of this call. + let result = unsafe { libc::poll(&raw mut descriptor, 1, timeout_ms) }; + if result == 0 { + continue; + } + if result < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if descriptor.revents & libc::POLLNVAL != 0 { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "transport connection socket became invalid", + )); + } + let mut socket_error = 0; + let mut socket_error_len = + libc::socklen_t::try_from(size_of::()).unwrap(); + // SAFETY: socket_error and its length describe a writable + // c_int, and stream owns a valid socket descriptor. + if unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_SOCKET, + libc::SO_ERROR, + (&raw mut socket_error).cast(), + &raw mut socket_error_len, + ) + } != 0 + { + return Err(io::Error::last_os_error()); + } + return if socket_error == 0 { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(socket_error)) + }; + } + } + + pub(super) fn unix_socket_address( + path: &Path, + ) -> io::Result<(libc::sockaddr_un, libc::socklen_t)> { + const SUN_PATH_CAPACITY: usize = + size_of::() - offset_of!(libc::sockaddr_un, sun_path); + let path = path.as_os_str().as_bytes(); + if path.is_empty() || path.len() >= SUN_PATH_CAPACITY || path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid transport Unix socket path", + )); + } + // SAFETY: all-zero is a valid starting representation for + // sockaddr_un; family, path, and platform length are set below. + let mut address = unsafe { std::mem::zeroed::() }; + address.sun_family = libc::AF_UNIX as libc::sa_family_t; + for (destination, source) in address.sun_path.iter_mut().zip(path) { + *destination = *source as libc::c_char; + } + let address_len = offset_of!(libc::sockaddr_un, sun_path) + path.len() + 1; + #[cfg(any( + target_os = "dragonfly", + target_os = "freebsd", + target_os = "macos", + target_os = "netbsd", + target_os = "openbsd" + ))] + { + address.sun_len = u8::try_from(address_len).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "transport Unix socket path is too long", + ) + })?; + } + Ok(( + address, + libc::socklen_t::try_from(address_len).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "transport Unix socket path is too long", + ) + })?, + )) + } + impl Listener { pub(super) fn accept(&self) -> io::Result> { let (stream, _) = self.inner.accept()?; @@ -159,11 +360,23 @@ pub mod transport { #[cfg(windows)] mod imp { use std::io; + use std::mem::{offset_of, size_of}; + use std::os::windows::io::{ + AsRawSocket, FromRawSocket, IntoRawSocket, OwnedSocket, RawSocket, + }; use std::path::Path; - use std::time::Duration; + use std::sync::OnceLock; + use std::time::{Duration, Instant}; use super::Stream; use uds_windows::{UnixListener, UnixStream}; + use windows_sys::Win32::Networking::WinSock::{ + AF_UNIX, FIONBIO, INVALID_SOCKET, POLLNVAL, POLLWRNORM, SO_ERROR, SOCK_STREAM, + SOCKADDR, SOCKADDR_UN, SOCKET, SOCKET_ERROR, SOL_SOCKET, WSA_FLAG_NO_HANDLE_INHERIT, + WSA_FLAG_OVERLAPPED, WSADATA, WSAEALREADY, WSAEINPROGRESS, WSAEINTR, WSAEWOULDBLOCK, + WSAGetLastError, WSAPOLLFD, WSAPoll, WSASocketW, WSAStartup, + connect as winsock_connect, getsockopt, ioctlsocket, + }; pub(super) struct Listener { inner: UnixListener, @@ -177,6 +390,185 @@ pub mod transport { Ok(Box::new(UnixStream::connect(path)?)) } + pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { + ensure_connect_time_remaining(deadline)?; + initialize_winsock()?; + let (address, address_len) = unix_socket_address(path)?; + // SAFETY: WSASocketW receives no borrowed protocol descriptor and + // returns a new socket handle on success. + let socket = unsafe { + WSASocketW( + AF_UNIX as i32, + SOCK_STREAM, + 0, + std::ptr::null(), + 0, + WSA_FLAG_OVERLAPPED | WSA_FLAG_NO_HANDLE_INHERIT, + ) + }; + if socket == INVALID_SOCKET { + return Err(last_winsock_error()); + } + // SAFETY: socket is a fresh successful WSASocketW result and this + // OwnedSocket takes its sole ownership. + let socket = unsafe { OwnedSocket::from_raw_socket(socket as RawSocket) }; + let socket_handle = winsock_handle(&socket)?; + let mut nonblocking = 1; + // SAFETY: socket is valid and nonblocking points to a writable + // u32 for the duration of the call. + if unsafe { ioctlsocket(socket_handle, FIONBIO, &raw mut nonblocking) } == SOCKET_ERROR + { + return Err(last_winsock_error()); + } + // SAFETY: address is an initialized SOCKADDR_UN with its exact + // Winsock-visible length. + let connected = unsafe { + winsock_connect(socket_handle, (&raw const address).cast::(), address_len) + }; + if connected == SOCKET_ERROR { + let error = last_winsock_error(); + let code = error.raw_os_error(); + let pending = + [Some(WSAEALREADY), Some(WSAEINPROGRESS), Some(WSAEINTR), Some(WSAEWOULDBLOCK)] + .contains(&code); + if !pending { + return Err(error); + } + wait_for_connect(socket_handle, deadline)?; + } + nonblocking = 0; + // SAFETY: socket remains valid and nonblocking points to a + // writable u32 for the duration of the call. + if unsafe { ioctlsocket(socket_handle, FIONBIO, &raw mut nonblocking) } == SOCKET_ERROR + { + return Err(last_winsock_error()); + } + let raw = socket.into_raw_socket(); + // SAFETY: ownership was transferred out of OwnedSocket exactly + // once and UnixStream accepts that same Winsock socket handle. + Ok(Box::new(unsafe { UnixStream::from_raw_socket(raw) })) + } + + fn winsock_handle(socket: &OwnedSocket) -> io::Result { + SOCKET::try_from(socket.as_raw_socket()).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "transport socket handle does not fit the Winsock ABI", + ) + }) + } + + fn initialize_winsock() -> io::Result<()> { + static STARTUP: OnceLock = OnceLock::new(); + let result = *STARTUP.get_or_init(|| { + // SAFETY: data is writable and lives for the duration of + // WSAStartup. Version 2.2 is the Winsock API used below. + unsafe { + let mut data = std::mem::zeroed::(); + WSAStartup(0x0202, &raw mut data) + } + }); + if result == 0 { Ok(()) } else { Err(io::Error::from_raw_os_error(result)) } + } + + fn ensure_connect_time_remaining(deadline: Instant) -> io::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::TimedOut, + "transport connection exceeded its deadline", + ) + }) + } + + fn wait_for_connect(socket: SOCKET, deadline: Instant) -> io::Result<()> { + loop { + let remaining = ensure_connect_time_remaining(deadline)?; + let timeout_ms = remaining.as_millis().max(1).min(i32::MAX as u128) as i32; + let mut descriptor = WSAPOLLFD { fd: socket, events: POLLWRNORM, revents: 0 }; + // SAFETY: descriptor points to one initialized WSAPOLLFD for + // the duration of this call. + let result = unsafe { WSAPoll(&raw mut descriptor, 1, timeout_ms) }; + if result == 0 { + continue; + } + if result == SOCKET_ERROR { + let error = last_winsock_error(); + if error.raw_os_error() == Some(WSAEINTR) { + continue; + } + return Err(error); + } + if descriptor.revents & POLLNVAL != 0 { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "transport connection socket became invalid", + )); + } + let mut socket_error = 0i32; + let mut socket_error_len = i32::try_from(size_of::()).unwrap(); + // SAFETY: socket_error and its length describe one writable + // i32, and socket remains a valid Winsock handle. + if unsafe { + getsockopt( + socket, + SOL_SOCKET, + SO_ERROR, + (&raw mut socket_error).cast(), + &raw mut socket_error_len, + ) + } == SOCKET_ERROR + { + return Err(last_winsock_error()); + } + return if socket_error == 0 { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(socket_error)) + }; + } + } + + fn unix_socket_address(path: &Path) -> io::Result<(SOCKADDR_UN, i32)> { + let path = path.to_str().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "transport Unix socket path is not valid UTF-8", + ) + })?; + let path = path.as_bytes(); + let capacity = size_of::() - offset_of!(SOCKADDR_UN, sun_path); + if path.is_empty() || path.len() >= capacity || path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid transport Unix socket path", + )); + } + let mut address = SOCKADDR_UN::default(); + address.sun_family = AF_UNIX; + for (destination, source) in address.sun_path.iter_mut().zip(path) { + *destination = *source as i8; + } + let length = offset_of!(SOCKADDR_UN, sun_path) + path.len() + 1; + Ok(( + address, + i32::try_from(length).map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidInput, + "transport Unix socket path is too long", + ) + })?, + )) + } + + fn last_winsock_error() -> io::Error { + // SAFETY: WSAGetLastError has no pointer arguments and reads the + // calling thread's Winsock error state. + io::Error::from_raw_os_error(unsafe { WSAGetLastError() }) + } + impl Listener { pub(super) fn accept(&self) -> io::Result> { let (stream, _) = self.inner.accept()?; @@ -210,6 +602,103 @@ pub mod transport { } } } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn expired_connect_deadline_fails_before_socket_resolution() { + let error = connect_until( + Path::new("deadline-expired-before-address-resolution"), + Instant::now() - Duration::from_millis(1), + ) + .err() + .expect("expired transport connect unexpectedly succeeded"); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + } + + #[cfg(unix)] + #[test] + fn deadline_connect_preserves_the_stream_contract() { + let path = std::env::temp_dir().join(format!( + "cmux-platform-connect-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + + let stream = connect_until(&path, Instant::now() + Duration::from_secs(1)).unwrap(); + let (_accepted, _) = listener.accept().unwrap(); + + drop(stream); + drop(listener); + std::fs::remove_file(path).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn deadline_connect_times_out_while_listener_backlog_is_saturated() { + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + + let path = std::env::temp_dir().join(format!( + "cmux-platform-backlog-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let (address, address_len) = imp::unix_socket_address(&path).unwrap(); + // SAFETY: socket has no pointer arguments and returns a new owned + // descriptor on success. + let listener = + unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + assert!(listener >= 0, "failed to create the test listener"); + // SAFETY: listener is a fresh successful socket result and this + // OwnedFd takes its sole ownership. + let listener = unsafe { OwnedFd::from_raw_fd(listener) }; + // SAFETY: address is initialized for this exact filesystem path + // and listener owns a valid AF_UNIX descriptor. + let result = unsafe { + libc::bind( + listener.as_raw_fd(), + (&raw const address).cast::(), + address_len, + ) + }; + assert_eq!(result, 0, "failed to bind the test listener"); + // SAFETY: listener remains a valid bound AF_UNIX descriptor. + let result = unsafe { libc::listen(listener.as_raw_fd(), 0) }; + assert_eq!(result, 0, "failed to create a zero-backlog test listener"); + let mut queued = Vec::new(); + let mut timeout_elapsed = None; + for _ in 0..32 { + let started = Instant::now(); + match connect_until(&path, started + Duration::from_millis(75)) { + Ok(stream) => queued.push(stream), + Err(error) => { + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + timeout_elapsed = Some(started.elapsed()); + break; + } + } + } + let elapsed = + timeout_elapsed.expect("could not saturate the listener backlog with 32 clients"); + assert!( + elapsed < Duration::from_secs(1), + "saturated listener connect exceeded its deadline bound: {elapsed:?}" + ); + drop(queued); + drop(listener); + std::fs::remove_file(path).unwrap(); + } + } } /// Runtime socket/pidfile directory for the current user. diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 1408c9a2d4cc..b551bd475c10 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -283,8 +283,10 @@ mod unix { use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; - use std::sync::mpsc::{SyncSender, TrySendError, sync_channel}; - use std::sync::{Arc, Condvar, Mutex}; + use std::sync::mpsc::{ + Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError, sync_channel, + }; + use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::thread; use std::time::{Duration, Instant}; @@ -302,6 +304,10 @@ mod unix { const HOST_LAUNCH_ROLLBACK_WAIT: Duration = Duration::from_secs(4); const HOST_LAUNCH_TIMEOUT: Duration = Duration::from_secs(10); const HOST_LAUNCH_CANCEL_POLL: Duration = Duration::from_millis(25); + const HOST_PROCESS_REAPER_CAPACITY: usize = 4_096; + const HOST_PROCESS_REAPER_POLL: Duration = Duration::from_millis(25); + const HOST_PROCESS_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); + static HOST_PROCESS_REAPER: OnceLock>> = OnceLock::new(); #[cfg(test)] static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); #[cfg(test)] @@ -309,24 +315,176 @@ mod unix { #[cfg(test)] static NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES: AtomicUsize = AtomicUsize::new(0); + struct HostProcessReaper { + sender: Sender, + active: Arc, + _worker: thread::JoinHandle<()>, + } + + struct HostProcessReaperLease { + sender: Sender, + active: Arc, + #[cfg(test)] + completion: Option>, + } + + impl Drop for HostProcessReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + #[cfg(test)] + if let Some(completion) = self.completion.take() { + let _ = completion.send(()); + } + } + } + + struct HostProcessReapRequest { + child: std::process::Child, + _lease: HostProcessReaperLease, + next_attempt: Instant, + retry_delay: Duration, + } + + impl HostProcessReaper { + fn start() -> std::io::Result { + let (sender, receiver) = std::sync::mpsc::channel(); + let active = Arc::new(AtomicUsize::new(0)); + let worker = spawn_host_process_reaper_worker(receiver)?; + Ok(Self { sender, active, _worker: worker }) + } + + fn lease(&self) -> std::io::Result { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < HOST_PROCESS_REAPER_CAPACITY).then_some(active + 1) + }) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "terminal-host process reaper capacity exhausted", + ) + })?; + Ok(HostProcessReaperLease { + sender: self.sender.clone(), + active: self.active.clone(), + #[cfg(test)] + completion: None, + }) + } + } + + fn reserve_host_process_reaper() -> std::io::Result { + let mut slot = HOST_PROCESS_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); + if slot.is_none() { + *slot = Some(HostProcessReaper::start()?); + } + slot.as_ref().expect("terminal-host process reaper initialized").lease() + } + + fn spawn_host_process_reaper_worker( + receiver: Receiver, + ) -> std::io::Result> { + #[cfg(test)] + if NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1)) + .is_ok() + { + return Err(std::io::Error::other("forced terminal-host reaper spawn failure")); + } + thread::Builder::new() + .name("terminal-host-process-reaper".into()) + .spawn(move || run_host_process_reaper(receiver)) + } + + fn run_host_process_reaper(receiver: Receiver) { + let mut pending = Vec::::new(); + loop { + let received = if pending.is_empty() { + receiver.recv().map_err(|_| RecvTimeoutError::Disconnected) + } else { + let now = Instant::now(); + let wait = pending + .iter() + .map(|request| request.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(HOST_PROCESS_REAPER_POLL); + receiver.recv_timeout(wait) + }; + match received { + Ok(request) => pending.push(request), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) if pending.is_empty() => return, + Err(RecvTimeoutError::Disconnected) => {} + } + pending.extend(receiver.try_iter()); + + let now = Instant::now(); + let mut index = 0; + while index < pending.len() { + if pending[index].next_attempt > now { + index += 1; + continue; + } + match pending[index].child.try_wait() { + Ok(Some(_)) => { + pending.swap_remove(index); + } + Ok(None) => { + let retry = pending[index].retry_delay; + pending[index].next_attempt = now + retry; + pending[index].retry_delay = + retry.saturating_mul(2).min(HOST_PROCESS_REAPER_RETRY_MAX); + index += 1; + } + Err(_) => { + let retry = pending[index].retry_delay; + pending[index].next_attempt = now + retry; + pending[index].retry_delay = + retry.saturating_mul(2).min(HOST_PROCESS_REAPER_RETRY_MAX); + index += 1; + } + } + } + } + } + + fn enqueue_host_process_reaper(lease: HostProcessReaperLease, child: std::process::Child) { + let sender = lease.sender.clone(); + sender + .send(HostProcessReapRequest { + child, + _lease: lease, + next_attempt: Instant::now(), + retry_delay: HOST_PROCESS_REAPER_POLL, + }) + .expect("terminal-host process reaper remains alive while registered"); + } + struct SpawnedHostProcess { child: Option, + reaper: Option, } impl SpawnedHostProcess { + fn with_reaper( + child: std::process::Child, + reaper: HostProcessReaperLease, + ) -> SpawnedHostProcess { + Self { child: Some(child), reaper: Some(reaper) } + } + #[cfg(test)] - fn new_for_test(child: std::process::Child) -> Self { - Self { child: Some(child) } + fn new_for_test(child: std::process::Child) -> (Self, Receiver<()>) { + let mut reaper = reserve_host_process_reaper().unwrap(); + let (completion, completed) = std::sync::mpsc::channel(); + reaper.completion = Some(completion); + (Self::with_reaper(child, reaper), completed) } fn child_mut(&mut self) -> &mut std::process::Child { self.child.as_mut().expect("terminal-host child is present") } - fn into_child(mut self) -> std::process::Child { - self.child.take().expect("terminal-host child is present") - } - fn wait_timeout(&mut self, timeout: Duration) -> bool { let deadline = Instant::now() + timeout; loop { @@ -334,6 +492,7 @@ mod unix { match child.try_wait() { Ok(Some(_)) => { self.child.take(); + self.reaper.take(); return true; } Ok(None) if Instant::now() < deadline => { @@ -346,24 +505,12 @@ mod unix { fn detach_reaper(mut self) { let Some(child) = self.child.take() else { return }; - let _ = spawn_host_process_reaper(child); + let reaper = + self.reaper.take().expect("spawned terminal host retains its reaper reservation"); + enqueue_host_process_reaper(reaper, child); } } - fn spawn_host_process_reaper(mut child: std::process::Child) -> std::io::Result<()> { - #[cfg(test)] - if NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1)) - .is_ok() - { - return Err(std::io::Error::other("forced terminal-host reaper spawn failure")); - } - thread::Builder::new().name("terminal-host-detached-reaper".into()).spawn(move || { - let _ = child.wait(); - })?; - Ok(()) - } - impl Drop for SpawnedHostProcess { fn drop(&mut self) { if let Some(child) = self.child.as_mut() { @@ -764,13 +911,7 @@ mod unix { /// kills and waits the child process through SpawnedHostProcess. pub(crate) fn commit_launched_host(&mut self) { let Some(process) = self.launch_process.take() else { return }; - let mut child = process.into_child(); - // Reaping is housekeeping after the ownership handoff. Failure to - // create this helper cannot turn a committed live Surface into an - // error; dropping Child leaves the independent host running. - let _ = thread::Builder::new().name("terminal-host-reaper".into()).spawn(move || { - let _ = child.wait(); - }); + process.detach_reaper(); } pub fn identity(&self) -> TerminalHostIdentity { @@ -1015,8 +1156,10 @@ mod unix { if libc::setsid() < 0 { Err(std::io::Error::last_os_error()) } else { Ok(()) } }); } + let reaper = reserve_host_process_reaper() + .context("reserve bounded terminal-host process cleanup")?; let child = command.spawn().context("spawn terminal-host process")?; - let mut process = SpawnedHostProcess { child: Some(child) }; + let mut process = SpawnedHostProcess::with_reaper(child, reaper); let host_pid = process.child_mut().id(); let stdin = process.child_mut().stdin.take().context("open terminal-host bootstrap stdin")?; @@ -3890,44 +4033,31 @@ mod unix { let child = Command::new("/bin/sh").arg("-c").arg("exit 0").spawn().expect("spawn test child"); let pid = libc::pid_t::try_from(child.id()).expect("child PID fits pid_t"); - let process = SpawnedHostProcess::new_for_test(child); + let (process, completed) = SpawnedHostProcess::new_for_test(child); NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES.store(1, Ordering::Release); process.detach_reaper(); - let deadline = Instant::now() + Duration::from_secs(1); - let reaped_by_owner = loop { - let mut status = 0; - // SAFETY: pid names the test-owned child and status is writable - // for the duration of this nonblocking wait. - let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; - if result == -1 { - let error = std::io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::ECHILD) { - break true; - } - panic!("waitpid failed: {error}"); - } - if result == pid { - break false; - } - if Instant::now() >= deadline { - // SAFETY: pid remains the exact test-owned child until it - // is reaped, and this branch is only timeout cleanup. - unsafe { - libc::kill(pid, libc::SIGKILL); - libc::waitpid(pid, &raw mut status, 0); - } - break false; - } - thread::sleep(Duration::from_millis(10)); - }; + let reaped_by_owner = completed.recv_timeout(Duration::from_secs(1)).is_ok(); NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES.store(0, Ordering::Release); + if !reaped_by_owner { + // SAFETY: pid remains the exact test-owned child while its + // reaper reservation is still outstanding. + unsafe { + libc::kill(pid, libc::SIGKILL); + } + } assert!( reaped_by_owner, "detached terminal-host cleanup abandoned the child after thread spawn failed" ); + let mut status = 0; + // SAFETY: completion proves the reaper dropped its sole Child + // handle after observing exit, so this call only verifies ECHILD. + let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; + assert_eq!(result, -1); + assert_eq!(std::io::Error::last_os_error().raw_os_error(), Some(libc::ECHILD)); } #[test] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index b7160766771c..0126ee664ab5 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -4,8 +4,6 @@ use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; #[cfg(unix)] use std::process::{Command, Stdio}; -#[cfg(test)] -use std::sync::atomic::AtomicUsize; #[cfg(unix)] use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; @@ -56,8 +54,6 @@ type TransportReader = BufReader>; #[cfg(unix)] static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); -#[cfg(test)] -static NEXT_BLOCKING_CONNECT_DELAY_MS: AtomicUsize = AtomicUsize::new(0); #[cfg(unix)] extern "C" fn handle_legacy_helper_signal(_: libc::c_int) { @@ -277,7 +273,7 @@ impl ServerProbe { path: &Path, deadline: Instant, ) -> anyhow::Result<(Self, TransportReader, Option)> { - let stream = blocking_transport_connect(path) + let stream = transport::connect_until(path, deadline) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.connect_failed))?; let peer_process_id = stream.peer_process_id().ok().flatten(); set_transport_deadline(stream.as_ref(), deadline)?; @@ -303,17 +299,6 @@ impl ServerProbe { } } -fn blocking_transport_connect(path: &Path) -> std::io::Result> { - #[cfg(test)] - { - let delay_ms = NEXT_BLOCKING_CONNECT_DELAY_MS.swap(0, Ordering::AcqRel); - if delay_ms > 0 { - std::thread::sleep(Duration::from_millis(delay_ms as u64)); - } - } - transport::connect(path) -} - #[derive(Debug)] pub(crate) struct IncompatibleLocalServer { message: String, @@ -1011,29 +996,25 @@ fn wait_for_disconnect_until( Ok(false) => return Ok(()), Ok(true) => {} Err(LifecycleLineError::Deadline) => { - if connection_is_gone(path) { + if !path.exists() { return Ok(()); } anyhow::bail!(crate::localization::catalog().server.shutdown_timed_out); } - Err(LifecycleLineError::Invalid | LifecycleLineError::Transport) => { - return shutdown_read_error(path); + Err(LifecycleLineError::Transport) => { + // A transport failure on the already-authenticated control + // stream is the disconnect this post-ACK wait is looking for. + // Reconnecting here can block on a saturated accept queue and + // cannot strengthen the identity already proven by the stream. + return Ok(()); + } + Err(LifecycleLineError::Invalid) => { + anyhow::bail!(crate::localization::catalog().server.transport_failed); } } } } -fn shutdown_read_error(path: &Path) -> anyhow::Result<()> { - if connection_is_gone(path) { - return Ok(()); - } - anyhow::bail!(crate::localization::catalog().server.transport_failed) -} - -fn connection_is_gone(path: &Path) -> bool { - transport::connect(path).is_err() -} - fn shell_quote(path: &Path) -> String { let value = path.display().to_string(); if !value.is_empty() @@ -1171,22 +1152,20 @@ mod tests { #[cfg(unix)] #[test] - fn lifecycle_connect_deadline_does_not_enter_the_blocking_transport_primitive() { + fn lifecycle_connect_deadline_expires_before_socket_resolution() { let path = PathBuf::from("/tmp").join(format!( "cmux-tui-connect-deadline-{}-{}.sock", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() )); - NEXT_BLOCKING_CONNECT_DELAY_MS.store(300, Ordering::Release); let started = Instant::now(); - let result = ServerProbe::connect_until(&path, started + Duration::from_millis(50)); + let result = ServerProbe::connect_until(&path, started - Duration::from_millis(1)); let elapsed = started.elapsed(); - NEXT_BLOCKING_CONNECT_DELAY_MS.store(0, Ordering::Release); assert!(result.is_err(), "missing lifecycle socket unexpectedly connected"); assert!( - elapsed < Duration::from_millis(250), - "lifecycle path entered an unbounded socket connect: {elapsed:?}" + elapsed < Duration::from_millis(50), + "expired lifecycle connect entered socket resolution: {elapsed:?}" ); } From 14c3e904fd8d8ebf5b556e1d035e8314f93223cf Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:21:10 -0700 Subject: [PATCH 091/521] test(tui): expose remaining upgrade ownership gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 39 ++++++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 95 ++++++++++++++++++++++ cmux-tui/crates/cmux-tui/src/cli.rs | 19 +++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 55 ++++++++++++- 4 files changed, 205 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index bc1f7db80dd8..a6eadc629c58 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -26,6 +26,8 @@ pub const CDP_EVENT_QUEUE_MAX_BYTES: usize = 32 * 1024 * 1024; #[cfg(test)] static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); +#[cfg(test)] +static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone, PartialEq, Eq)] pub struct ScreencastFrame { @@ -354,6 +356,13 @@ impl CdpClient { anyhow::bail!("CDP connection deadline expired"); } let endpoint = WsEndpoint::parse(web_socket_url)?; + #[cfg(test)] + { + let delay_ms = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); + if delay_ms != 0 { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } let mut addrs = (endpoint.host.as_str(), endpoint.port).to_socket_addrs()?; let addr = addrs.next().ok_or_else(|| { anyhow::anyhow!("no socket address for {}:{}", endpoint.host, endpoint.port) @@ -1182,7 +1191,7 @@ mod tests { use std::io::{Read, Write}; use std::net::TcpListener; use std::sync::mpsc::sync_channel; - use std::sync::{Arc, Barrier}; + use std::sync::{Arc, Barrier, Mutex}; use std::thread; use std::time::{Duration, Instant}; @@ -1190,6 +1199,8 @@ mod tests { use super::*; + static RESOLVE_TEST_LOCK: Mutex<()> = Mutex::new(()); + #[test] fn screencast_frame_rejects_terminal_control_bytes() { let params = json!({ @@ -1424,6 +1435,32 @@ mod tests { ); } + #[test] + fn websocket_resolution_uses_the_connection_deadline() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); + let (events, _receiver) = sync_channel(1); + + let started = Instant::now(); + let result = CdpClient::connect_with_timeout( + "ws://localhost:0/devtools/browser/fake", + events, + Duration::from_millis(50), + ); + let elapsed = started.elapsed(); + // A bounded resolver may still be finishing this injected operation + // after the caller's deadline. Let it drain before another test can + // use the process-wide resolver. + thread::sleep(Duration::from_millis(350)); + NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); + + assert!(result.is_err(), "invalid local endpoint unexpectedly connected"); + assert!( + elapsed < Duration::from_millis(200), + "hostname resolution exceeded the connection deadline: {elapsed:?}" + ); + } + #[test] fn concurrent_calls_complete_while_reader_receives_events() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bd33d3455e29..af9835db6a55 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -42,6 +42,8 @@ type TerminalAdoptionSurfaceFactory = Arc anyhow::Result> + Send + Sync>; #[cfg(test)] type NewPaneAfterSpawnHook = Arc) + Send + Sync>; +#[cfg(test)] +type BrowserTabAfterSpawnHook = Arc) + Send + Sync>; const TERMINAL_DIMENSION_MAX: u16 = 10_000; const WORKSPACE_REGISTRY_LIMIT: usize = 4_096; @@ -1513,6 +1515,8 @@ pub struct Mux { #[cfg(test)] new_pane_after_spawn: Mutex>, #[cfg(test)] + browser_tab_after_spawn: Mutex>, + #[cfg(test)] terminal_adoption_after_attach: Mutex>>, #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex>, @@ -1762,6 +1766,8 @@ impl Mux { #[cfg(test)] new_pane_after_spawn: Mutex::new(None), #[cfg(test)] + browser_tab_after_spawn: Mutex::new(None), + #[cfg(test)] terminal_adoption_after_attach: Mutex::new(None), #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex::new(None), @@ -6504,6 +6510,10 @@ impl Mux { }; let surface = self.spawn_browser_surface(url, size, None)?; + #[cfg(test)] + if let Some(hook) = self.browser_tab_after_spawn.lock().unwrap().clone() { + hook(surface.clone()); + } let active_at = self.next_active_at(); let notifications = self.surface_notifications(); let attached = { @@ -10426,6 +10436,91 @@ mod tests { .expect("browser worker exited after failed attach"); } + #[test] + fn failed_browser_tab_attach_retains_its_shutdown_owner() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + }); + let runtime = BrowserRuntime::connect_external_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + server.join().unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + while !runtime.is_closed() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(runtime.is_closed(), "test CDP runtime did not observe disconnect"); + + let mux = test_mux(); + let initial = mux.new_workspace(None, Some((80, 24))).unwrap(); + let pane = mux.with_state(|state| state.pane_of(initial.id).unwrap()); + let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); + let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); + let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); + *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ + move || { + bootstrap_reached_tx.send(()).unwrap(); + release_bootstrap_rx.lock().unwrap().recv().unwrap(); + } + })); + *mux.browser_tab_after_spawn.lock().unwrap() = Some(Arc::new({ + let mux = mux.clone(); + let runtime = runtime.clone(); + move |surface| { + surface.as_browser().unwrap().install_shutdown_session_for_test( + runtime.clone(), + "target-race", + "session-race", + ); + mux.state.lock().unwrap().panes.remove(&pane); + } + })); + + let result = mux.new_browser_tab("about:blank".into(), Some(pane), Some((80, 24))); + bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + mux.request_daemon_shutdown(); + release_bootstrap_tx.send(()).unwrap(); + let deadline = Instant::now() + Duration::from_secs(1); + while mux.async_surface_creations.inner.state.lock().unwrap().active != 0 + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + + assert!(result.is_err(), "browser attached after its pane disappeared"); + assert_eq!( + mux.shutdown_owners.len(), + 1, + "failed browser attachment discarded its retryable target owner" + ); + initial.kill(); + runtime.shutdown(); + } + #[test] fn notification_sets_unread_and_clears_when_tab_is_viewed() { let mux = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index d5a85cde02a3..f338f827080e 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -1922,6 +1922,25 @@ mod tests { assert_eq!(server_supports_capability(&mut reader, "unused"), Ok(false)); } + #[test] + fn completed_probe_drains_response_after_closed_peer_rejects_deadline_refresh() { + let read_timeouts = Arc::new(Mutex::new(Vec::new())); + let stream = ScriptedStream { + reads: VecDeque::from([Ok( + b"{\"id\":0,\"ok\":true,\"data\":{\"app\":\"cmux-tui\",\"capabilities\":[]}}\n" + .to_vec(), + )]), + current: io::Cursor::new(Vec::new()), + writes: Vec::new(), + read_timeouts, + fail_read_timeout_call: Some(3), + }; + transport::Stream::set_read_timeout(&stream, Some(Duration::from_millis(250))).unwrap(); + let mut reader = BufReader::new(Box::new(stream) as Box); + + assert_eq!(server_supports_capability(&mut reader, "unused"), Ok(false)); + } + #[test] fn plugin_verb_is_registered_as_local_with_help() { let plugin = verb_by_name("plugin").expect("plugin verb registered"); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 792258568457..c3cfb8883459 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -359,6 +359,34 @@ fn legacy_server_process_helper() { assert!(Instant::now() < deadline, "descendant did not become a zombie"); std::thread::sleep(Duration::from_millis(10)); } + } else if scenario == "dead-owned-surface" { + let mut command = Command::new("/bin/bash"); + command + .args([ + "--noprofile", + "--norc", + "-c", + concat!( + "trap '' HUP; ", + "(trap '' HUP; while :; do sleep 60; done) & ", + "printf '%s' \"$!\" > \"$CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE\"" + ), + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut direct = command.spawn().unwrap(); + assert!(direct.wait().unwrap().success()); + let pid_file = + PathBuf::from(std::env::var_os("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE").unwrap()); + let _ = wait_for_pid_file(&pid_file, Duration::from_secs(5)); } else if scenario == "reparent-on-close" { let mut command = Command::new("/bin/bash"); command @@ -499,6 +527,7 @@ fn legacy_server_process_helper() { | "unowned-surface" | "browser-surface" | "dead-surface" + | "dead-owned-surface" )); let surfaces: &[u64] = match (scenario.as_str(), snapshot_index) { ("zombie-child", _) => &[], @@ -511,7 +540,7 @@ fn legacy_server_process_helper() { .map(|surface| { if scenario == "browser-surface" { serde_json::json!({"surface": surface, "kind": "browser"}) - } else if scenario == "dead-surface" { + } else if matches!(scenario.as_str(), "dead-surface" | "dead-owned-surface") { serde_json::json!({"surface": surface, "kind": "pty", "dead": true}) } else { serde_json::json!({"surface": surface, "kind": "pty"}) @@ -1087,7 +1116,10 @@ fn packaged_launcher_name_is_used_in_upgrade_instructions() { assert_eq!(output.status.code(), Some(1)); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("cmux server stop"), "{stderr}"); - assert!(stderr.contains("then run `cmux` again"), "{stderr}"); + assert!( + stderr.contains(&format!("then run `cmux ping --socket {}` again", socket.display())), + "{stderr}" + ); server.join().unwrap(); let _ = fs::remove_file(&socket); let _ = fs::remove_dir_all(&dir); @@ -1472,6 +1504,25 @@ fn server_stop_closes_exited_legacy_placeholders_without_a_pid() { assert_eq!(status.signal(), Some(libc::SIGKILL)); } +#[cfg(unix)] +#[test] +fn server_stop_fails_closed_for_a_dead_legacy_surface_with_a_live_session() { + let mut server = + LegacyServerProcess::start("legacy-server-dead-owned", "dead-owned-surface", None); + let descendant_pid = wait_for_pid_file(&server.descendant_pid_file, Duration::from_secs(5)); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert!(!output.status.success(), "live process behind a dead surface was ignored"); + assert!(server.child.try_wait().unwrap().is_none(), "legacy server was killed on ambiguity"); + assert!(process_exists(descendant_pid), "ambiguous pane process was killed without ownership"); +} + #[cfg(unix)] #[test] fn published_host_launch_errors_reap_the_complete_pty_session() { From 8d9e8d88a5abd1cbf4c993cc6f4036463c2bbd03 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:52:07 -0700 Subject: [PATCH 092/521] fix(tui): close remaining upgrade ownership gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 185 ++++++++++++---- cmux-tui/crates/cmux-tui-core/src/mux.rs | 208 +++++++++--------- .../crates/cmux-tui/src/server_lifecycle.rs | 102 +++++---- 3 files changed, 310 insertions(+), 185 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index a6eadc629c58..a29f68bd0163 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1,9 +1,11 @@ use std::collections::{HashMap, VecDeque}; use std::io::{Read, Write}; use std::mem::size_of; -use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +use std::net::{IpAddr, SocketAddr, TcpStream, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::mpsc::{Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel}; +use std::sync::mpsc::{ + Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, +}; use std::sync::{Arc, Mutex, Weak}; use std::time::{Duration, Instant}; @@ -17,6 +19,8 @@ use tungstenite::{Error as WsError, Message, WebSocket, client}; /// between CDP layers cannot expand the maximum pending event count. pub const CDP_EVENT_QUEUE_CAPACITY: usize = 64; const CDP_INGRESS_EVENT_CAPACITY: usize = 1024; +const CDP_RESOLVER_QUEUE_CAPACITY: usize = 64; +const CDP_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); /// Maximum estimated retained bytes in each bounded CDP event queue. /// /// The estimate covers dynamically retained event payloads and uses saturating @@ -29,6 +33,42 @@ static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); +static CDP_RESOLVER: std::sync::OnceLock>> = std::sync::OnceLock::new(); + +struct CdpResolver { + requests: SyncSender, + _worker: std::thread::JoinHandle<()>, +} + +struct ResolveRequest { + host: String, + port: u16, + response: SyncSender>>, +} + +impl CdpResolver { + fn spawn() -> std::io::Result { + let (requests, receiver) = sync_channel::(CDP_RESOLVER_QUEUE_CAPACITY); + let worker = + std::thread::Builder::new().name("cmux-tui-cdp-resolver".into()).spawn(move || { + while let Ok(request) = receiver.recv() { + #[cfg(test)] + { + let delay_ms = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); + if delay_ms != 0 { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + let result = (request.host.as_str(), request.port) + .to_socket_addrs() + .map(Iterator::collect); + let _ = request.response.send(result); + } + })?; + Ok(Self { requests, _worker: worker }) + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ScreencastFrame { pub session_id: String, @@ -342,6 +382,69 @@ impl Write for DeadlineTcpStream { } } +fn deadline_after(timeout: Duration, expired: &str) -> anyhow::Result { + if timeout.is_zero() { + anyhow::bail!("{expired}"); + } + Instant::now().checked_add(timeout).ok_or_else(|| anyhow::anyhow!("CDP deadline overflow")) +} + +fn remaining_until(deadline: Instant, expired: &str) -> anyhow::Result { + deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| anyhow::anyhow!("{expired}")) +} + +fn submit_resolve_request(mut request: ResolveRequest) -> anyhow::Result<()> { + let resolver = CDP_RESOLVER.get_or_init(|| Mutex::new(None)); + let mut resolver = resolver.lock().unwrap(); + if resolver.is_none() { + *resolver = Some(CdpResolver::spawn()?); + } + match resolver.as_ref().unwrap().requests.try_send(request) { + Ok(()) => Ok(()), + Err(TrySendError::Full(_)) => anyhow::bail!("CDP resolver queue is full"), + Err(TrySendError::Disconnected(returned)) => { + request = returned; + *resolver = Some(CdpResolver::spawn()?); + resolver.as_ref().unwrap().requests.try_send(request).map_err(|error| match error { + TrySendError::Full(_) => anyhow::anyhow!("CDP resolver queue is full"), + TrySendError::Disconnected(_) => anyhow::anyhow!("CDP resolver worker stopped"), + }) + } + } +} + +fn resolve_socket_addr_until( + host: &str, + port: u16, + deadline: Instant, +) -> anyhow::Result { + if let Ok(ip) = host.parse::() { + remaining_until(deadline, "CDP address resolution deadline expired")?; + return Ok(SocketAddr::new(ip, port)); + } + + let (response, receiver) = sync_channel(1); + submit_resolve_request(ResolveRequest { host: host.to_string(), port, response })?; + let remaining = remaining_until(deadline, "CDP address resolution deadline expired")?; + let addresses = match receiver.recv_timeout(remaining) { + Ok(result) => result?, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + anyhow::bail!("CDP address resolution deadline expired") + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + anyhow::bail!("CDP resolver worker stopped") + } + }; + remaining_until(deadline, "CDP address resolution deadline expired")?; + addresses + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("no socket address for {host}:{port}")) +} + impl CdpClient { pub fn connect(web_socket_url: &str, events: SyncSender) -> anyhow::Result { Self::connect_with_timeout(web_socket_url, events, Duration::from_secs(5)) @@ -352,22 +455,10 @@ impl CdpClient { events: SyncSender, timeout: Duration, ) -> anyhow::Result { - if timeout.is_zero() { - anyhow::bail!("CDP connection deadline expired"); - } + let deadline = deadline_after(timeout, "CDP connection deadline expired")?; let endpoint = WsEndpoint::parse(web_socket_url)?; - #[cfg(test)] - { - let delay_ms = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); - if delay_ms != 0 { - std::thread::sleep(Duration::from_millis(delay_ms)); - } - } - let mut addrs = (endpoint.host.as_str(), endpoint.port).to_socket_addrs()?; - let addr = addrs.next().ok_or_else(|| { - anyhow::anyhow!("no socket address for {}:{}", endpoint.host, endpoint.port) - })?; - Self::connect_to_addr_with_timeout(web_socket_url, addr, events, timeout) + let addr = resolve_socket_addr_until(&endpoint.host, endpoint.port, deadline)?; + Self::connect_to_addr_until(web_socket_url, addr, events, deadline, timeout) } pub fn reconnect_with_timeout( @@ -389,15 +480,18 @@ impl CdpClient { events: SyncSender, timeout: Duration, ) -> anyhow::Result { - if timeout.is_zero() { - anyhow::bail!("CDP connection deadline expired"); - } - let deadline = Instant::now() - .checked_add(timeout) - .ok_or_else(|| anyhow::anyhow!("CDP connection deadline overflow"))?; - let remaining = deadline - .checked_duration_since(Instant::now()) - .ok_or_else(|| anyhow::anyhow!("CDP connection deadline expired"))?; + let deadline = deadline_after(timeout, "CDP connection deadline expired")?; + Self::connect_to_addr_until(web_socket_url, addr, events, deadline, timeout) + } + + fn connect_to_addr_until( + web_socket_url: &str, + addr: SocketAddr, + events: SyncSender, + deadline: Instant, + timeout: Duration, + ) -> anyhow::Result { + let remaining = remaining_until(deadline, "CDP connection deadline expired")?; let stream = TcpStream::connect_timeout(&addr, remaining)?; let stream = DeadlineTcpStream::new(stream, deadline); stream.set_nodelay(true)?; @@ -1071,19 +1165,20 @@ impl HttpEndpoint { } fn fetch_json_version(host: &str, port: u16) -> anyhow::Result { - let mut addrs = (host, port).to_socket_addrs()?; - let addr = - addrs.next().ok_or_else(|| anyhow::anyhow!("no socket address for {host}:{port}"))?; - let mut stream = TcpStream::connect_timeout(&addr, Duration::from_millis(250))?; + let deadline = deadline_after(CDP_DISCOVERY_TIMEOUT, "CDP discovery deadline exceeded")?; + let addr = resolve_socket_addr_until(host, port, deadline)?; + let remaining = remaining_until(deadline, "CDP discovery deadline exceeded")?; + let mut stream = TcpStream::connect_timeout(&addr, remaining.min(Duration::from_millis(250)))?; stream.set_nodelay(true)?; - stream.set_read_timeout(Some(Duration::from_millis(500)))?; - stream.set_write_timeout(Some(Duration::from_millis(500)))?; - write!( - stream, - "GET /json/version HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n" - )?; + let request = + format!("GET /json/version HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n"); + let remaining = remaining_until(deadline, "CDP discovery deadline exceeded")?; + stream.set_write_timeout(Some(remaining.min(Duration::from_millis(500))))?; + stream.write_all(request.as_bytes())?; + let remaining = remaining_until(deadline, "CDP discovery deadline exceeded")?; + stream.set_write_timeout(Some(remaining.min(Duration::from_millis(500))))?; stream.flush()?; - let response = read_http_response(&mut stream)?; + let response = read_http_response_until(&mut stream, 64 * 1024, deadline)?; let body = response .split_once("\r\n\r\n") .map(|(_, body)| body) @@ -1096,22 +1191,30 @@ fn fetch_json_version(host: &str, port: u16) -> anyhow::Result { .ok_or_else(|| anyhow::anyhow!("/json/version missing webSocketDebuggerUrl")) } +#[cfg(test)] fn read_http_response(stream: &mut TcpStream) -> anyhow::Result { read_http_response_with_limits(stream, 64 * 1024, Duration::from_secs(2)) } +#[cfg(test)] fn read_http_response_with_limits( stream: &mut TcpStream, max_bytes: usize, timeout: Duration, ) -> anyhow::Result { - let deadline = Instant::now() + timeout; + let deadline = deadline_after(timeout, "CDP discovery deadline exceeded")?; + read_http_response_until(stream, max_bytes, deadline) +} + +fn read_http_response_until( + stream: &mut TcpStream, + max_bytes: usize, + deadline: Instant, +) -> anyhow::Result { let mut bytes = Vec::new(); let mut buf = [0u8; 1024]; loop { - let remaining = deadline - .checked_duration_since(Instant::now()) - .ok_or_else(|| anyhow::anyhow!("CDP discovery deadline exceeded"))?; + let remaining = remaining_until(deadline, "CDP discovery deadline exceeded")?; stream.set_read_timeout(Some(remaining.min(Duration::from_millis(500))))?; match stream.read(&mut buf) { Ok(0) => break, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index af9835db6a55..18fb6c98f43d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6389,8 +6389,8 @@ impl Mux { let Some(workspace_index) = state.workspaces.iter().position(|workspace| workspace.id == workspace_id) else { - state.surfaces.remove(&surface.id); - surface.kill(); + drop(state); + self.discard_spawned(vec![surface]); anyhow::bail!("workspace disappeared while creating browser tab"); }; state.insert_pane(pane); @@ -6516,7 +6516,7 @@ impl Mux { } let active_at = self.next_active_at(); let notifications = self.surface_notifications(); - let attached = { + let (attached, retired) = { let mut state = self.state.lock().unwrap(); match state.panes.get_mut(&target) { Some(pane) => { @@ -6534,25 +6534,27 @@ impl Mux { surface.id, ) .expect("new browser tab is present in tree snapshot"); - Some(TreeDelta { - kind: TreeDeltaKind::TabAdded, - workspace, - screen: Some(screen), - pane: Some(target), - surface: Some(surface.id), - index: Some(index), - entity, - workspace_revision: None, - }) - } - None => { - state.surfaces.remove(&surface.id); - None + ( + Some(TreeDelta { + kind: TreeDeltaKind::TabAdded, + workspace, + screen: Some(screen), + pane: Some(target), + surface: Some(surface.id), + index: Some(index), + entity, + workspace_revision: None, + }), + None, + ) } + None => (None, take_surface_for_retirement(self, &mut state, surface.id)), } }; let Some(delta) = attached else { - surface.kill(); + if let Some(retired) = retired { + self.retire_surface_runtime(retired); + } anyhow::bail!("pane disappeared while creating browser tab"); }; self.emit_tree_delta(delta, true); @@ -6576,86 +6578,96 @@ impl Mux { let pending_surface = self.pending_workspace_surface(surface.id); let notifications = self.surface_notifications(); let active_at = self.next_active_at(); - let (delta, selection_resync) = { + let (attachment, retired) = { let mut state = self.state.lock().unwrap(); - let Some(wi) = state.workspace_index(workspace) else { - state.surfaces.remove(&surface.id); - surface.kill(); - anyhow::bail!("workspace disappeared while creating browser tab"); - }; - let target = state.workspaces[wi].active_screen_ref().map(|screen| screen.active_pane); - if let Some(target) = target { - let Some((_, si)) = state.screen_of(target) else { - state.surfaces.remove(&surface.id); - surface.kill(); - anyhow::bail!("workspace active pane disappeared while creating browser tab"); - }; - let Some(pane) = state.panes.get_mut(&target) else { - state.surfaces.remove(&surface.id); - surface.kill(); - anyhow::bail!("workspace active pane disappeared while creating browser tab"); + let attachment = 'attach: { + let Some(wi) = state.workspace_index(workspace) else { + break 'attach Err("workspace disappeared while creating browser tab"); }; - pane.tabs.push(surface.id); - pane.active_tab = pane.tabs.len() - 1; - pane.active_at = active_at; - let index = pane.tabs.len() - 1; - let screen = state.workspaces[wi].screens[si].id; - let entity = crate::server::tree_entity_json( - &state, - ¬ifications, - TreeDeltaKind::TabAdded, - surface.id, - ) - .expect("new browser tab is present in tree snapshot"); - ( - TreeDelta { - kind: TreeDeltaKind::TabAdded, - workspace, - screen: Some(screen), - pane: Some(target), - surface: Some(surface.id), - index: Some(index), - entity, - workspace_revision: None, - }, - true, - ) - } else { - let (pane_id, pane) = self.make_pane(surface.id); - let screen_id = self.next_id(); - state.insert_pane(pane); - stamp_pane_focus(self, &mut state, pane_id); - state.workspaces[wi].screens.push(Screen { - id: screen_id, - name: None, - root: Node::Leaf(pane_id), - active_pane: pane_id, - zoomed_pane: None, - zellij_auto_layout: Some(vec![pane_id]), - }); - state.workspaces[wi].active_screen = 0; - let entity = crate::server::tree_entity_json( - &state, - ¬ifications, - TreeDeltaKind::ScreenAdded, - screen_id, - ) - .expect("first browser screen is present in tree snapshot"); - ( - TreeDelta { - kind: TreeDeltaKind::ScreenAdded, - workspace, - screen: Some(screen_id), - pane: None, - surface: None, - index: Some(0), - entity, - workspace_revision: None, - }, - false, - ) - } + let target = + state.workspaces[wi].active_screen_ref().map(|screen| screen.active_pane); + if let Some(target) = target { + let Some((_, si)) = state.screen_of(target) else { + break 'attach Err( + "workspace active pane disappeared while creating browser tab", + ); + }; + let Some(pane) = state.panes.get_mut(&target) else { + break 'attach Err( + "workspace active pane disappeared while creating browser tab", + ); + }; + pane.tabs.push(surface.id); + pane.active_tab = pane.tabs.len() - 1; + pane.active_at = active_at; + let index = pane.tabs.len() - 1; + let screen = state.workspaces[wi].screens[si].id; + let entity = crate::server::tree_entity_json( + &state, + ¬ifications, + TreeDeltaKind::TabAdded, + surface.id, + ) + .expect("new browser tab is present in tree snapshot"); + Ok(( + TreeDelta { + kind: TreeDeltaKind::TabAdded, + workspace, + screen: Some(screen), + pane: Some(target), + surface: Some(surface.id), + index: Some(index), + entity, + workspace_revision: None, + }, + true, + )) + } else { + let (pane_id, pane) = self.make_pane(surface.id); + let screen_id = self.next_id(); + state.insert_pane(pane); + stamp_pane_focus(self, &mut state, pane_id); + state.workspaces[wi].screens.push(Screen { + id: screen_id, + name: None, + root: Node::Leaf(pane_id), + active_pane: pane_id, + zoomed_pane: None, + zellij_auto_layout: Some(vec![pane_id]), + }); + state.workspaces[wi].active_screen = 0; + let entity = crate::server::tree_entity_json( + &state, + ¬ifications, + TreeDeltaKind::ScreenAdded, + screen_id, + ) + .expect("first browser screen is present in tree snapshot"); + Ok(( + TreeDelta { + kind: TreeDeltaKind::ScreenAdded, + workspace, + screen: Some(screen_id), + pane: None, + surface: None, + index: Some(0), + entity, + workspace_revision: None, + }, + false, + )) + } + }; + let retired = attachment + .is_err() + .then(|| take_surface_for_retirement(self, &mut state, surface.id)) + .flatten(); + (attachment, retired) }; + if let Some(retired) = retired { + self.retire_surface_runtime(retired); + } + let (delta, selection_resync) = attachment.map_err(anyhow::Error::msg)?; drop(pending_surface); self.emit_tree_delta(delta, selection_resync); drop(workspace_lifecycle); @@ -8266,16 +8278,14 @@ impl Mux { { let mut state = self.state.lock().unwrap(); for id in &ids { - state.surfaces.remove(id); + let _ = take_surface_for_retirement(self, &mut state, *id); } } if batch.closed != 0 { self.emit_terminal_registry_changed(®istry, batch.revision); } drop(registry); - for surface in spawned { - surface.kill(); - } + self.retire_surface_runtimes(spawned); } #[allow(clippy::too_many_arguments)] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 0126ee664ab5..a12bc3e3b7fa 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -242,18 +242,7 @@ impl ServerProbe { // A server may close immediately after its complete identity // response. macOS then rejects timeout socket options with // EINVAL even though the response remains authoritative. - Err(error) - if error.raw_os_error() == Some(libc::EINVAL) - || matches!( - error.kind(), - std::io::ErrorKind::InvalidInput - | std::io::ErrorKind::NotConnected - | std::io::ErrorKind::BrokenPipe - | std::io::ErrorKind::ConnectionReset - ) => - { - Ok(probe) - } + Err(error) if closed_peer_timeout_error(&error) => Ok(probe), Err(_) => { Err(anyhow::anyhow!(crate::localization::catalog().server.transport_failed)) } @@ -463,31 +452,28 @@ impl ServerLifecycle { if surface.kind != "pty" { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } - if !surface.dead { - let process_request_id = take_legacy_request_id(next_request_id)?; - set_transport_deadline(self.reader.get_mut().as_ref(), deadline) - .map_err(|_| LegacyConnectionInterrupted)?; - write_json_line( - self.reader.get_mut(), - &json!({ - "id": process_request_id, - "cmd": "process-info", - "surface": surface.id, - }), - ) + let process_request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline) .map_err(|_| LegacyConnectionInterrupted)?; - let process_response = - read_response_until(&mut self.reader, process_request_id, deadline) - .map_err(|_| LegacyConnectionInterrupted)?; - let process_data = response_data(&process_response)?; - let pid = process_data - .get("pid") - .and_then(Value::as_u64) - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .filter(|pid| *pid > 1) - .ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; + write_json_line( + self.reader.get_mut(), + &json!({ + "id": process_request_id, + "cmd": "process-info", + "surface": surface.id, + }), + ) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_response = + read_response_until(&mut self.reader, process_request_id, deadline) + .map_err(|_| LegacyConnectionInterrupted)?; + let process_data = response_data(&process_response)?; + let pid = process_data + .get("pid") + .and_then(Value::as_u64) + .and_then(|pid| libc::pid_t::try_from(pid).ok()) + .filter(|pid| *pid > 1); + if let Some(pid) = pid { let owner = capture_process_session(pid, expected, captured, deadline) .map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) @@ -498,6 +484,8 @@ impl ServerLifecycle { if owner_ids.insert(owner.id()) { owners.push(owner); } + } else if !surface.dead { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } let request_id = take_legacy_request_id(next_request_id)?; @@ -801,14 +789,34 @@ pub(crate) fn write_json_line(writer: &mut dyn Write, value: &Value) -> std::io: } fn set_transport_deadline(stream: &dyn transport::Stream, deadline: Instant) -> anyhow::Result<()> { + set_transport_deadline_raw(stream, deadline) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed)) +} + +fn set_transport_deadline_raw( + stream: &dyn transport::Stream, + deadline: Instant, +) -> std::io::Result<()> { let remaining = deadline .checked_duration_since(Instant::now()) .filter(|remaining| !remaining.is_zero()) - .ok_or_else(|| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "server lifecycle deadline expired") + })?; stream .set_read_timeout(Some(remaining)) .and_then(|()| stream.set_write_timeout(Some(remaining))) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed)) +} + +fn closed_peer_timeout_error(error: &std::io::Error) -> bool { + error.raw_os_error() == Some(libc::EINVAL) + || matches!( + error.kind(), + std::io::ErrorKind::InvalidInput + | std::io::ErrorKind::NotConnected + | std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + ) } fn read_response_until( @@ -888,13 +896,17 @@ fn read_bounded_lifecycle_line_until( if Instant::now() >= deadline { return Err(LifecycleLineError::Deadline); } - set_transport_deadline(reader.get_mut().as_ref(), deadline).map_err(|_| { - if Instant::now() >= deadline { - LifecycleLineError::Deadline - } else { - LifecycleLineError::Transport - } - })?; + if let Err(error) = set_transport_deadline_raw(reader.get_mut().as_ref(), deadline) + && !closed_peer_timeout_error(&error) + { + return Err( + if Instant::now() >= deadline || error.kind() == std::io::ErrorKind::TimedOut { + LifecycleLineError::Deadline + } else { + LifecycleLineError::Transport + }, + ); + } let available = match reader.fill_buf() { Ok(available) => available, Err(error) From 18a356f0f2b13e8b635e3b27c5e5b96f73c8acc5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:27:37 -0700 Subject: [PATCH 093/521] test(tui): expose bounded shutdown review gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 27 +++++++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 45 +++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 23 ++++++++ .../src/terminal_host_runtime.rs | 55 +++++++++++++++++++ 4 files changed, 150 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index a29f68bd0163..8500c96a642c 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1564,6 +1564,33 @@ mod tests { ); } + #[test] + fn expired_resolution_does_not_block_the_next_request() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); + + let first = + resolve_socket_addr_until("localhost", 1, Instant::now() + Duration::from_millis(50)); + let started = Instant::now(); + let second = + resolve_socket_addr_until("localhost", 2, Instant::now() + Duration::from_millis(100)); + let elapsed = started.elapsed(); + + // Unblock the old single-worker implementation before failing so the + // process-wide test resolver cannot contaminate a later test. + if second.is_err() { + thread::sleep(Duration::from_millis(250)); + } + NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); + + assert!(first.is_err(), "injected slow resolution unexpectedly completed"); + assert!(second.is_ok(), "expired DNS work blocked the next request: {second:?}"); + assert!( + elapsed < Duration::from_millis(200), + "the next DNS request waited behind expired work: {elapsed:?}" + ); + } + #[test] fn concurrent_calls_complete_while_reader_receives_events() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 18fb6c98f43d..28a96728127a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12632,6 +12632,51 @@ mod tests { assert!(topology_retained); } + #[cfg(unix)] + #[test] + fn server_shutdown_rejects_host_records_beyond_owner_capacity() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let root = std::env::temp_dir().join(format!( + "cmux-shutdown-host-capacity-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let terminal_id = TerminalId::random().unwrap().to_hex(); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: terminal_id.clone(), + incarnation: crate::terminal_host::HostIncarnation::random().unwrap().to_hex(), + endpoint: format!( + "/tmp/cmux-th-{}/{}.sock", + std::fs::metadata(&root).unwrap().uid(), + terminal_id + ), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: String::new(), + supports_set_defaults: false, + supports_terminate_only: false, + }; + let record_path = record.record_path(&root); + std::fs::write(&record_path, serde_json::to_vec(&record).unwrap()).unwrap(); + std::fs::set_permissions(&record_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + let mux = Mux::new_for_test( + "strict-shutdown-capacity", + SurfaceOptions { terminal_host_root: Some(root.clone()), ..SurfaceOptions::default() }, + ); + mux.set_shutdown_owner_capacity_for_test(0); + + let error = mux.close_all_surfaces_for_shutdown().unwrap_err(); + let _ = std::fs::remove_dir_all(&root); + + assert!( + format!("{error:#}").contains("capacity"), + "shutdown staged a host beyond owner capacity: {error:#}" + ); + } + #[cfg(unix)] #[test] fn server_shutdown_preflights_process_control_before_removing_topology() { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 00e5c9ff8f94..8226b692f096 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -3284,6 +3284,29 @@ mod tests { assert_eq!(signals.load(Ordering::Relaxed), 0); } + #[cfg(unix)] + #[test] + fn completed_natural_cleanup_does_not_revalidate_reusable_session_id() { + // A completed child may have its numeric session ID reassigned. Use + // this test process's live session as a deterministic reused value. + // SAFETY: getsid(0) only queries the calling process. + let reused_session = unsafe { libc::getsid(0) }; + assert!(reused_session > 1); + let signals = Arc::new(AtomicUsize::new(0)); + let process = LocalPtyProcess::new( + u32::try_from(reused_session).ok(), + Box::new(CountingChildKiller(signals.clone())), + ); + process.group_escalation_complete.store(true, Ordering::Release); + process.child_reaped.store(true, Ordering::Release); + + assert!( + process.terminate_and_wait(Instant::now() + Duration::from_millis(100)), + "exact completed cleanup was rejected through a reused numeric session ID" + ); + assert_eq!(signals.load(Ordering::Relaxed), 0); + } + #[cfg(unix)] #[test] fn normal_child_exit_cleans_same_session_descendants_before_reaping() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index b551bd475c10..e8b321c29b8f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3921,6 +3921,61 @@ mod unix { let _ = fs::remove_dir_all(root); } + #[test] + fn strict_record_loader_rejects_fifo_without_blocking() { + let root = std::env::temp_dir().join(format!( + "cmux-host-record-fifo-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + prepare_private_dir(&root).unwrap(); + let path = root.join(format!("{}.json", TerminalId::random().unwrap().to_hex())); + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: c_path is a valid, NUL-terminated path owned by this test. + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + + let loader_root = root.clone(); + let (result_tx, result_rx) = sync_channel(1); + let loader = thread::spawn(move || { + let _ = result_tx.send(load_terminal_host_records_strict(&loader_root)); + }); + let completed = match result_rx.recv_timeout(Duration::from_millis(500)) { + Ok(result) => Some(result), + Err(RecvTimeoutError::Timeout) => { + let mut writer = OpenOptions::new().write(true).open(&path).unwrap(); + writer.write_all(b"{}").unwrap(); + None + } + Err(RecvTimeoutError::Disconnected) => panic!("record loader stopped"), + }; + loader.join().unwrap(); + let _ = fs::remove_dir_all(&root); + + assert!(completed.is_some(), "strict terminal-host scan blocked while opening a FIFO"); + assert!(completed.unwrap().is_err(), "FIFO was accepted as a discovery record"); + } + + #[test] + fn strict_record_loader_rejects_oversized_file_before_decoding() { + let root = std::env::temp_dir().join(format!( + "cmux-host-record-oversized-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + prepare_private_dir(&root).unwrap(); + let path = root.join(format!("{}.json", TerminalId::random().unwrap().to_hex())); + fs::write(&path, vec![b'x'; MAX_LAUNCH_PAYLOAD + 1]).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + + let error = load_terminal_host_records_strict(&root).unwrap_err(); + let _ = fs::remove_dir_all(&root); + + assert!( + format!("{error:#}").contains("exceeds size limit"), + "oversized record reached JSON decoding: {error:#}" + ); + } + #[test] fn legacy_record_is_adoptable_shape_but_never_unsafely_reaped() { let (v2_path, v2, lease) = record_fixture("legacy"); From 014c3134a05cac99ddcae6eba60a910ce4a3e2a3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:43:13 -0700 Subject: [PATCH 094/521] fix(tui): bound shutdown discovery and resolution --- cmux-tui/Cargo.lock | 898 +++++++++++++++++- cmux-tui/Cargo.toml | 2 + cmux-tui/crates/cmux-tui-cdp/Cargo.toml | 2 + cmux-tui/crates/cmux-tui-cdp/src/client.rs | 134 ++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 55 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 3 + .../src/terminal_host_runtime.rs | 100 +- 7 files changed, 1122 insertions(+), 72 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index c4f22b202521..dfba41dea6b2 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -32,6 +32,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "atomic" version = "0.6.1" @@ -179,6 +190,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -229,7 +251,9 @@ version = "0.1.0" dependencies = [ "anyhow", "base64", + "hickory-resolver", "serde_json", + "tokio", "tungstenite", ] @@ -252,7 +276,7 @@ dependencies = [ "sha2", "tungstenite", "uds_windows", - "windows-sys", + "windows-sys 0.61.2", "zeroize", ] @@ -277,6 +301,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "compact_str" version = "0.9.1" @@ -300,6 +334,22 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -309,12 +359,45 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "critical-section" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crossterm" version = "0.29.0" @@ -446,6 +529,17 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "document-features" version = "0.2.12" @@ -480,7 +574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -567,6 +661,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs4" version = "1.1.0" @@ -574,7 +677,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e72ed92b67c146290f88e9c89d60ca163ea417a446f61ffd7b72df3e7f1dfd5" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", ] [[package]] @@ -583,6 +695,23 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "futures-task" version = "0.3.32" @@ -596,6 +725,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -611,6 +741,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -632,6 +773,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -688,6 +830,76 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "http" version = "1.4.2" @@ -704,12 +916,115 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -742,6 +1057,28 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + [[package]] name = "itertools" version = "0.13.0" @@ -766,6 +1103,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "js-sys" version = "0.3.103" @@ -848,6 +1234,12 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "litrs" version = "1.0.0" @@ -924,9 +1316,32 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nix" version = "0.28.0" @@ -1002,6 +1417,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "ordered-float" @@ -1059,6 +1478,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pest" version = "2.8.7" @@ -1192,6 +1617,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1207,6 +1641,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -1266,6 +1711,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1291,6 +1747,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "ratatui" version = "0.30.2" @@ -1430,6 +1892,26 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rusqlite" version = "0.40.1" @@ -1468,7 +1950,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1483,6 +1965,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1555,7 +2046,7 @@ checksum = "9eb6ea5562eeaed6936b8b54e086aa0f88b9e5b1bef45beb038e2519fa1185b1" dependencies = [ "cfg-if", "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1565,7 +2056,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1576,7 +2067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1639,6 +2130,22 @@ dependencies = [ "libc", ] +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.3" @@ -1657,6 +2164,22 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1718,6 +2241,55 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.27.0" @@ -1728,7 +2300,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1741,7 +2313,7 @@ dependencies = [ "parking_lot", "rustix", "signal-hook", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1868,6 +2440,57 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "toml" version = "0.8.23" @@ -1909,6 +2532,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + [[package]] name = "tungstenite" version = "0.29.0" @@ -1945,7 +2587,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1977,6 +2619,30 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2016,6 +2682,16 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2148,6 +2824,12 @@ dependencies = [ "wezterm-dynamic", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -2164,6 +2846,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2176,6 +2867,44 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -2185,6 +2914,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "0.7.15" @@ -2209,6 +3002,35 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.8.54" @@ -2229,12 +3051,66 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + [[package]] name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zmij" version = "1.0.23" diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index 7095bd92ca0e..5763d693d41e 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -58,6 +58,8 @@ sha2 = "0.10" zeroize = "1.9" subtle = "2.6.1" parking_lot = "0.12" +hickory-resolver = { version = "0.26.1", default-features = false, features = ["system-config", "tokio"] } +tokio = { version = "1.47.1", features = ["net", "rt-multi-thread", "sync", "time"] } [profile.release] lto = "thin" diff --git a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml index 3a5ecbecdc3a..d5296679d2ae 100644 --- a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml @@ -13,5 +13,7 @@ workspace = true [dependencies] anyhow.workspace = true base64.workspace = true +hickory-resolver.workspace = true serde_json.workspace = true +tokio.workspace = true tungstenite.workspace = true diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 8500c96a642c..10a6c7a5650f 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::io::{Read, Write}; use std::mem::size_of; -use std::net::{IpAddr, SocketAddr, TcpStream, ToSocketAddrs}; +use std::net::{IpAddr, SocketAddr, TcpStream}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{ Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, @@ -9,7 +9,10 @@ use std::sync::mpsc::{ use std::sync::{Arc, Mutex, Weak}; use std::time::{Duration, Instant}; +use hickory_resolver::TokioResolver; use serde_json::{Value, json}; +use tokio::runtime::{Builder as RuntimeBuilder, Runtime}; +use tokio::sync::Semaphore; use tungstenite::client::IntoClientRequest; use tungstenite::{Error as WsError, Message, WebSocket, client}; @@ -33,39 +36,88 @@ static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); -static CDP_RESOLVER: std::sync::OnceLock>> = std::sync::OnceLock::new(); +static CDP_RESOLVER: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); struct CdpResolver { - requests: SyncSender, - _worker: std::thread::JoinHandle<()>, + resolver: TokioResolver, + permits: Arc, + runtime: Runtime, } struct ResolveRequest { host: String, port: u16, + deadline: Instant, response: SyncSender>>, + #[cfg(test)] + delay: Duration, } impl CdpResolver { fn spawn() -> std::io::Result { - let (requests, receiver) = sync_channel::(CDP_RESOLVER_QUEUE_CAPACITY); - let worker = - std::thread::Builder::new().name("cmux-tui-cdp-resolver".into()).spawn(move || { - while let Ok(request) = receiver.recv() { + let runtime = RuntimeBuilder::new_multi_thread() + .worker_threads(1) + .thread_name("cmux-tui-cdp-resolver") + .enable_all() + .build()?; + let resolver = { + let _runtime = runtime.enter(); + hickory_resolver::Resolver::builder_tokio() + .map_err(|error| std::io::Error::other(error.to_string()))? + .build() + .map_err(|error| std::io::Error::other(error.to_string()))? + }; + Ok(Self { + resolver, + permits: Arc::new(Semaphore::new(CDP_RESOLVER_QUEUE_CAPACITY)), + runtime, + }) + } + + fn submit(&self, request: ResolveRequest) -> anyhow::Result<()> { + let permit = self + .permits + .clone() + .try_acquire_owned() + .map_err(|_| anyhow::anyhow!("CDP resolver queue is full"))?; + let resolver = self.resolver.clone(); + let _task = self.runtime.spawn(async move { + let deadline = tokio::time::Instant::from_std(request.deadline); + let result = if deadline <= tokio::time::Instant::now() { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )) + } else { + let lookup = async { #[cfg(test)] - { - let delay_ms = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); - if delay_ms != 0 { - std::thread::sleep(Duration::from_millis(delay_ms)); - } + if !request.delay.is_zero() { + tokio::time::sleep(request.delay).await; } - let result = (request.host.as_str(), request.port) - .to_socket_addrs() - .map(Iterator::collect); - let _ = request.response.send(result); + resolver + .lookup_ip(request.host) + .await + .map(|addresses| { + addresses + .iter() + .map(|address| SocketAddr::new(address, request.port)) + .collect() + }) + .map_err(|error| std::io::Error::other(error.to_string())) + }; + match tokio::time::timeout_at(deadline, lookup).await { + Ok(result) => result, + Err(_) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )), } - })?; - Ok(Self { requests, _worker: worker }) + }; + let _ = request.response.send(result); + drop(permit); + }); + Ok(()) } } @@ -396,24 +448,16 @@ fn remaining_until(deadline: Instant, expired: &str) -> anyhow::Result .ok_or_else(|| anyhow::anyhow!("{expired}")) } -fn submit_resolve_request(mut request: ResolveRequest) -> anyhow::Result<()> { +fn submit_resolve_request(request: ResolveRequest) -> anyhow::Result<()> { let resolver = CDP_RESOLVER.get_or_init(|| Mutex::new(None)); - let mut resolver = resolver.lock().unwrap(); - if resolver.is_none() { - *resolver = Some(CdpResolver::spawn()?); - } - match resolver.as_ref().unwrap().requests.try_send(request) { - Ok(()) => Ok(()), - Err(TrySendError::Full(_)) => anyhow::bail!("CDP resolver queue is full"), - Err(TrySendError::Disconnected(returned)) => { - request = returned; - *resolver = Some(CdpResolver::spawn()?); - resolver.as_ref().unwrap().requests.try_send(request).map_err(|error| match error { - TrySendError::Full(_) => anyhow::anyhow!("CDP resolver queue is full"), - TrySendError::Disconnected(_) => anyhow::anyhow!("CDP resolver worker stopped"), - }) + let resolver = { + let mut state = resolver.lock().unwrap(); + if state.is_none() { + *state = Some(Arc::new(CdpResolver::spawn()?)); } - } + state.as_ref().unwrap().clone() + }; + resolver.submit(request) } fn resolve_socket_addr_until( @@ -427,7 +471,14 @@ fn resolve_socket_addr_until( } let (response, receiver) = sync_channel(1); - submit_resolve_request(ResolveRequest { host: host.to_string(), port, response })?; + submit_resolve_request(ResolveRequest { + host: host.to_string(), + port, + deadline, + response, + #[cfg(test)] + delay: Duration::from_millis(NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel)), + })?; let remaining = remaining_until(deadline, "CDP address resolution deadline expired")?; let addresses = match receiver.recv_timeout(remaining) { Ok(result) => result?, @@ -1304,6 +1355,11 @@ mod tests { static RESOLVE_TEST_LOCK: Mutex<()> = Mutex::new(()); + fn warm_resolver() { + resolve_socket_addr_until("localhost", 0, Instant::now() + Duration::from_secs(5)) + .expect("initialize test resolver"); + } + #[test] fn screencast_frame_rejects_terminal_control_bytes() { let params = json!({ @@ -1541,6 +1597,7 @@ mod tests { #[test] fn websocket_resolution_uses_the_connection_deadline() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + warm_resolver(); NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); let (events, _receiver) = sync_channel(1); @@ -1551,10 +1608,6 @@ mod tests { Duration::from_millis(50), ); let elapsed = started.elapsed(); - // A bounded resolver may still be finishing this injected operation - // after the caller's deadline. Let it drain before another test can - // use the process-wide resolver. - thread::sleep(Duration::from_millis(350)); NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); assert!(result.is_err(), "invalid local endpoint unexpectedly connected"); @@ -1567,6 +1620,7 @@ mod tests { #[test] fn expired_resolution_does_not_block_the_next_request() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + warm_resolver(); NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); let first = diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 28a96728127a..8ca3f024d776 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5064,20 +5064,37 @@ impl Mux { #[cfg(unix)] let terminal_host_records = { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - let required = self - .state - .lock() - .unwrap() - .surfaces - .values() - .filter(|surface| !surface.is_dead()) - .filter_map(|surface| surface.terminal_host_identity()) - .collect::>(); - match root { + let capacity = self.shutdown_owner_capacity(); + let (required, surface_owner_keys) = { + let state = self.state.lock().unwrap(); + let required = state + .surfaces + .values() + .filter(|surface| !surface.is_dead()) + .filter_map(|surface| surface.terminal_host_identity()) + .collect::>(); + let surface_owner_keys = state + .surfaces + .values() + .filter_map(|surface| { + surface.shutdown_owner().map(|owner| shutdown_owner_key(surface.id, &owner)) + }) + .collect::>(); + (required, surface_owner_keys) + }; + let mut owner_keys = self + .shutdown_owners + .snapshot() + .into_iter() + .map(|(key, _)| key) + .collect::>(); + owner_keys.extend(surface_owner_keys); + let records = match root { Some(root) => { - let records = - crate::terminal_host_runtime::load_terminal_host_records_strict(&root) - .context("load terminal hosts for server shutdown")?; + let records = crate::terminal_host_runtime::load_terminal_host_records_strict( + &root, capacity, deadline, + ) + .context("load terminal hosts for server shutdown")?; let available = records .iter() .map(|(_, record)| { @@ -5102,7 +5119,19 @@ impl Mux { None => anyhow::bail!( "load terminal hosts for server shutdown: terminal-host root is unavailable" ), + }; + owner_keys.extend(records.iter().map(|(_, record)| ShutdownOwnerKey::Hosted { + terminal_id: record.terminal_id.clone(), + incarnation: record.incarnation.clone(), + })); + if owner_keys.len() > capacity { + anyhow::bail!( + "load terminal hosts for server shutdown: shutdown owner capacity \ + {capacity} cannot retain {} unique owners", + owner_keys.len() + ); } + records }; let (surfaces, retained_count, tree_changed) = { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 8226b692f096..7049db2ea93a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1076,6 +1076,9 @@ impl LocalPtyProcess { self.exited.1.notify_all(); crate::process_session::wake_child_reaper(); if child_reaped { + if self.group_escalation_complete.load(Ordering::Acquire) { + return true; + } let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { self.abandon_termination(); return false; diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index e8b321c29b8f..4cdc26f523bd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -307,6 +307,7 @@ mod unix { const HOST_PROCESS_REAPER_CAPACITY: usize = 4_096; const HOST_PROCESS_REAPER_POLL: Duration = Duration::from_millis(25); const HOST_PROCESS_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); + const MAX_TERMINAL_HOST_RECORD_BYTES: usize = MAX_LAUNCH_PAYLOAD; static HOST_PROCESS_REAPER: OnceLock>> = OnceLock::new(); #[cfg(test)] static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); @@ -1677,7 +1678,10 @@ mod unix { /// for every record before it discards topology. pub fn load_terminal_host_records_strict( root: &Path, + max_records: usize, + deadline: Instant, ) -> anyhow::Result> { + ensure_terminal_host_scan_before(deadline)?; let mut records = Vec::new(); let mut identities = HashSet::new(); let entries = match fs::read_dir(root) { @@ -1686,17 +1690,18 @@ mod unix { Err(error) => return Err(error.into()), }; for entry in entries { + ensure_terminal_host_scan_before(deadline)?; let entry = entry?; let path = entry.path(); if path.extension().and_then(|value| value.to_str()) != Some("json") { continue; } - let bytes = fs::read(&path) - .with_context(|| format!("read terminal-host record {}", path.display()))?; - let record = serde_json::from_slice::(&bytes) - .with_context(|| format!("decode terminal-host record {}", path.display()))?; - validate_terminal_host_record(&path, &record) - .with_context(|| format!("validate terminal-host record {}", path.display()))?; + if records.len() >= max_records { + anyhow::bail!( + "terminal-host record count exceeds shutdown owner capacity {max_records}" + ); + } + let record = read_terminal_host_record_until(&path, deadline)?; if !identities.insert((record.terminal_id.clone(), record.incarnation.clone())) { anyhow::bail!( "duplicate terminal-host identity {}:{}", @@ -1706,10 +1711,80 @@ mod unix { } records.push((path, record)); } + ensure_terminal_host_scan_before(deadline)?; records.sort_by(|left, right| left.0.cmp(&right.0)); Ok(records) } + fn ensure_terminal_host_scan_before(deadline: Instant) -> anyhow::Result<()> { + if Instant::now() >= deadline { + anyhow::bail!("terminal-host record scan exceeded the shutdown deadline"); + } + Ok(()) + } + + fn read_terminal_host_record_until( + path: &Path, + deadline: Instant, + ) -> anyhow::Result { + ensure_terminal_host_scan_before(deadline)?; + let mut file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .with_context(|| format!("open terminal-host record {}", path.display()))?; + let metadata = file + .metadata() + .with_context(|| format!("inspect terminal-host record {}", path.display()))?; + let expected_uid = fs::metadata( + path.parent() + .ok_or_else(|| anyhow::anyhow!("terminal-host record has no parent directory"))?, + )? + .uid(); + if !metadata.file_type().is_file() + || metadata.uid() != expected_uid + || metadata.nlink() != 1 + || metadata.mode() & 0o077 != 0 + { + anyhow::bail!("terminal-host record permissions or ownership are unsafe"); + } + if metadata.len() > MAX_TERMINAL_HOST_RECORD_BYTES as u64 { + anyhow::bail!("terminal-host record {} exceeds size limit", path.display()); + } + + let mut bytes = Vec::with_capacity(metadata.len() as usize); + let mut buffer = [0u8; 8 * 1024]; + loop { + ensure_terminal_host_scan_before(deadline)?; + let remaining = MAX_TERMINAL_HOST_RECORD_BYTES + 1 - bytes.len(); + let read_len = buffer.len().min(remaining); + let read = file + .read(&mut buffer[..read_len]) + .with_context(|| format!("read terminal-host record {}", path.display()))?; + if read == 0 { + break; + } + bytes.extend_from_slice(&buffer[..read]); + if bytes.len() > MAX_TERMINAL_HOST_RECORD_BYTES { + anyhow::bail!("terminal-host record {} exceeds size limit", path.display()); + } + } + ensure_terminal_host_scan_before(deadline)?; + let current = fs::symlink_metadata(path) + .with_context(|| format!("reinspect terminal-host record {}", path.display()))?; + if !current.file_type().is_file() + || current.dev() != metadata.dev() + || current.ino() != metadata.ino() + { + anyhow::bail!("terminal-host record changed while being read"); + } + let record = serde_json::from_slice::(&bytes) + .with_context(|| format!("decode terminal-host record {}", path.display()))?; + validate_terminal_host_record(path, &record) + .with_context(|| format!("validate terminal-host record {}", path.display()))?; + Ok(record) + } + /// Terminate one exact host incarnation and retain its discovery record /// until process-bound liveness proves that host is dead. pub(crate) fn terminate_and_confirm_terminal_host_record( @@ -3937,7 +4012,11 @@ mod unix { let loader_root = root.clone(); let (result_tx, result_rx) = sync_channel(1); let loader = thread::spawn(move || { - let _ = result_tx.send(load_terminal_host_records_strict(&loader_root)); + let _ = result_tx.send(load_terminal_host_records_strict( + &loader_root, + 1, + Instant::now() + Duration::from_secs(1), + )); }); let completed = match result_rx.recv_timeout(Duration::from_millis(500)) { Ok(result) => Some(result), @@ -3967,7 +4046,12 @@ mod unix { fs::write(&path, vec![b'x'; MAX_LAUNCH_PAYLOAD + 1]).unwrap(); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); - let error = load_terminal_host_records_strict(&root).unwrap_err(); + let error = load_terminal_host_records_strict( + &root, + 1, + Instant::now() + Duration::from_secs(1), + ) + .unwrap_err(); let _ = fs::remove_dir_all(&root); assert!( From eabac8969f468768706f9fb6a8a9b68683a6b0af Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:43:57 -0700 Subject: [PATCH 095/521] test(tui): expose resolver initialization deadline --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 10a6c7a5650f..92a8d824d695 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -35,6 +35,8 @@ pub const CDP_EVENT_QUEUE_MAX_BYTES: usize = 32 * 1024 * 1024; static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); +#[cfg(test)] +static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -56,6 +58,13 @@ struct ResolveRequest { impl CdpResolver { fn spawn() -> std::io::Result { + #[cfg(test)] + { + let delay_ms = NEXT_RESOLVER_INIT_DELAY_MS.swap(0, Ordering::AcqRel); + if delay_ms != 0 { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } let runtime = RuntimeBuilder::new_multi_thread() .worker_threads(1) .thread_name("cmux-tui-cdp-resolver") @@ -1617,6 +1626,29 @@ mod tests { ); } + #[test] + fn resolver_initialization_obeys_the_first_request_deadline() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + *CDP_RESOLVER.get_or_init(|| Mutex::new(None)).lock().unwrap() = None; + NEXT_RESOLVER_INIT_DELAY_MS.store(300, Ordering::Release); + + let started = Instant::now(); + let result = + resolve_socket_addr_until("localhost", 1, Instant::now() + Duration::from_millis(50)); + let elapsed = started.elapsed(); + // A bounded initializer may still be completing after the caller's + // deadline. Let the one process-wide initialization finish before a + // later resolver test starts. + thread::sleep(Duration::from_millis(350)); + NEXT_RESOLVER_INIT_DELAY_MS.store(0, Ordering::Release); + + assert!(result.is_err(), "expired first resolution unexpectedly completed"); + assert!( + elapsed < Duration::from_millis(200), + "resolver initialization escaped the first request deadline: {elapsed:?}" + ); + } + #[test] fn expired_resolution_does_not_block_the_next_request() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); From fadd94551052fadedcfb27e20c8b5d067dac04c3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 19:46:48 -0700 Subject: [PATCH 096/521] fix(tui): bound resolver initialization --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 93 +++++++++++++++++----- 1 file changed, 73 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 92a8d824d695..6195d6bc5d45 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -6,7 +6,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{ Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, }; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Condvar, Mutex, Weak}; use std::time::{Duration, Instant}; use hickory_resolver::TokioResolver; @@ -42,9 +42,16 @@ static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); struct CdpResolver { - resolver: TokioResolver, + resolver: Arc<(Mutex, Condvar)>, permits: Arc, runtime: Runtime, + _initializer: std::thread::JoinHandle<()>, +} + +enum ResolverState { + Initializing, + Ready(Arc), + Failed(String), } struct ResolveRequest { @@ -58,29 +65,43 @@ struct ResolveRequest { impl CdpResolver { fn spawn() -> std::io::Result { - #[cfg(test)] - { - let delay_ms = NEXT_RESOLVER_INIT_DELAY_MS.swap(0, Ordering::AcqRel); - if delay_ms != 0 { - std::thread::sleep(Duration::from_millis(delay_ms)); - } - } let runtime = RuntimeBuilder::new_multi_thread() .worker_threads(1) .thread_name("cmux-tui-cdp-resolver") .enable_all() .build()?; - let resolver = { - let _runtime = runtime.enter(); - hickory_resolver::Resolver::builder_tokio() - .map_err(|error| std::io::Error::other(error.to_string()))? - .build() - .map_err(|error| std::io::Error::other(error.to_string()))? - }; + let resolver = Arc::new((Mutex::new(ResolverState::Initializing), Condvar::new())); + let initializer_state = resolver.clone(); + let runtime_handle = runtime.handle().clone(); + let initializer = std::thread::Builder::new() + .name("cmux-tui-cdp-resolver-init".into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + #[cfg(test)] + { + let delay_ms = NEXT_RESOLVER_INIT_DELAY_MS.swap(0, Ordering::AcqRel); + if delay_ms != 0 { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + let _runtime = runtime_handle.enter(); + hickory_resolver::Resolver::builder_tokio() + .map_err(|error| error.to_string()) + .and_then(|builder| builder.build().map_err(|error| error.to_string())) + })) + .unwrap_or_else(|_| Err("resolver initializer panicked".to_string())); + let (state, ready) = &*initializer_state; + *state.lock().unwrap() = match result { + Ok(resolver) => ResolverState::Ready(Arc::new(resolver)), + Err(error) => ResolverState::Failed(error), + }; + ready.notify_all(); + })?; Ok(Self { resolver, permits: Arc::new(Semaphore::new(CDP_RESOLVER_QUEUE_CAPACITY)), runtime, + _initializer: initializer, }) } @@ -90,7 +111,7 @@ impl CdpResolver { .clone() .try_acquire_owned() .map_err(|_| anyhow::anyhow!("CDP resolver queue is full"))?; - let resolver = self.resolver.clone(); + let resolver = self.resolver_until(request.deadline)?; let _task = self.runtime.spawn(async move { let deadline = tokio::time::Instant::from_std(request.deadline); let result = if deadline <= tokio::time::Instant::now() { @@ -128,6 +149,31 @@ impl CdpResolver { }); Ok(()) } + + fn resolver_until(&self, deadline: Instant) -> anyhow::Result> { + let (state, ready) = &*self.resolver; + let mut state = state.lock().unwrap(); + loop { + match &*state { + ResolverState::Ready(resolver) => return Ok(resolver.clone()), + ResolverState::Failed(error) => { + anyhow::bail!("CDP resolver initialization failed: {error}") + } + ResolverState::Initializing => {} + } + let remaining = remaining_until(deadline, "CDP address resolution deadline expired")?; + let (next, timeout) = ready.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && matches!(*state, ResolverState::Initializing) { + anyhow::bail!("CDP address resolution deadline expired"); + } + } + } + + fn initialization_failed(&self) -> bool { + let state = self.resolver.0.lock().unwrap(); + matches!(&*state, ResolverState::Failed(_)) + } } #[derive(Debug, Clone, PartialEq, Eq)] @@ -458,15 +504,22 @@ fn remaining_until(deadline: Instant, expired: &str) -> anyhow::Result } fn submit_resolve_request(request: ResolveRequest) -> anyhow::Result<()> { - let resolver = CDP_RESOLVER.get_or_init(|| Mutex::new(None)); + let registry = CDP_RESOLVER.get_or_init(|| Mutex::new(None)); let resolver = { - let mut state = resolver.lock().unwrap(); + let mut state = registry.lock().unwrap(); if state.is_none() { *state = Some(Arc::new(CdpResolver::spawn()?)); } state.as_ref().unwrap().clone() }; - resolver.submit(request) + let result = resolver.submit(request); + if result.is_err() && resolver.initialization_failed() { + let mut state = registry.lock().unwrap(); + if state.as_ref().is_some_and(|current| Arc::ptr_eq(current, &resolver)) { + *state = None; + } + } + result } fn resolve_socket_addr_until( From 0cc8295263c7f0b4e2b2917e6d42990d4b4e91de Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:42:37 -0700 Subject: [PATCH 097/521] test(tui): expose final shutdown ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 69 +++++++++++++++++++ .../src/terminal_host_runtime.rs | 58 ++++++++++++++++ 2 files changed, 127 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8ca3f024d776..cdbad3694692 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12706,6 +12706,75 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn rejected_shutdown_preflight_preserves_browser_ownership() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + }); + let runtime = BrowserRuntime::connect_external_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + server.join().unwrap(); + let disconnect_deadline = Instant::now() + Duration::from_secs(1); + while !runtime.is_closed() && Instant::now() < disconnect_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(runtime.is_closed(), "test browser runtime did not observe disconnect"); + let mux = test_mux(); + let options = mux.surface_options.lock().unwrap().clone(); + let surface = browser::new_surface( + 999, + "about:blank".to_string(), + (80, 24), + (8, 16), + &options, + Arc::downgrade(&mux), + ); + surface.as_browser().unwrap().install_shutdown_session_for_test( + runtime.clone(), + "target-preflight", + "session-preflight", + ); + insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface.clone()).unwrap(); + mux.set_shutdown_owner_capacity_for_test(0); + + let error = mux.close_all_surfaces_for_shutdown().unwrap_err(); + let topology_retained = mux.surface(surface.id).is_some(); + let surface_was_live = !surface.is_dead(); + let owner_retained = surface.shutdown_owner().is_some(); + runtime.shutdown(); + + assert!(format!("{error:#}").contains("capacity")); + assert!(topology_retained, "shutdown preflight removed the browser surface"); + assert!(surface_was_live, "shutdown preflight killed the retained browser surface"); + assert!(owner_retained, "shutdown preflight consumed the browser target owner"); + } + #[cfg(unix)] #[test] fn server_shutdown_preflights_process_control_before_removing_topology() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 4cdc26f523bd..21bd569881d6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3867,6 +3867,64 @@ mod unix { mod tests { use super::*; + #[cfg(target_os = "linux")] + #[test] + fn saturated_listener_never_returns_an_unconnected_host_stream() { + let path = std::env::temp_dir().join(format!( + "cmux-host-backlog-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let (address, address_len) = unix_socket_address(&path).unwrap(); + // SAFETY: socket has no pointer arguments and returns a new owned + // descriptor on success. + let listener = + unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + assert!(listener >= 0, "failed to create the test listener"); + // SAFETY: listener is a fresh successful socket result and this + // OwnedFd takes its sole ownership. + let listener = unsafe { OwnedFd::from_raw_fd(listener) }; + // SAFETY: address is initialized for this exact filesystem path + // and listener owns a valid AF_UNIX descriptor. + let result = unsafe { + libc::bind( + listener.as_raw_fd(), + (&raw const address).cast::(), + address_len, + ) + }; + assert_eq!(result, 0, "failed to bind the test listener"); + // SAFETY: listener remains a valid bound AF_UNIX descriptor. + let result = unsafe { libc::listen(listener.as_raw_fd(), 0) }; + assert_eq!(result, 0, "failed to create a zero-backlog test listener"); + + let mut queued = Vec::new(); + let mut timeout_elapsed = None; + for _ in 0..32 { + let started = Instant::now(); + match connect_until(&path, started + Duration::from_millis(75)) { + Ok(stream) => queued.push(stream), + Err(error) => { + assert_eq!( + error.downcast_ref::().map(std::io::Error::kind), + Some(std::io::ErrorKind::TimedOut) + ); + timeout_elapsed = Some(started.elapsed()); + break; + } + } + } + let elapsed = + timeout_elapsed.expect("saturated host listener returned unconnected streams"); + assert!( + elapsed < Duration::from_secs(1), + "saturated host connect exceeded its deadline bound: {elapsed:?}" + ); + drop(queued); + drop(listener); + fs::remove_file(path).unwrap(); + } + fn record_fixture(name: &str) -> (PathBuf, TerminalHostRecord, HostLivenessLease) { let root = std::env::temp_dir().join(format!( "cmux-host-record-{name}-{}-{}", From bba970e5e67c7ea46d8ce878493efce40ca0ff33 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 20:48:22 -0700 Subject: [PATCH 098/521] fix(tui): preserve shutdown ownership through preflight --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 4 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 13 +- cmux-tui/crates/cmux-tui-core/src/platform.rs | 14 ++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 31 +++++ .../src/terminal_host_runtime.rs | 114 ++---------------- 5 files changed, 65 insertions(+), 111 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 825d357b5bb9..3d1c3319b81e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -1335,6 +1335,10 @@ impl BrowserSurface { self.session.lock().unwrap().take().map(BrowserShutdownOwner) } + pub(crate) fn has_shutdown_owner(&self) -> bool { + self.session.lock().unwrap().is_some() + } + pub fn latest_frame(&self) -> Option { let state = self.state.lock().unwrap(); if matches!(state.status, BrowserStatus::Failed(_)) { diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index cdbad3694692..602c43ff9a72 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -20,7 +20,9 @@ use crate::event_bus::{MuxEventBroadcaster, MuxEventReceiver}; use crate::layout::{Rect, layout_screen}; use crate::model::{Node, Pane, Screen, State, Workspace}; use crate::pairing::PairingBroker; -use crate::surface::{DefaultColors, Surface, SurfaceOptions, SurfaceShutdownOwner}; +use crate::surface::{ + DefaultColors, Surface, SurfaceOptions, SurfaceShutdownOwner, SurfaceShutdownOwnerIdentity, +}; use crate::terminal_host::TerminalId; use crate::terminal_host_runtime::TerminalHostIdentity; #[cfg(unix)] @@ -5076,8 +5078,13 @@ impl Mux { let surface_owner_keys = state .surfaces .values() - .filter_map(|surface| { - surface.shutdown_owner().map(|owner| shutdown_owner_key(surface.id, &owner)) + .filter_map(|surface| match surface.shutdown_owner_identity()? { + SurfaceShutdownOwnerIdentity::Surface => { + Some(ShutdownOwnerKey::Surface(surface.id)) + } + SurfaceShutdownOwnerIdentity::Hosted { terminal_id, incarnation } => { + Some(ShutdownOwnerKey::Hosted { terminal_id, incarnation }) + } }) .collect::>(); (required, surface_owner_keys) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 3688d4a178e1..ea7f3d13fafd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -36,6 +36,14 @@ pub mod transport { imp::connect_until(path, deadline) } + #[cfg(unix)] + pub(crate) fn connect_unix_until( + path: &Path, + deadline: Instant, + ) -> io::Result { + imp::connect_unix_until(path, deadline) + } + impl Listener { pub fn accept(&self) -> io::Result> { self.inner.accept() @@ -67,6 +75,10 @@ pub mod transport { } pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { + connect_unix_until(path, deadline).map(|stream| Box::new(stream) as Box) + } + + pub(super) fn connect_unix_until(path: &Path, deadline: Instant) -> io::Result { ensure_connect_time_remaining(deadline)?; let (address, address_len) = unix_socket_address(path)?; // SAFETY: socket has no pointer arguments and returns a new owned @@ -131,7 +143,7 @@ pub mod transport { break; } stream.set_nonblocking(false)?; - Ok(Box::new(stream)) + Ok(stream) } fn retry_connect_after_would_block( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 7049db2ea93a..eb59c3f653a4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -685,6 +685,15 @@ pub(crate) struct SurfaceShutdownOwner { kind: SurfaceShutdownOwnerKind, } +pub(crate) enum SurfaceShutdownOwnerIdentity { + Surface, + #[cfg(unix)] + Hosted { + terminal_id: String, + incarnation: String, + }, +} + enum SurfaceShutdownOwnerKind { Local(LocalProcess), #[cfg(unix)] @@ -2698,6 +2707,28 @@ impl Surface { } } + /// Describe retained shutdown ownership without consuming or mutating it. + pub(crate) fn shutdown_owner_identity(&self) -> Option { + match self { + Surface::Pty(pty) => { + if pty.local_process.is_some() { + return Some(SurfaceShutdownOwnerIdentity::Surface); + } + #[cfg(unix)] + if let Some(owner) = &pty.hosted_shutdown_owner { + return Some(SurfaceShutdownOwnerIdentity::Hosted { + terminal_id: owner.record.terminal_id.clone(), + incarnation: owner.record.incarnation.clone(), + }); + } + None + } + Surface::Browser(browser) => { + browser.has_shutdown_owner().then_some(SurfaceShutdownOwnerIdentity::Surface) + } + } + } + /// Start runtime termination. Durable terminal-host death is confirmed /// separately through the process-bound discovery record. pub(crate) fn shutdown_owner(&self) -> Option { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 21bd569881d6..19b914156be1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -275,8 +275,12 @@ mod unix { use std::collections::{HashMap, HashSet}; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Write}; + #[cfg(all(test, target_os = "linux"))] use std::mem::{offset_of, size_of}; - use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; + use std::os::fd::{AsRawFd, RawFd}; + #[cfg(all(test, target_os = "linux"))] + use std::os::fd::{FromRawFd, OwnedFd}; + #[cfg(test)] use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; use std::os::unix::net::{UnixListener, UnixStream}; @@ -2002,114 +2006,10 @@ mod unix { } fn connect_once_until(path: &Path, deadline: Instant) -> std::io::Result { - let (address, address_len) = unix_socket_address(path)?; - // SAFETY: socket has no pointer arguments and returns a new owned - // descriptor on success. - let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; - if descriptor < 0 { - return Err(std::io::Error::last_os_error()); - } - // SAFETY: descriptor is a fresh successful socket result and this - // OwnedFd takes its sole ownership. - let descriptor = unsafe { OwnedFd::from_raw_fd(descriptor) }; - // SAFETY: F_GETFD only reads flags from this valid descriptor. - let descriptor_flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFD) }; - if descriptor_flags < 0 { - return Err(std::io::Error::last_os_error()); - } - // SAFETY: F_SETFD updates flags on this valid descriptor. - if unsafe { - libc::fcntl(descriptor.as_raw_fd(), libc::F_SETFD, descriptor_flags | libc::FD_CLOEXEC) - } < 0 - { - return Err(std::io::Error::last_os_error()); - } - let stream = UnixStream::from(descriptor); - stream.set_nonblocking(true)?; - // SAFETY: address is an initialized sockaddr_un with its exact - // kernel-visible length, and stream owns a valid AF_UNIX socket. - let connected = unsafe { - libc::connect( - stream.as_raw_fd(), - (&raw const address).cast::(), - address_len, - ) - }; - if connected != 0 { - let error = std::io::Error::last_os_error(); - let code = error.raw_os_error(); - if code != Some(libc::EINPROGRESS) - && code != Some(libc::EAGAIN) - && code != Some(libc::EWOULDBLOCK) - { - return Err(error); - } - wait_for_connect(&stream, deadline)?; - } - stream.set_nonblocking(false)?; - Ok(stream) - } - - fn wait_for_connect(stream: &UnixStream, deadline: Instant) -> std::io::Result<()> { - loop { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "terminal host connection exceeded its deadline", - )); - }; - if remaining.is_zero() { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "terminal host connection exceeded its deadline", - )); - } - let timeout_ms = remaining.as_millis().max(1).min(i32::MAX as u128) as i32; - let mut descriptor = - libc::pollfd { fd: stream.as_raw_fd(), events: libc::POLLOUT, revents: 0 }; - // SAFETY: descriptor points to one initialized pollfd for the - // duration of this call. - let result = unsafe { libc::poll(&mut descriptor, 1, timeout_ms) }; - if result == 0 { - continue; - } - if result < 0 { - let error = std::io::Error::last_os_error(); - if error.kind() == std::io::ErrorKind::Interrupted { - continue; - } - return Err(error); - } - if descriptor.revents & libc::POLLNVAL != 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "terminal host connection socket became invalid", - )); - } - let mut socket_error = 0; - let mut socket_error_len = libc::socklen_t::try_from(size_of::()).unwrap(); - // SAFETY: socket_error and its length describe a writable c_int, - // and stream owns a valid socket descriptor. - if unsafe { - libc::getsockopt( - stream.as_raw_fd(), - libc::SOL_SOCKET, - libc::SO_ERROR, - (&raw mut socket_error).cast(), - &raw mut socket_error_len, - ) - } != 0 - { - return Err(std::io::Error::last_os_error()); - } - return if socket_error == 0 { - Ok(()) - } else { - Err(std::io::Error::from_raw_os_error(socket_error)) - }; - } + crate::platform::transport::connect_unix_until(path, deadline) } + #[cfg(all(test, target_os = "linux"))] fn unix_socket_address(path: &Path) -> std::io::Result<(libc::sockaddr_un, libc::socklen_t)> { const SUN_PATH_CAPACITY: usize = size_of::() - offset_of!(libc::sockaddr_un, sun_path); From fef8efde8f1d711d962040185e5ce51c141a388b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:26:55 -0700 Subject: [PATCH 099/521] test(tui): expose last-surface shutdown race --- cmux-tui/crates/cmux-tui/tests/cli.rs | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index c3cfb8883459..4c26d2e5fb26 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -317,6 +317,7 @@ fn legacy_server_process_helper() { | "cleanup-failure" | "applied-close-error" | "applied-close-disconnect" + | "exit-after-close" | "persistent-close-error" | "browser-surface" ) { @@ -521,6 +522,7 @@ fn legacy_server_process_helper() { | "kill-caller" | "applied-close-error" | "applied-close-disconnect" + | "exit-after-close" | "persistent-close-error" | "zombie-child" | "reparent-on-close" @@ -610,6 +612,13 @@ fn legacy_server_process_helper() { } assert_eq!(close_request["cmd"].as_str(), Some("close-surface")); assert_eq!(close_request["surface"].as_u64(), Some(expected_surface)); + if scenario == "exit-after-close" { + drop(reader); + drop(stream); + drop(listener); + fs::remove_file(&socket).unwrap(); + return; + } if scenario == "applied-close-disconnect" { drop(reader); drop(stream); @@ -1675,6 +1684,32 @@ fn server_stop_reconnects_after_an_applied_legacy_close_drops_the_control_stream assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_completes_when_the_last_legacy_surface_exits_with_its_server() { + let mut server = + LegacyServerProcess::start("legacy-server-exit-after-close", "exit-after-close", None); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + assert!(server.child.wait().unwrap().success()); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + !process_exists(descendant_pid), + "last-surface server exit leaked its captured PTY owner" + ); +} + #[cfg(unix)] #[test] fn server_stop_refuses_when_a_legacy_close_error_leaves_the_surface_present() { From 7257271651dd8d48e0964c418e750a3918c08893 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:38:43 -0700 Subject: [PATCH 100/521] fix(tui): retain legacy owners across server exit --- .../crates/cmux-tui/src/server_lifecycle.rs | 125 +++++++++++++++--- .../src/server_lifecycle/legacy_process.rs | 3 + cmux-tui/crates/cmux-tui/tests/cli.rs | 19 ++- 3 files changed, 128 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a12bc3e3b7fa..91cc2413c6d6 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -80,9 +80,26 @@ fn ensure_legacy_helper_active() -> anyhow::Result<()> { Ok(()) } +#[cfg(unix)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LegacyConnectionPhase { + Listing, + Inspecting, + ClosingVerifiedSnapshot, +} + #[cfg(unix)] #[derive(Debug)] -struct LegacyConnectionInterrupted; +struct LegacyConnectionInterrupted { + phase: LegacyConnectionPhase, +} + +#[cfg(unix)] +impl LegacyConnectionInterrupted { + fn during(phase: LegacyConnectionPhase) -> Self { + Self { phase } + } +} #[cfg(unix)] impl std::fmt::Display for LegacyConnectionInterrupted { @@ -370,6 +387,7 @@ impl ServerLifecycle { let mut consecutive_empty_scans = 0; let mut owners = Vec::::new(); let mut owner_ids = HashSet::::new(); + let mut previous_snapshot_completed = false; for _ in 0..LEGACY_MAX_SCAN_ROUNDS { ensure_legacy_helper_active()?; if Instant::now() >= deadline { @@ -384,13 +402,34 @@ impl ServerLifecycle { deadline, ) { Ok(closed) => closed, - Err(error) if error.downcast_ref::().is_some() => { - self.reconnect_legacy_server(expected, deadline)?; + Err(error) => { + let Some(phase) = error + .downcast_ref::() + .map(|error| error.phase) + else { + return Err(error); + }; + let verified_exit = phase == LegacyConnectionPhase::ClosingVerifiedSnapshot + || (phase == LegacyConnectionPhase::Listing && previous_snapshot_completed); + if verified_exit && legacy_server_retired(expected, &self.path)? { + return Ok(owners); + } + if let Err(reconnect_error) = self.reconnect_legacy_server(expected, deadline) { + if verified_exit + && wait_for_legacy_server_retirement_until( + expected, &self.path, deadline, + )? + { + return Ok(owners); + } + return Err(reconnect_error); + } consecutive_empty_scans = 0; + previous_snapshot_completed = false; continue; } - Err(error) => return Err(error), }; + previous_snapshot_completed = true; if closed == 0 { consecutive_empty_scans += 1; if consecutive_empty_scans == LEGACY_STABLE_EMPTY_SCANS { @@ -436,14 +475,14 @@ impl ServerLifecycle { ) -> anyhow::Result { let list_request_id = take_legacy_request_id(next_request_id)?; set_transport_deadline(self.reader.get_mut().as_ref(), deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + .map_err(|_| LegacyConnectionInterrupted::during(LegacyConnectionPhase::Listing))?; write_json_line( self.reader.get_mut(), &json!({"id": list_request_id, "cmd": "list-workspaces"}), ) - .map_err(|_| LegacyConnectionInterrupted)?; + .map_err(|_| LegacyConnectionInterrupted::during(LegacyConnectionPhase::Listing))?; let response = read_response_until(&mut self.reader, list_request_id, deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + .map_err(|_| LegacyConnectionInterrupted::during(LegacyConnectionPhase::Listing))?; let data = response_data(&response)?; let surfaces = legacy_surfaces(data)?; @@ -452,9 +491,17 @@ impl ServerLifecycle { if surface.kind != "pty" { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } + } + + // Capture every process-backed surface before sending the first + // topology mutation. If the last close also exits the older server, + // the complete verified snapshot remains available for cleanup. + for surface in &surfaces { + ensure_legacy_helper_active()?; let process_request_id = take_legacy_request_id(next_request_id)?; - set_transport_deadline(self.reader.get_mut().as_ref(), deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline).map_err(|_| { + LegacyConnectionInterrupted::during(LegacyConnectionPhase::Inspecting) + })?; write_json_line( self.reader.get_mut(), &json!({ @@ -463,10 +510,11 @@ impl ServerLifecycle { "surface": surface.id, }), ) - .map_err(|_| LegacyConnectionInterrupted)?; + .map_err(|_| LegacyConnectionInterrupted::during(LegacyConnectionPhase::Inspecting))?; let process_response = - read_response_until(&mut self.reader, process_request_id, deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + read_response_until(&mut self.reader, process_request_id, deadline).map_err( + |_| LegacyConnectionInterrupted::during(LegacyConnectionPhase::Inspecting), + )?; let process_data = response_data(&process_response)?; let pid = process_data .get("pid") @@ -487,17 +535,27 @@ impl ServerLifecycle { } else if !surface.dead { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } + } + for surface in &surfaces { + ensure_legacy_helper_active()?; let request_id = take_legacy_request_id(next_request_id)?; - set_transport_deadline(self.reader.get_mut().as_ref(), deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline).map_err(|_| { + LegacyConnectionInterrupted::during(LegacyConnectionPhase::ClosingVerifiedSnapshot) + })?; write_json_line( self.reader.get_mut(), &json!({"id": request_id, "cmd": "close-surface", "surface": surface.id}), ) - .map_err(|_| LegacyConnectionInterrupted)?; - let _response = read_response_until(&mut self.reader, request_id, deadline) - .map_err(|_| LegacyConnectionInterrupted)?; + .map_err(|_| { + LegacyConnectionInterrupted::during(LegacyConnectionPhase::ClosingVerifiedSnapshot) + })?; + let _response = + read_response_until(&mut self.reader, request_id, deadline).map_err(|_| { + LegacyConnectionInterrupted::during( + LegacyConnectionPhase::ClosingVerifiedSnapshot, + ) + })?; // Older servers can commit topology removal, then report a late // runtime-cleanup error. Reconcile every close response against // subsequent authoritative workspace snapshots. A genuinely @@ -675,6 +733,39 @@ fn verified_legacy_process( .ok_or_else(|| anyhow::anyhow!(messages.legacy_peer_mismatch)) } +#[cfg(unix)] +fn legacy_process_exited(expected: ProcessIdentity) -> anyhow::Result { + let messages = &crate::localization::catalog().server; + match ProcessIdentity::capture(expected.pid()) { + Ok(None) => Ok(true), + Ok(Some(current)) if current == expected => Ok(false), + Ok(Some(_)) => Err(anyhow::anyhow!(messages.legacy_peer_mismatch)), + Err(_) => Err(anyhow::anyhow!(messages.legacy_peer_unavailable)), + } +} + +#[cfg(unix)] +fn legacy_server_retired(expected: ProcessIdentity, path: &Path) -> anyhow::Result { + Ok(!path.exists() && legacy_process_exited(expected)?) +} + +#[cfg(unix)] +fn wait_for_legacy_server_retirement_until( + expected: ProcessIdentity, + path: &Path, + deadline: Instant, +) -> anyhow::Result { + loop { + if legacy_server_retired(expected, path)? { + return Ok(true); + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + std::thread::sleep(remaining.min(LEGACY_HELPER_POLL_INTERVAL)); + } +} + #[cfg(all(unix, test))] fn terminate_legacy_process_tree(process: ProcessIdentity) -> anyhow::Result<()> { terminate_process_tree(process) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index aa2b69883030..cc1f977911d1 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -721,6 +721,9 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { return Err(io::Error::other("process metadata id mismatch")); } let fields = remainder.split_whitespace().collect::>(); + if fields.first() == Some(&"Z") { + return Ok(None); + } let parent = fields .get(1) .and_then(|value| value.parse::().ok()) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 4c26d2e5fb26..adb80edf8c1b 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -187,6 +187,21 @@ fn process_exists(pid: u32) -> bool { std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) } +#[cfg(target_os = "linux")] +fn process_is_active(pid: u32) -> bool { + let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + stat.rsplit_once(") ") + .and_then(|(_, fields)| fields.split_whitespace().next()) + .map_or_else(|| process_exists(pid), |state| state != "Z") +} + +#[cfg(all(unix, not(target_os = "linux")))] +fn process_is_active(pid: u32) -> bool { + process_exists(pid) +} + #[cfg(unix)] fn process_group_exists(pid: u32) -> bool { let Ok(pid) = libc::pid_t::try_from(pid) else { return false }; @@ -1701,11 +1716,11 @@ fn server_stop_completes_when_the_last_legacy_surface_exits_with_its_server() { assert_success(&output); assert!(server.child.wait().unwrap().success()); let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { + while process_is_active(descendant_pid) && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); } assert!( - !process_exists(descendant_pid), + !process_is_active(descendant_pid), "last-surface server exit leaked its captured PTY owner" ); } From bfedef590e758d46002872a773da8f714f21174d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:35:05 -0700 Subject: [PATCH 101/521] test(tui): expose final upgrade review gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 39 +++++++++++++++++++ cmux-tui/crates/cmux-tui-core/build.rs | 23 +++++++++++ .../cmux-tui-core/src/process_session.rs | 13 +++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 16 ++++++++ 4 files changed, 91 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 6195d6bc5d45..ecc1471cf27b 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1935,6 +1935,45 @@ mod tests { ); } + #[test] + fn malformed_target_listing_never_confirms_absence() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + let request = ws.read().unwrap(); + let Message::Text(request) = request else { panic!("expected text request") }; + let request: Value = serde_json::from_str(&request).unwrap(); + assert_eq!(request["method"], "Target.getTargets"); + ws.send(Message::Text( + json!({ + "id": request["id"], + "result": { + "targetInfos": [ + {"targetId": "known-target"}, + {"type": "page"}, + {"targetId": 7} + ] + } + }) + .to_string() + .into(), + )) + .unwrap(); + }); + let (event_tx, _event_rx) = sync_channel(1); + let client = + CdpClient::connect(&format!("ws://{addr}/devtools/browser/fake"), event_tx).unwrap(); + + let result = + client.target_exists_with_timeout("missing-target", Duration::from_millis(500)); + + drop(client); + server.join().unwrap(); + assert!(result.is_err(), "malformed target metadata confirmed target absence"); + } + #[test] fn socket_shutdown_disconnects_a_backpressured_event_receiver() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index 8d51e93015a8..be49d725594c 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -339,4 +339,27 @@ mod tests { assert!(source_changed.contains("-dirty-")); assert_ne!(source_changed, clean); } + + #[test] + fn ghostty_identity_tracks_native_package_sources() { + let repository = TestRepository::new("ghostty-packages"); + fs::create_dir(repository.root.join("include")).unwrap(); + fs::create_dir(repository.root.join("src")).unwrap(); + fs::create_dir_all(repository.root.join("pkg/simdutf")).unwrap(); + fs::write(repository.root.join("build.zig"), b"build").unwrap(); + fs::write(repository.root.join("build.zig.zon"), b"dependencies").unwrap(); + fs::write(repository.root.join("include/ghostty.h"), b"header").unwrap(); + fs::write(repository.root.join("src/main.zig"), b"source").unwrap(); + fs::write(repository.root.join("pkg/simdutf/input"), b"one").unwrap(); + run_git(&repository.root, &["add", "."]); + run_git(&repository.root, &["commit", "-qm", "add native inputs"]); + let clean = + source_identity(&repository.root, &repository.root, GHOSTTY_SOURCE_PATHS).unwrap(); + + fs::write(repository.root.join("pkg/simdutf/input"), b"two").unwrap(); + let changed = + source_identity(&repository.root, &repository.root, GHOSTTY_SOURCE_PATHS).unwrap(); + + assert_ne!(changed, clean, "Ghostty package changes retained the clean build identity"); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 00046cfe4e06..28c79e212a2c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -419,10 +419,21 @@ fn accept_natural_reaper_command( #[cfg(test)] thread_local! { static PROCESS_TABLE_SCAN_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static STABLE_PROCESS_PREFLIGHT_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; static STABLE_PROCESS_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; } +#[cfg(test)] +pub(crate) fn reset_stable_process_preflight_count_for_test() { + STABLE_PROCESS_PREFLIGHT_COUNT.set(0); +} + +#[cfg(test)] +pub(crate) fn stable_process_preflight_count_for_test() -> usize { + STABLE_PROCESS_PREFLIGHT_COUNT.get() +} + pub(crate) struct SessionProcessSnapshot { sessions: HashSet, state: Mutex, @@ -974,6 +985,8 @@ pub fn require_stable_process_signaling() -> io::Result<()> { } pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::Result<()> { + #[cfg(test)] + STABLE_PROCESS_PREFLIGHT_COUNT.set(STABLE_PROCESS_PREFLIGHT_COUNT.get() + 1); #[cfg(test)] if FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.get() { return Err(io::Error::new( diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 236d9b4a36fa..eed3657c8bad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -5274,6 +5274,22 @@ mod tests { assert_eq!(PROTOCOL_VERSION, 10); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn repeated_identify_never_rescans_the_global_process_table() { + let mux = test_mux(); + crate::process_session::reset_stable_process_preflight_count_for_test(); + + handle_command(&mux, 0, Command::Identify, &test_writer()).unwrap(); + handle_command(&mux, 0, Command::Identify, &test_writer()).unwrap(); + + assert_eq!( + crate::process_session::stable_process_preflight_count_for_test(), + 0, + "ordinary identity requests repeated the process-control startup preflight" + ); + } + #[test] fn identify_exposes_shutdown_cleanup_health() { let mux = test_mux(); From 97b43bbf5b5d7d52e62150dc8d36aca6a38a1231 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:42:44 -0700 Subject: [PATCH 102/521] fix(tui): close final upgrade review gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 18 ++++++++-- cmux-tui/crates/cmux-tui-core/build.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 ++ .../cmux-tui-core/src/process_session.rs | 34 +++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 +- 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index ecc1471cf27b..626f70b2664c 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -772,10 +772,22 @@ impl CdpClient { .get("targetInfos") .and_then(Value::as_array) .ok_or_else(|| anyhow::anyhow!("Target.getTargets response missing targetInfos"))?; - Ok(targets + targets .iter() - .filter_map(|target| target.get("targetId").and_then(Value::as_str).map(str::to_string)) - .collect()) + .enumerate() + .map(|(index, target)| { + target + .get("targetId") + .and_then(Value::as_str) + .filter(|target_id| !target_id.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + anyhow::anyhow!( + "Target.getTargets targetInfos[{index}] missing a valid targetId" + ) + }) + }) + .collect() } pub fn close_target_detached(&self, target_id: &str) -> anyhow::Result<()> { diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index be49d725594c..1009ed4c3a8b 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -9,7 +9,7 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; const FNV_OFFSET: u64 = 0xcbf29ce484222325; const FNV_PRIME: u64 = 0x100000001b3; const CMUX_TUI_SOURCE_PATHS: &[&str] = &["Cargo.toml", "Cargo.lock", "bindings/rust", "crates"]; -const GHOSTTY_SOURCE_PATHS: &[&str] = &["build.zig", "build.zig.zon", "include", "src"]; +const GHOSTTY_SOURCE_PATHS: &[&str] = &["build.zig", "build.zig.zon", "include", "pkg", "src"]; fn main() { for name in [ diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 602c43ff9a72..452fae5e4574 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1715,6 +1715,8 @@ impl Mux { let workspace_id_by_key = workspaces.iter().map(|workspace| (workspace.key.clone(), workspace.id)).collect(); surface_options.browser_session_name = session.clone(); + #[cfg(unix)] + crate::process_session::initialize_stable_process_signaling(); let mux = Arc::new(Mux { workspace_registry: Mutex::new(registry), state: Mutex::new(State { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 28c79e212a2c..501033ceddd7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -30,6 +30,23 @@ const NATURAL_REAP_MAX_ATTEMPTS: usize = 3; static NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); static NATURAL_REAPER: OnceLock>> = OnceLock::new(); +static STABLE_PROCESS_SIGNALING: OnceLock> = + OnceLock::new(); + +struct StableProcessSignalingFailure { + kind: io::ErrorKind, + message: String, +} + +impl StableProcessSignalingFailure { + fn from_error(error: io::Error) -> Self { + Self { kind: error.kind(), message: error.to_string() } + } + + fn to_error(&self) -> io::Error { + io::Error::new(self.kind, self.message.clone()) + } +} #[cfg(test)] thread_local! { @@ -984,6 +1001,23 @@ pub fn require_stable_process_signaling() -> io::Result<()> { require_stable_process_signaling_until(Instant::now() + PROCESS_SESSION_PREFLIGHT_TIMEOUT) } +/// Cache the process-level capability once per server generation. Startup, +/// identity responses, and PTY creation share this result; shutdown still +/// performs a fresh bounded preflight immediately before mutating topology. +pub(crate) fn initialize_stable_process_signaling() { + let _ = stable_process_signaling(); +} + +pub(crate) fn require_cached_stable_process_signaling() -> io::Result<()> { + stable_process_signaling().as_ref().map_err(StableProcessSignalingFailure::to_error).copied() +} + +fn stable_process_signaling() -> &'static Result<(), StableProcessSignalingFailure> { + STABLE_PROCESS_SIGNALING.get_or_init(|| { + require_stable_process_signaling().map_err(StableProcessSignalingFailure::from_error) + }) +} + pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::Result<()> { #[cfg(test)] STABLE_PROCESS_PREFLIGHT_COUNT.set(STABLE_PROCESS_PREFLIGHT_COUNT.get() + 1); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index eed3657c8bad..743268530e6b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -3094,7 +3094,7 @@ fn advertised_capabilities() -> Vec<&'static str> { // Atomic shutdown is advertised only where PTY session descendants can // be enumerated and signaled through exact process handles. #[cfg(any(target_os = "macos", target_os = "linux"))] - if crate::process_session::require_stable_process_signaling().is_ok() { + if crate::process_session::require_cached_stable_process_signaling().is_ok() { capabilities.push(SERVER_SHUTDOWN_CAPABILITY); } capabilities diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index eb59c3f653a4..f51d84bb12b5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1225,7 +1225,7 @@ impl Surface { terminal_id: Option, ) -> anyhow::Result> { #[cfg(unix)] - crate::process_session::require_stable_process_signaling()?; + crate::process_session::require_cached_stable_process_signaling()?; #[cfg(unix)] if let Some(root) = opts.terminal_host_root.clone() { let default_colors = mux.upgrade().map(|mux| mux.default_colors()).unwrap_or_default(); From 97890746ce744211d958a6456a2b6323d2ad0966 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:53:13 -0700 Subject: [PATCH 103/521] test(tui): support argument-free replay assertions --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 91cc2413c6d6..85b86e55de20 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1223,7 +1223,6 @@ mod tests { "{message}" ); let replay_arguments = std::env::args_os().skip(1).collect::>(); - assert!(!replay_arguments.is_empty(), "test runner must provide an argument"); let mut replay = launcher; for argument in replay_arguments { replay.push(' '); @@ -1247,6 +1246,10 @@ mod tests { #[test] fn replay_command_preserves_quoted_and_empty_arguments() { + assert_eq!( + replay_command_from("cmux-tui".to_string(), std::iter::empty::<&str>()), + "cmux-tui" + ); assert_eq!( replay_command_from("cmux-tui".to_string(), ["--session", "two words", "", "it's"]), "cmux-tui --session 'two words' '' 'it'\"'\"'s'" From 8b51c6b60f5917b8e76d22469921a235b70dee77 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:07:44 -0700 Subject: [PATCH 104/521] test(tui): expose stranded process fences --- .../src/server_lifecycle/legacy_process.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index cc1f977911d1..3a2e61da6e25 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -12,6 +12,8 @@ const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); #[cfg(test)] thread_local! { static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static STABLE_HANDLE_CAPTURE_BUDGET: std::cell::Cell> = + const { std::cell::Cell::new(None) }; } fn ensure_helper_active() -> io::Result<()> { @@ -88,6 +90,19 @@ impl ProcessIdentity { } fn stable_handle(self) -> io::Result> { + #[cfg(test)] + STABLE_HANDLE_CAPTURE_BUDGET.with(|budget| { + if let Some(remaining) = budget.get() { + if remaining == 0 { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "forced stable-handle capture failure", + )); + } + budget.set(Some(remaining - 1)); + } + Ok(()) + })?; let Some(process) = StableProcessHandle::capture(self.pid)? else { return Ok(None); }; @@ -828,6 +843,61 @@ mod tests { ); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn frozen_tree_drop_resumes_with_its_original_process_proof() { + use std::io::Read as _; + use std::os::fd::AsRawFd as _; + + let mut child = + Command::new("yes").stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap(); + let mut stdout = child.stdout.take().unwrap(); + let descriptor = stdout.as_raw_fd(); + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFL) }; + assert!(flags >= 0); + assert_eq!(unsafe { libc::fcntl(descriptor, libc::F_SETFL, flags | libc::O_NONBLOCK) }, 0); + let pid = libc::pid_t::try_from(child.id()).unwrap(); + let process = ProcessIdentity::capture(pid).unwrap().unwrap(); + STABLE_HANDLE_CAPTURE_BUDGET.set(Some(1)); + + let tree = + FrozenProcessTree::freeze(process, Instant::now() + Duration::from_secs(1)).unwrap(); + let mut output = [0_u8; 4_096]; + loop { + match stdout.read(&mut output) { + Ok(0) => break, + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => break, + Err(error) => panic!("failed to drain stopped child output: {error}"), + } + } + drop(tree); + let deadline = Instant::now() + Duration::from_millis(250); + let mut resumed = false; + while Instant::now() < deadline { + match stdout.read(&mut output) { + Ok(0) => break, + Ok(_) => { + resumed = true; + break; + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => panic!("failed to read resumed child output: {error}"), + } + std::thread::sleep(Duration::from_millis(5)); + } + + STABLE_HANDLE_CAPTURE_BUDGET.set(None); + // SAFETY: only the exact test-owned child can still use this unreaped PID. + unsafe { + libc::kill(pid, libc::SIGCONT); + } + let _ = child.kill(); + let _ = child.wait(); + + assert!(resumed, "dropping a process fence left its exact process stopped"); + } + #[test] fn termination_accepts_a_verified_process_that_already_exited() { let mut child = Command::new("sleep") From 570524541c9ba9b5cd22860901d278c30f0b630d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:12:34 -0700 Subject: [PATCH 105/521] fix(tui): retain process proofs across fences --- .../src/server_lifecycle/legacy_process.rs | 207 ++++++++---------- 1 file changed, 88 insertions(+), 119 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 3a2e61da6e25..21e1292f58cb 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -49,6 +49,7 @@ impl ProcessIdentity { process_snapshot(pid).map(|snapshot| snapshot.map(|snapshot| snapshot.identity)) } + #[cfg(test)] fn signal(self, signal: libc::c_int) -> io::Result { let Some(process) = self.stable_handle()? else { return Ok(ExactSignalResult::Gone); @@ -60,35 +61,6 @@ impl ProcessIdentity { } } - fn signal_in_session( - self, - session: libc::pid_t, - signal: libc::c_int, - ) -> io::Result { - let Some(process) = self.stable_handle()? else { - return Ok(ExactSignalResult::Gone); - }; - // Bracket the PID-based session query with the stable handle. If the - // PID is recycled during getsid(2), no signal reaches its new owner. - let current_session = unsafe { libc::getsid(self.pid) }; - if current_session < 0 { - let error = io::Error::last_os_error(); - return if error.raw_os_error() == Some(libc::ESRCH) { - Ok(ExactSignalResult::Gone) - } else { - Err(error) - }; - } - if current_session != session || !process.matches_current()? { - return Ok(ExactSignalResult::Gone); - } - if process.signal(signal)? { - Ok(ExactSignalResult::Signaled) - } else { - Ok(ExactSignalResult::Gone) - } - } - fn stable_handle(self) -> io::Result> { #[cfg(test)] STABLE_HANDLE_CAPTURE_BUDGET.with(|budget| { @@ -125,12 +97,69 @@ struct ProcessSnapshot { parent: libc::pid_t, } +#[cfg(test)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExactSignalResult { Signaled, Gone, } +struct ProcessFence { + identity: ProcessIdentity, + process: StableProcessHandle, + armed: bool, +} + +impl ProcessFence { + fn acquire(identity: ProcessIdentity) -> io::Result> { + let Some(process) = identity.stable_handle()? else { return Ok(None) }; + if !process.signal(libc::SIGSTOP)? { + return Ok(None); + } + Ok(Some(Self { identity, process, armed: true })) + } + + fn acquire_in_session( + identity: ProcessIdentity, + session: libc::pid_t, + ) -> io::Result> { + let Some(process) = identity.stable_handle()? else { return Ok(None) }; + if !process.matches_current()? { + return Ok(None); + } + // SAFETY: getsid only reads metadata. Exact-handle probes bracket the + // reusable PID so a replacement process is never stopped. + let current_session = unsafe { libc::getsid(identity.pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; + } + if current_session != session || !process.matches_current()? { + return Ok(None); + } + if !process.signal(libc::SIGSTOP)? { + return Ok(None); + } + Ok(Some(Self { identity, process, armed: true })) + } + + fn kill(&mut self) -> io::Result<()> { + let result = self.process.signal(libc::SIGKILL); + if result.is_ok() { + self.armed = false; + } + result.map(|_| ()) + } +} + +impl Drop for ProcessFence { + fn drop(&mut self) { + if self.armed { + let _ = self.process.signal(libc::SIGCONT); + } + } +} + #[cfg(test)] pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> { terminate_process_tree_until(process, Instant::now() + PROCESS_TREE_RETRY_TIMEOUT) @@ -277,18 +306,11 @@ pub(super) fn capture_process_session( "PTY owner was not captured in the verified server process tree", )); } - if process.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { - return Ok(None); - } + let Some(process_fence) = ProcessFence::acquire(process)? else { return Ok(None) }; let captured = (|| { ensure_helper_active()?; - if ProcessIdentity::capture(pid)? != Some(process) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "PTY owner process identity changed after fencing", - )); - } - // SAFETY: getsid only queries exact processes revalidated above. + // SAFETY: the retained exact process handle keeps this stopped PID + // bound to the verified process instance. let session = unsafe { libc::getsid(pid) }; if session <= 1 { return Err(io::Error::last_os_error()); @@ -320,7 +342,7 @@ pub(super) fn capture_process_session( members.dedup(); Ok(CapturedProcessSession(CapturedSession { id: session, members })) })(); - let _ = process.signal(libc::SIGCONT); + drop(process_fence); captured.map(Some) } @@ -337,7 +359,7 @@ impl CapturedSession { Ok(Some(mut probe)) => { match cmux_tui_core::process_session::kill_until_only_reserved( self.id, - probe.process.pid, + probe.process.identity.pid, deadline, ) { Ok(true) => match probe.kill() { @@ -371,9 +393,7 @@ impl CapturedSession { } struct SessionProbe { - process: ProcessIdentity, - session: libc::pid_t, - armed: bool, + process: ProcessFence, } impl SessionProbe { @@ -383,56 +403,38 @@ impl SessionProbe { if ProcessIdentity::capture(process.pid)? != Some(*process) { continue; } - if process.signal_in_session(session.id, libc::SIGSTOP)? != ExactSignalResult::Signaled - { + let Some(process) = ProcessFence::acquire_in_session(*process, session.id)? else { continue; - } - return Ok(Some(Self { process: *process, session: session.id, armed: true })); + }; + return Ok(Some(Self { process })); } Ok(None) } fn kill(&mut self) -> io::Result<()> { - let _ = self.process.signal_in_session(self.session, libc::SIGKILL)?; - self.armed = false; - Ok(()) - } -} - -impl Drop for SessionProbe { - fn drop(&mut self) { - if self.armed { - let _ = self.process.signal_in_session(self.session, libc::SIGCONT); - } + self.process.kill() } } struct FrozenProcessTree { - root: ProcessIdentity, - descendants: Vec, - armed: bool, + root: ProcessFence, + descendants: Vec, } impl FrozenProcessTree { fn freeze(root: ProcessIdentity, deadline: Instant) -> io::Result { ensure_helper_active()?; - let mut tree = Self { root, descendants: Vec::new(), armed: true }; - if root.signal(libc::SIGSTOP)? == ExactSignalResult::Gone { + let Some(root) = ProcessFence::acquire(root)? else { return Err(io::Error::new( io::ErrorKind::NotFound, "server exited before its process tree was fenced", )); - } - if ProcessIdentity::capture(root.pid)? != Some(root) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "server process identity changed after fencing", - )); - } + }; + let mut tree = Self { root, descendants: Vec::new() }; let helper_pid = libc::pid_t::try_from(std::process::id()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid helper process id"))?; - let mut known = HashSet::from([root.pid, helper_pid]); + let mut known = HashSet::from([tree.root.identity.pid, helper_pid]); for _ in 0..PROCESS_TREE_MAX_ROUNDS { ensure_helper_active()?; if Instant::now() >= deadline { @@ -441,8 +443,9 @@ impl FrozenProcessTree { "legacy process tree did not stabilize before the shutdown deadline", )); } - let parents = - std::iter::once(root).chain(tree.descendants.iter().copied()).collect::>(); + let parents = std::iter::once(tree.root.identity) + .chain(tree.descendants.iter().map(|process| process.identity)) + .collect::>(); let mut added = false; for parent in parents { ensure_helper_active()?; @@ -457,18 +460,11 @@ impl FrozenProcessTree { if snapshot.parent != parent.pid { continue; } - match snapshot.identity.signal(libc::SIGSTOP)? { - ExactSignalResult::Gone => continue, - ExactSignalResult::Signaled => {} - } - if ProcessIdentity::capture(pid)? != Some(snapshot.identity) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "descendant process identity changed after fencing", - )); - } + let Some(process) = ProcessFence::acquire(snapshot.identity)? else { + continue; + }; known.insert(pid); - tree.descendants.push(snapshot.identity); + tree.descendants.push(process); added = true; } } @@ -482,7 +478,7 @@ impl FrozenProcessTree { fn captured_sessions(&self) -> io::Result> { // SAFETY: getsid only queries live, exact processes held stopped by // this FrozenProcessTree. - let root_session = unsafe { libc::getsid(self.root.pid) }; + let root_session = unsafe { libc::getsid(self.root.identity.pid) }; if root_session < 0 { return Err(io::Error::last_os_error()); } @@ -495,16 +491,16 @@ impl FrozenProcessTree { for process in &self.descendants { // SAFETY: every descendant remains stopped and exact until this // FrozenProcessTree is dropped. - let session = unsafe { libc::getsid(process.pid) }; + let session = unsafe { libc::getsid(process.identity.pid) }; if session < 0 { return Err(io::Error::last_os_error()); } if session > 1 && session != root_session && session != helper_session { if let Some(captured) = sessions.iter_mut().find(|captured| captured.id == session) { - captured.members.push(*process); + captured.members.push(process.identity); } else { - sessions.push(CapturedSession { id: session, members: vec![*process] }); + sessions.push(CapturedSession { id: session, members: vec![process.identity] }); } } } @@ -517,37 +513,10 @@ impl FrozenProcessTree { } fn kill(mut self) -> io::Result<()> { - if !self.armed { - return Ok(()); - } - for process in std::iter::once(self.root).chain(self.descendants.iter().copied()) { - if let Some(current) = ProcessIdentity::capture(process.pid)? - && current != process - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "frozen process identity changed", - )); - } - } - for process in self.descendants.iter().rev().copied() { - let _ = process.signal(libc::SIGKILL)?; - } - let _ = self.root.signal(libc::SIGKILL)?; - self.armed = false; - Ok(()) - } -} - -impl Drop for FrozenProcessTree { - fn drop(&mut self) { - if !self.armed { - return; - } - for process in self.descendants.iter().rev().copied() { - let _ = process.signal(libc::SIGCONT); + for process in self.descendants.iter_mut().rev() { + process.kill()?; } - let _ = self.root.signal(libc::SIGCONT); + self.root.kill() } } From fc9cc49bd528f932be78cd0c0a3a1210f3180289 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:42:44 -0700 Subject: [PATCH 106/521] test(tui): enforce Windows cross-builds --- .github/workflows/cmux-tui-build-package.yml | 1 - .github/workflows/cmux-tui.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 451dd1448657..e46b8ed6b7fc 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -184,7 +184,6 @@ jobs: if: inputs.include_windows runs-on: ${{ vars.WINDOWS_RUNNER || 'windows-latest' }} timeout-minutes: 60 - continue-on-error: true permissions: contents: read steps: diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 2cf7561f9d05..8a4156fe60f1 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -232,7 +232,6 @@ jobs: name: windows experimental (x86_64-gnu) runs-on: windows-latest timeout-minutes: 40 - continue-on-error: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From 4bfe92b01ffe4fa122aa594df6284097074fc4ec Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:46:54 -0700 Subject: [PATCH 107/521] fix(tui): isolate Unix child reaper plumbing --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 38 +++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index f51d84bb12b5..147f605786f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -810,6 +810,28 @@ struct LocalPtyProcess { normal_cleanup_started: AtomicBool, } +#[cfg(unix)] +type LocalChildReaperLease = crate::process_session::ReservedChildReaperLease; + +#[cfg(not(unix))] +struct LocalChildReaperLease; + +#[cfg(unix)] +type LocalProcessSnapshot<'a> = Option<&'a crate::process_session::SessionProcessSnapshot>; + +#[cfg(not(unix))] +type LocalProcessSnapshot = (); + +#[cfg(unix)] +fn reserve_local_child_reaper() -> std::io::Result { + crate::process_session::reserve_child_reaper() +} + +#[cfg(not(unix))] +fn reserve_local_child_reaper() -> std::io::Result { + Ok(LocalChildReaperLease) +} + impl LocalPtyProcess { fn new(pid: Option, killer: Box) -> Arc { #[cfg(not(unix))] @@ -845,7 +867,7 @@ impl LocalPtyProcess { fn spawn_reaper( self: &Arc, mut child: Box, - reaper: crate::process_session::ReservedChildReaperLease, + reaper: LocalChildReaperLease, ) -> std::io::Result<()> { let process = self.clone(); std::thread::Builder::new().name("surface-child".into()).spawn(move || { @@ -913,6 +935,7 @@ impl LocalPtyProcess { } #[cfg(not(unix))] { + let _ = reaper; let _ = child.wait(); } *process.exited.0.lock().unwrap() = true; @@ -1058,7 +1081,11 @@ impl LocalPtyProcess { } fn terminate_and_wait(&self, deadline: Instant) -> bool { - self.terminate_and_wait_inner(deadline, None) + #[cfg(unix)] + let snapshot = None; + #[cfg(not(unix))] + let snapshot = (); + self.terminate_and_wait_inner(deadline, snapshot) } #[cfg(unix)] @@ -1073,7 +1100,8 @@ impl LocalPtyProcess { fn terminate_and_wait_inner( &self, deadline: Instant, - #[cfg(unix)] snapshot: Option<&crate::process_session::SessionProcessSnapshot>, + #[cfg(unix)] snapshot: LocalProcessSnapshot<'_>, + #[cfg(not(unix))] snapshot: LocalProcessSnapshot, ) -> bool { #[cfg(unix)] { @@ -1123,6 +1151,7 @@ impl LocalPtyProcess { } #[cfg(not(unix))] { + let _ = snapshot; let _ = self.request_hangup(); } self.wait_for_exit_until(deadline) @@ -1250,8 +1279,7 @@ impl Surface { return Self::spawn_hosted(id, opts, mux, attachment, true); } let _ = terminal_id; - let reaper = crate::process_session::reserve_child_reaper() - .context("reserve bounded PTY session cleanup")?; + let reaper = reserve_local_child_reaper().context("reserve bounded PTY session cleanup")?; let pty = native_pty_system().openpty(PtySize { rows: opts.rows, cols: opts.cols, From 97d47d22950fca1a37a6d17f73db3a76c82e0853 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:00:45 -0700 Subject: [PATCH 108/521] test(tui): expose daemon exit ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 136 +++++++++++++++++- .../src/terminal_host_runtime.rs | 22 +++ .../cmux-tui-core/src/workspace_registry.rs | 15 +- 3 files changed, 171 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 452fae5e4574..8fb2406e98a9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1530,6 +1530,8 @@ pub struct Mux { browser_runtime_connect: Mutex>>, #[cfg(test)] shutdown_owner_capacity: AtomicUsize, + #[cfg(test)] + shutdown_attempt_timeout: Mutex>, browser_runtime: BrowserRuntimeSlot, cell_pixels: Mutex<(u16, u16)>, default_colors: Mutex, @@ -1783,6 +1785,8 @@ impl Mux { browser_runtime_connect: Mutex::new(None), #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), + #[cfg(test)] + shutdown_attempt_timeout: Mutex::new(None), browser_runtime: BrowserRuntimeSlot::default(), cell_pixels: Mutex::new((8, 16)), default_colors: Mutex::new(DefaultColors::default()), @@ -4984,7 +4988,7 @@ impl Mux { self.browser_runtime.stop(); #[cfg(unix)] self.request_terminal_adoption_stop(); - let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + let deadline = Instant::now() + self.shutdown_attempt_timeout(); let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; let _ = self.surface_creations.stop_and_wait_until(deadline); let _ = self.async_surface_creations.stop_and_wait_until(deadline); @@ -5000,6 +5004,19 @@ impl Mux { } } + fn shutdown_attempt_timeout(&self) -> Duration { + #[cfg(test)] + if let Some(timeout) = *self.shutdown_attempt_timeout.lock().unwrap() { + return timeout; + } + crate::server::SERVER_SHUTDOWN_TIMEOUT + } + + #[cfg(test)] + fn set_shutdown_attempt_timeout_for_test(&self, timeout: Duration) { + *self.shutdown_attempt_timeout.lock().unwrap() = Some(timeout); + } + pub fn request_shutdown(&self) { self.shutdown_requested.store(true, Ordering::Release); let watchers = { @@ -12086,6 +12103,54 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn terminal_adoption_retries_when_the_registry_read_is_uncertain() { + use std::os::unix::fs::MetadataExt; + + let root = std::env::temp_dir().join(format!( + "cmux-adoption-registry-error-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let terminal_id = TerminalId::random().unwrap().to_hex(); + let record_path = root.join(format!("{terminal_id}.json")); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: terminal_id.clone(), + incarnation: crate::terminal_host::HostIncarnation::random().unwrap().to_hex(), + endpoint: format!( + "/tmp/cmux-th-{}/{}.sock", + std::fs::metadata(&root).unwrap().uid(), + terminal_id + ), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: String::new(), + supports_set_defaults: false, + supports_terminate_only: false, + }; + let task = TerminalAdoptionTask { + options: SurfaceOptions::default(), + record, + record_path, + next_attempt: Instant::now(), + delay: Duration::from_millis(100), + }; + let mux = Mux::new_for_test("adoption-registry-error", SurfaceOptions::default()); + mux.workspace_registry.lock().unwrap().set_terminal_record_read_failure_for_test(true); + + let complete = mux.try_terminal_adoption(&task); + + mux.workspace_registry.lock().unwrap().set_terminal_record_read_failure_for_test(false); + let _ = std::fs::remove_dir_all(root); + assert!( + !complete, + "an uncertain registry read was treated as proof that the host should be cleaned up" + ); + } + #[cfg(unix)] #[test] fn blocked_terminal_adoption_does_not_stall_an_unrelated_terminal() { @@ -12831,6 +12896,75 @@ mod tests { assert!(!mux.browser_runtime.has_runtime_for_test()); } + #[test] + fn daemon_exit_retries_until_async_browser_bootstrap_releases_ownership() { + let mux = test_mux(); + mux.set_shutdown_attempt_timeout_for_test(Duration::from_millis(25)); + let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); + let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); + let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); + *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ + move || { + bootstrap_reached_tx.send(()).unwrap(); + release_bootstrap_rx.lock().unwrap().recv().unwrap(); + } + })); + mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); + bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); + let shutdown = std::thread::spawn({ + let mux = mux.clone(); + move || { + mux.shutdown(); + shutdown_done_tx.send(()).unwrap(); + } + }); + let finished_early = shutdown_done_rx.recv_timeout(Duration::from_millis(100)).is_ok(); + release_bootstrap_tx.send(()).unwrap(); + if !finished_early { + shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + } + shutdown.join().unwrap(); + + assert!( + !finished_early, + "daemon exit completed while browser bootstrap still owned in-flight work" + ); + assert!(!mux.browser_runtime.has_runtime_for_test()); + } + + #[test] + fn daemon_exit_retains_active_local_owner_until_termination_succeeds() { + let mux = test_mux(); + mux.set_shutdown_attempt_timeout_for_test(Duration::from_millis(25)); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let (failing, attempts) = surface.set_recovering_server_shutdown_for_test(); + let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); + let shutdown = std::thread::spawn({ + let mux = mux.clone(); + move || { + mux.shutdown(); + shutdown_done_tx.send(()).unwrap(); + } + }); + let deadline = Instant::now() + Duration::from_secs(1); + while attempts.load(Ordering::Acquire) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let retained = mux.shutdown_owners.len(); + + failing.store(false, Ordering::Release); + shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + shutdown.join().unwrap(); + + assert_eq!( + retained, 1, + "daemon exit discarded the active local process owner after termination failed" + ); + assert!(mux.shutdown_owners.is_empty()); + } + #[test] fn browser_runtime_connection_does_not_hold_the_shared_slot() { let mux = Mux::new_for_test( diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 19b914156be1..e34acac3b353 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -319,6 +319,8 @@ mod unix { static NEXT_HOST_NORMAL_CLEANUP_FAILURES: AtomicUsize = AtomicUsize::new(0); #[cfg(test)] static NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES: AtomicUsize = AtomicUsize::new(0); + #[cfg(test)] + static HOST_PROCESS_INLINE_WAITS: AtomicUsize = AtomicUsize::new(0); struct HostProcessReaper { sender: Sender, @@ -520,6 +522,8 @@ mod unix { fn drop(&mut self) { if let Some(child) = self.child.as_mut() { let _ = child.kill(); + #[cfg(test)] + HOST_PROCESS_INLINE_WAITS.fetch_add(1, Ordering::AcqRel); let _ = child.wait(); } } @@ -4157,6 +4161,24 @@ mod unix { assert_eq!(std::io::Error::last_os_error().raw_os_error(), Some(libc::ECHILD)); } + #[test] + fn spawned_host_process_drop_transfers_waiting_to_the_shared_reaper() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + HOST_PROCESS_INLINE_WAITS.store(0, Ordering::Release); + let child = + Command::new("/bin/sh").arg("-c").arg("exit 0").spawn().expect("spawn test child"); + let (process, completed) = SpawnedHostProcess::new_for_test(child); + + drop(process); + + completed.recv_timeout(Duration::from_secs(1)).unwrap(); + assert_eq!( + HOST_PROCESS_INLINE_WAITS.load(Ordering::Acquire), + 0, + "dropping a launch guard waited for the child on the cancellation thread" + ); + } + #[test] fn stalled_host_handshake_is_time_bounded() { let (record_path, record, lease) = record_fixture("handshake-timeout"); diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index bdda06273783..70e5461f9992 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -8,7 +8,7 @@ use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; #[cfg(test)] -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use anyhow::Context; use fs4::FileExt; @@ -179,6 +179,8 @@ pub struct WorkspaceRegistry { _lease: Option, #[cfg(test)] terminal_snapshot_count: AtomicUsize, + #[cfg(test)] + fail_terminal_record_reads: AtomicBool, } impl std::fmt::Debug for WorkspaceRegistry { @@ -307,6 +309,8 @@ impl WorkspaceRegistry { _lease: lease, #[cfg(test)] terminal_snapshot_count: AtomicUsize::new(0), + #[cfg(test)] + fail_terminal_record_reads: AtomicBool::new(false), }) } @@ -403,10 +407,19 @@ impl WorkspaceRegistry { /// Includes tombstones and is intended for reconciliation and idempotent /// close handling, not frontend materialization. pub fn terminal_record(&self, terminal_id: &str) -> anyhow::Result> { + #[cfg(test)] + if self.fail_terminal_record_reads.load(Ordering::Relaxed) { + anyhow::bail!("forced terminal record read failure"); + } validate_terminal_identity("terminal id", terminal_id)?; read_terminal(&self.connection, terminal_id) } + #[cfg(test)] + pub(crate) fn set_terminal_record_read_failure_for_test(&self, enabled: bool) { + self.fail_terminal_record_reads.store(enabled, Ordering::Relaxed); + } + pub(crate) fn terminal_ids_including_tombstones(&self) -> anyhow::Result> { #[cfg(test)] self.terminal_snapshot_count.fetch_add(1, Ordering::Relaxed); From c2ee0c5a7fed9db16b0c1a7471c1c8da0b4d8034 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:03:19 -0700 Subject: [PATCH 109/521] fix(tui): retain ownership through daemon exit --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 153 ++++++++++++------ .../src/terminal_host_runtime.rs | 16 +- 2 files changed, 111 insertions(+), 58 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8fb2406e98a9..0d5b99115404 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -57,6 +57,7 @@ const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); const SHUTDOWN_FANOUT_WORKERS: usize = 32; const SHUTDOWN_RECONCILE_INITIAL_DELAY: Duration = Duration::from_millis(25); const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); +const SERVER_EXIT_RETRY_DELAY: Duration = Duration::from_millis(25); const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; #[cfg(unix)] const TERMINAL_ADOPTION_QUEUE_CAPACITY: usize = SHUTDOWN_OWNER_CAPACITY; @@ -2503,8 +2504,10 @@ impl Mux { #[cfg(unix)] fn try_terminal_adoption(self: &Arc, task: &TerminalAdoptionTask) -> bool { let terminal_id = &task.record.terminal_id; - let terminal = - self.workspace_registry.lock().unwrap().terminal_record(terminal_id).ok().flatten(); + let terminal = match self.workspace_registry.lock().unwrap().terminal_record(terminal_id) { + Ok(terminal) => terminal, + Err(_) => return false, + }; let Some(terminal) = terminal else { return cleanup_terminal_host_record(&task.record, &task.record_path); }; @@ -4983,24 +4986,113 @@ impl Mux { .collect() } - pub fn shutdown(&self) { + fn shutdown_browser_runtimes_until(&self, deadline: Instant) -> bool { + let retained = self.shutdown_owners.snapshot(); + let current_browser_runtime = self.browser_runtime.take_for_shutdown(); + let mut browser_runtimes = Vec::new(); + for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { + if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { + browser_runtimes.push(runtime.clone()); + } + } + if let Some(runtime) = ¤t_browser_runtime + && !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) + { + browser_runtimes.push(runtime.clone()); + } + + let mut failed = false; + for runtime in browser_runtimes { + let is_current = current_browser_runtime + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &runtime)); + let surface_failed = retained.iter().any(|(_, owner)| { + owner + .browser_runtime() + .is_some_and(|owner_runtime| Arc::ptr_eq(owner_runtime, &runtime)) + }); + if surface_failed && runtime.source() == BrowserSource::External { + if is_current { + self.browser_runtime.restore_for_shutdown(runtime); + } + continue; + } + if runtime.shutdown_until(deadline) { + if runtime.source() == BrowserSource::Launched { + self.shutdown_owners.remove_browser_runtime(&runtime); + } + } else { + failed = true; + if is_current { + self.browser_runtime.restore_for_shutdown(runtime); + } + } + } + failed + } + + fn shutdown_once_until(&self, deadline: Instant) -> anyhow::Result<()> { self.shutting_down.store(true, Ordering::Release); self.browser_runtime.stop(); #[cfg(unix)] self.request_terminal_adoption_stop(); - let deadline = Instant::now() + self.shutdown_attempt_timeout(); - let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { return }; - let _ = self.surface_creations.stop_and_wait_until(deadline); - let _ = self.async_surface_creations.stop_and_wait_until(deadline); #[cfg(unix)] - let _ = self.wait_for_terminal_adoption_workers_until(deadline); + crate::process_session::require_stable_process_signaling_until(deadline) + .context("preflight process control for daemon exit")?; + let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; + if !self.surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("surface creation remains active at the daemon exit deadline"); + } + if !self.async_surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("browser bootstrap remains active at the daemon exit deadline"); + } + #[cfg(unix)] + if !self.wait_for_terminal_adoption_workers_until(deadline) { + anyhow::bail!("terminal adoption remains active at the daemon exit deadline"); + } let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); + for surface in &surfaces { + match surface.shutdown_owner_identity() { + Some(SurfaceShutdownOwnerIdentity::Surface) => { + anyhow::ensure!( + self.shutdown_owners.stage_surface(surface).is_some(), + "surface {} lost its shutdown owner during daemon exit", + surface.id + ); + } + #[cfg(unix)] + Some(SurfaceShutdownOwnerIdentity::Hosted { .. }) | None => {} + #[cfg(not(unix))] + None => {} + } + } for surface in &surfaces { surface.disconnect_for_daemon_shutdown(); } self.terminate_staged_shutdown_owners_until(deadline); - if let Some(runtime) = self.browser_runtime.take_for_shutdown() { - let _ = runtime.shutdown_until(deadline); + let browser_runtime_failed = self.shutdown_browser_runtimes_until(deadline); + let retained = self.shutdown_owners.len(); + if retained != 0 || browser_runtime_failed { + anyhow::bail!( + "daemon exit retained {retained} surface owner(s); browser runtime failure: \ + {browser_runtime_failed}" + ); + } + Ok(()) + } + + /// Finish every fallible cleanup phase before allowing the server process + /// to release its socket. Each attempt is bounded, but the process keeps + /// exact ownership and retries until all local work has stopped or moved + /// to its durable external host. + pub fn shutdown(&self) { + self.shutdown_owner_reconciler.stop(); + loop { + let deadline = Instant::now() + self.shutdown_attempt_timeout(); + if self.shutdown_once_until(deadline).is_ok() { + return; + } + std::thread::sleep(SERVER_EXIT_RETRY_DELAY); } } @@ -5223,46 +5315,7 @@ impl Mux { let _ = self.shutdown_owners.stage(key, owner); } self.terminate_staged_shutdown_owners_until(deadline); - let retained = self.shutdown_owners.snapshot(); - let current_browser_runtime = self.browser_runtime.take_for_shutdown(); - let mut browser_runtimes = Vec::new(); - for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { - if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { - browser_runtimes.push(runtime.clone()); - } - } - if let Some(runtime) = ¤t_browser_runtime - && !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) - { - browser_runtimes.push(runtime.clone()); - } - let mut browser_runtime_failed = false; - for runtime in browser_runtimes { - let is_current = current_browser_runtime - .as_ref() - .is_some_and(|current| Arc::ptr_eq(current, &runtime)); - let surface_failed = retained.iter().any(|(_, owner)| { - owner - .browser_runtime() - .is_some_and(|owner_runtime| Arc::ptr_eq(owner_runtime, &runtime)) - }); - if surface_failed && runtime.source() == BrowserSource::External { - if is_current { - self.browser_runtime.restore_for_shutdown(runtime); - } - continue; - } - if runtime.shutdown_until(deadline) { - if runtime.source() == BrowserSource::Launched { - self.shutdown_owners.remove_browser_runtime(&runtime); - } - } else { - browser_runtime_failed = true; - if is_current { - self.browser_runtime.restore_for_shutdown(runtime); - } - } - } + let browser_runtime_failed = self.shutdown_browser_runtimes_until(deadline); let retained = self.shutdown_owners.snapshot(); let mut failed_surface_ids = retained diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index e34acac3b353..524ffe3833ad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -520,12 +520,11 @@ mod unix { impl Drop for SpawnedHostProcess { fn drop(&mut self) { - if let Some(child) = self.child.as_mut() { - let _ = child.kill(); - #[cfg(test)] - HOST_PROCESS_INLINE_WAITS.fetch_add(1, Ordering::AcqRel); - let _ = child.wait(); - } + let Some(mut child) = self.child.take() else { return }; + let _ = child.kill(); + let reaper = + self.reaper.take().expect("spawned terminal host retains its reaper reservation"); + enqueue_host_process_reaper(reaper, child); } } @@ -916,8 +915,9 @@ mod unix { } /// Commit the launch ownership handoff after every fallible Surface - /// setup step succeeds. Until then, dropping this attachment exact- - /// kills and waits the child process through SpawnedHostProcess. + /// setup step succeeds. Until then, dropping this attachment asks the + /// host to exit, exact-kills it after the bounded rollback window, and + /// transfers the remaining wait to the shared process reaper. pub(crate) fn commit_launched_host(&mut self) { let Some(process) = self.launch_process.take() else { return }; process.detach_reaper(); From 76ddd33e87e96f898f8c813c0464088b3d2b1778 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:41:10 -0700 Subject: [PATCH 110/521] test(tui): expose adoption and uv replay gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 67 ++++++++++++++++++++++ cmux-tui/dist/scripts/test_package_pypi.py | 12 ++++ 2 files changed, 79 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0d5b99115404..b3156e6094e9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12204,6 +12204,73 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn terminal_adoption_retries_when_the_running_transition_write_fails() { + let mux = test_mux(); + let workspace = + mux.create_empty_workspace(Some("adoption-write-retry".into()), None, None).unwrap(); + let terminal_id = TerminalId::random().unwrap().to_hex(); + let incarnation = crate::terminal_host::HostIncarnation::random().unwrap().to_hex(); + { + let mut registry = mux.workspace_registry.lock().unwrap(); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + "adoption-write-retry-reserve", + &RegistryTerminal { + terminal_id: terminal_id.clone(), + workspace_key: workspace.key, + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({}), + exit: None, + }, + ) + .unwrap(); + commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "adoption-write-retry-running", + &terminal_id, + TerminalLifecycle::Running, + Some(&incarnation), + None, + ) + .unwrap(); + } + let task = TerminalAdoptionTask { + options: SurfaceOptions::default(), + record: crate::terminal_host_runtime::TerminalHostRecord { + record_version: 2, + terminal_id: terminal_id.clone(), + incarnation, + endpoint: String::new(), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 1, + host_start_nonce: "01".repeat(32), + workspace_key: String::new(), + supports_set_defaults: true, + supports_terminate_only: true, + }, + record_path: std::path::PathBuf::new(), + next_attempt: Instant::now(), + delay: Duration::from_millis(100), + }; + mux.set_terminal_close_failure_for_test(true).unwrap(); + + let complete = mux.try_terminal_adoption(&task); + + mux.set_terminal_close_failure_for_test(false).unwrap(); + let terminal = + mux.workspace_registry.lock().unwrap().terminal_record(&terminal_id).unwrap().unwrap(); + assert_eq!(terminal.lifecycle, TerminalLifecycle::Running); + assert!( + !complete, + "a failed Running-to-Adopting write permanently completed the adoption task" + ); + } + #[cfg(unix)] #[test] fn blocked_terminal_adoption_does_not_stall_an_unrelated_terminal() { diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index b32c1a714e2b..4fe16f801731 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -3,7 +3,9 @@ from __future__ import annotations +import os import unittest +from unittest.mock import patch from package_pypi import wheel_bytes @@ -26,6 +28,16 @@ def test_uv_cache_restarts_the_same_version(self) -> None: "uvx cmux==1.2.3", ) + def test_custom_uv_cache_restarts_the_same_version(self) -> None: + with patch.dict(os.environ, {"UV_CACHE_DIR": "/tmp/uv-cache"}): + self.assertEqual( + launcher_command( + "1.2.3", + "/tmp/uv-cache/archive-v0/abcdefgh/bin/cmux", + ), + "uvx cmux==1.2.3", + ) + def test_pipx_run_cache_restarts_the_same_version(self) -> None: self.assertEqual( launcher_command( From c2094659e9eec420379b0ba84ab7bf487edee432 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:45:50 -0700 Subject: [PATCH 111/521] fix(tui): retry authoritative adoption handoffs --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 82 +++++++++++++++++------- cmux-tui/dist/scripts/package_pypi.py | 15 +++++ 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b3156e6094e9..7a2ef5c5466f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2511,31 +2511,38 @@ impl Mux { let Some(terminal) = terminal else { return cleanup_terminal_host_record(&task.record, &task.record_path); }; - if matches!(terminal.lifecycle, TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned) { - if terminal.lifecycle == TerminalLifecycle::Exited { - let _ = self.materialize_exited_terminal(terminal_id, &task.options); + if terminal.lifecycle == TerminalLifecycle::Exited { + if self.materialize_exited_terminal(terminal_id, &task.options).is_err() { + return false; } return cleanup_terminal_host_record(&task.record, &task.record_path); } + if terminal.lifecycle == TerminalLifecycle::Tombstoned { + return cleanup_terminal_host_record(&task.record, &task.record_path); + } if terminal .incarnation .as_deref() .is_some_and(|incarnation| incarnation != task.record.incarnation) { - let _ = self.mark_terminal_exited_and_materialize( - terminal_id, - "terminal-incarnation-mismatch", - "host-incarnation-mismatch", - &task.options, - ); + if self + .mark_terminal_exited_and_materialize( + terminal_id, + "terminal-incarnation-mismatch", + "host-incarnation-mismatch", + &task.options, + ) + .is_err() + { + return false; + } return cleanup_terminal_host_record(&task.record, &task.record_path); } if terminal.lifecycle == TerminalLifecycle::Running { - let already_live = self - .resolve_terminal(terminal_id) - .ok() - .flatten() - .is_some_and(|resolution| resolution.surface.is_some()); + let already_live = match self.resolve_terminal(terminal_id) { + Ok(resolution) => resolution.is_some_and(|resolution| resolution.surface.is_some()), + Err(_) => return false, + }; if already_live { return true; } @@ -2550,7 +2557,37 @@ impl Mux { ) .is_err() { - return true; + let current = + match self.workspace_registry.lock().unwrap().terminal_record(terminal_id) { + Ok(Some(current)) => current, + Ok(None) | Err(_) => return false, + }; + if current.lifecycle == TerminalLifecycle::Exited { + if self.materialize_exited_terminal(terminal_id, &task.options).is_err() { + return false; + } + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if current.lifecycle == TerminalLifecycle::Tombstoned { + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if current.incarnation.as_deref() != Some(task.record.incarnation.as_str()) { + return false; + } + match current.lifecycle { + TerminalLifecycle::Adopting => {} + TerminalLifecycle::Running => { + return match self.resolve_terminal(terminal_id) { + Ok(resolution) => { + resolution.is_some_and(|resolution| resolution.surface.is_some()) + } + Err(_) => false, + }; + } + TerminalLifecycle::Launching + | TerminalLifecycle::Exited + | TerminalLifecycle::Tombstoned => return false, + } } } if terminal_host_record_liveness(&task.record_path, &task.record) @@ -2560,13 +2597,14 @@ impl Mux { &task.record_path, &task.record, ); - let _ = self.mark_terminal_exited_and_materialize( - terminal_id, - "terminal-host-proven-dead", - "host-process-ended-before-adoption", - &task.options, - ); - return true; + return self + .mark_terminal_exited_and_materialize( + terminal_id, + "terminal-host-proven-dead", + "host-process-ended-before-adoption", + &task.options, + ) + .is_ok(); } let Ok(_creation) = self.begin_surface_creation() else { return true; diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index ec91f268e8a7..c4e90c6fabde 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -111,6 +111,21 @@ def wheel_bytes(version: str, tag: str, binary: bytes) -> list[tuple[str, bytes, def _launcher_command(argv0: str) -> str: parts = pathlib.PurePath(argv0).parts + uv_cache = os.environ.get("UV_CACHE_DIR") + if uv_cache and argv0: + executable = pathlib.PurePath(os.path.abspath(os.path.expanduser(argv0))) + cache = pathlib.PurePath(os.path.abspath(os.path.expanduser(uv_cache))) + try: + relative = executable.relative_to(cache) + except ValueError: + pass + else: + if any( + candidate.startswith("archive-v") + and candidate.removeprefix("archive-v").isdigit() + for candidate in relative.parts + ): + return "uvx cmux==" + PACKAGE_VERSION for index, part in enumerate(parts): if part != "uv": continue From ca39a6a5172440bb21c2b7add8fb681e7ebb65a1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:21:05 -0700 Subject: [PATCH 112/521] test(tui): expose unbounded daemon cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 28 +++++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 33 +++++++++++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 45 +++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7a2ef5c5466f..5af1c2f8fb88 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -13123,6 +13123,34 @@ mod tests { assert!(mux.shutdown_owners.is_empty()); } + #[test] + fn daemon_exit_bounds_permanent_cleanup_failures() { + let mux = test_mux(); + mux.set_shutdown_attempt_timeout_for_test(Duration::from_millis(25)); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let (failing, _attempts) = surface.set_recovering_server_shutdown_for_test(); + let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); + let shutdown = std::thread::spawn({ + let mux = mux.clone(); + move || { + shutdown_done_tx.send(mux.shutdown()).unwrap(); + } + }); + + let completed_before_deadline = + shutdown_done_rx.recv_timeout(Duration::from_millis(400)).is_ok(); + failing.store(false, Ordering::Release); + if !completed_before_deadline { + shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + } + shutdown.join().unwrap(); + + assert!( + completed_before_deadline, + "daemon exit retried a permanent cleanup failure without a total deadline" + ); + } + #[test] fn browser_runtime_connection_does_not_hold_the_shared_slot() { let mux = Mux::new_for_test( diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 743268530e6b..46a08c926485 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -5666,6 +5666,39 @@ mod tests { assert_eq!(response["data"]["generation"], generation); } + #[cfg(unix)] + #[test] + fn daemon_shutdown_rejects_unavailable_process_control_before_ack() { + let mux = test_mux(); + let outbound = Arc::new(BoundedOutbound::default()); + let writer = MessageWriter::new(QueuedSink { outbound: outbound.clone(), control: None }); + let local = mux.control_clients.register(ClientTransport::Unix, writer.clone()); + let (_, generation) = mux.registry_identity(); + crate::process_session::set_process_session_preflight_failure_for_test(true); + + assert!(handle_message( + &mux, + local, + &json!({ + "id": 95, + "cmd": "shutdown-daemon", + "pid": std::process::id(), + "generation": generation, + }) + .to_string(), + &writer, + )); + + crate::process_session::set_process_session_preflight_failure_for_test(false); + let response: Value = serde_json::from_str(&outbound.try_pop().unwrap()).unwrap(); + assert_eq!(response["ok"], false); + assert!(!mux.daemon_shutdown_requested()); + assert!( + !mux.daemon_handoff_pending.load(Ordering::Acquire), + "failed preflight left daemon handoff admission fenced" + ); + } + #[test] fn daemon_shutdown_atomically_fences_native_browser_ownership() { let owned = test_mux(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index adb80edf8c1b..36c3332ad237 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1206,6 +1206,51 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { fs::remove_dir_all(dir).unwrap(); } +#[cfg(unix)] +#[test] +fn daemon_handoff_exits_when_the_interactive_driver_cannot_progress() { + let dir = unique_temp_dir("daemon-handoff-blocked-interactive-driver"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let state = dir.join("state"); + let mut server = PtyChild::start_with_env( + &["--socket", socket.to_str().unwrap(), "--state", state.to_str().unwrap()], + &[ + ("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER", std::ffi::OsStr::new("1")), + ("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS", std::ffi::OsStr::new("100")), + ], + ); + wait_for_socket_path(&socket); + let identity = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "identify"})); + + let response = json_socket_request( + &socket, + serde_json::json!({ + "id": 2, + "cmd": "shutdown-daemon", + "pid": identity["pid"], + "generation": identity["generation"], + }), + ); + assert_eq!(response["accepted"], true); + + let deadline = Instant::now() + Duration::from_secs(3); + let status = loop { + if let Some(status) = server.child.try_wait().unwrap() { + break status; + } + assert!( + Instant::now() < deadline, + "server remained alive after acknowledging daemon handoff" + ); + std::thread::sleep(Duration::from_millis(10)); + }; + assert!(status.success(), "server exited with {status}"); + assert!(transport::connect(&socket).is_err()); + drop(server); + fs::remove_dir_all(dir).unwrap(); +} + #[cfg(unix)] #[test] fn server_stop_cancels_a_blocked_terminal_host_launch() { From 5aba3dc7dfc40d1d9510bac59fdd525b4eab79f4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 03:32:59 -0700 Subject: [PATCH 113/521] fix(tui): bound accepted daemon handoffs --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 115 ++++++++++++------ .../cmux-tui-core/tests/browser_runtime.rs | 14 +-- cmux-tui/crates/cmux-tui-core/tests/pty.rs | 4 +- .../tests/websocket_transport.rs | 18 +-- cmux-tui/crates/cmux-tui/src/config.rs | 2 +- cmux-tui/crates/cmux-tui/src/main.rs | 19 +-- 6 files changed, 107 insertions(+), 65 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5af1c2f8fb88..24f70f00e910 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -57,7 +57,10 @@ const SHUTDOWN_TERMINATION_TIMEOUT: Duration = Duration::from_millis(100); const SHUTDOWN_FANOUT_WORKERS: usize = 32; const SHUTDOWN_RECONCILE_INITIAL_DELAY: Duration = Duration::from_millis(25); const SHUTDOWN_RECONCILE_MAX_DELAY: Duration = Duration::from_secs(1); -const SERVER_EXIT_RETRY_DELAY: Duration = Duration::from_millis(25); +const SERVER_EXIT_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(25); +const SERVER_EXIT_RETRY_MAX_DELAY: Duration = Duration::from_secs(1); +#[cfg(test)] +const SERVER_EXIT_TEST_ATTEMPT_BUDGET: u32 = 8; const SHUTDOWN_OWNER_CAPACITY: usize = 4_096; #[cfg(unix)] const TERMINAL_ADOPTION_QUEUE_CAPACITY: usize = SHUTDOWN_OWNER_CAPACITY; @@ -1120,7 +1123,7 @@ impl ShutdownRequestWatchInner { } } -/// One-shot notification for a successful server-shutdown request. +/// One-shot notification that the server process has accepted an exit request. #[derive(Clone)] pub struct ShutdownRequestWatch { inner: Arc, @@ -5120,18 +5123,33 @@ impl Mux { } /// Finish every fallible cleanup phase before allowing the server process - /// to release its socket. Each attempt is bounded, but the process keeps - /// exact ownership and retries until all local work has stopped or moved - /// to its durable external host. - pub fn shutdown(&self) { + /// to release its socket. Retried work keeps exact ownership, while one + /// absolute deadline guarantees an accepted daemon handoff cannot wedge + /// the old process or its socket forever. + pub fn shutdown(&self) -> anyhow::Result<()> { self.shutdown_owner_reconciler.stop(); + let overall_deadline = Instant::now() + self.shutdown_total_timeout(); + let mut retry_delay = SERVER_EXIT_RETRY_INITIAL_DELAY; + let mut last_error = None; loop { - let deadline = Instant::now() + self.shutdown_attempt_timeout(); - if self.shutdown_once_until(deadline).is_ok() { - return; + let now = Instant::now(); + if now >= overall_deadline { + break; + } + let attempt_deadline = (now + self.shutdown_attempt_timeout()).min(overall_deadline); + match self.shutdown_once_until(attempt_deadline) { + Ok(()) => return Ok(()), + Err(error) => last_error = Some(error), } - std::thread::sleep(SERVER_EXIT_RETRY_DELAY); + let Some(remaining) = overall_deadline.checked_duration_since(Instant::now()) else { + break; + }; + std::thread::sleep(retry_delay.min(remaining)); + retry_delay = retry_delay.saturating_mul(2).min(SERVER_EXIT_RETRY_MAX_DELAY); } + let error = + last_error.unwrap_or_else(|| anyhow::anyhow!("daemon cleanup deadline expired")); + Err(error.context("daemon cleanup did not finish before the process exit deadline")) } fn shutdown_attempt_timeout(&self) -> Duration { @@ -5142,6 +5160,14 @@ impl Mux { crate::server::SERVER_SHUTDOWN_TIMEOUT } + fn shutdown_total_timeout(&self) -> Duration { + #[cfg(test)] + if let Some(timeout) = *self.shutdown_attempt_timeout.lock().unwrap() { + return timeout.saturating_mul(SERVER_EXIT_TEST_ATTEMPT_BUDGET); + } + crate::server::SERVER_SHUTDOWN_TIMEOUT + } + #[cfg(test)] fn set_shutdown_attempt_timeout_for_test(&self, timeout: Duration) { *self.shutdown_attempt_timeout.lock().unwrap() = Some(timeout); @@ -5413,7 +5439,20 @@ impl Mux { /// browser owns this mux. New owner announcements are rejected until the /// response is queued or the reservation is cancelled. pub fn begin_daemon_handoff(&self, requesting_client: u64) -> anyhow::Result<()> { - self.control_clients.begin_daemon_handoff(requesting_client, &self.daemon_handoff_pending) + self.control_clients + .begin_daemon_handoff(requesting_client, &self.daemon_handoff_pending)?; + let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + let preflight = (|| { + #[cfg(unix)] + crate::process_session::require_stable_process_signaling_until(deadline) + .context("preflight process control for daemon handoff")?; + let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; + Ok(()) + })(); + if preflight.is_err() { + self.cancel_daemon_handoff(); + } + preflight } pub fn cancel_daemon_handoff(&self) { @@ -5431,6 +5470,7 @@ impl Mux { self.daemon_shutdown_requested.store(true, Ordering::Release); #[cfg(unix)] self.request_terminal_adoption_stop(); + self.request_shutdown(); } pub fn daemon_shutdown_requested(&self) -> bool { @@ -10955,7 +10995,7 @@ mod tests { assert!(!closed_early, "workspace closed before layout commit"); assert!(applied.unwrap().is_ok()); assert_eq!(close_result.unwrap(), Some(2)); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -12401,7 +12441,7 @@ mod tests { while !mux.terminal_adoptions.lock().unwrap().is_empty() && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); } - mux.shutdown(); + mux.shutdown().unwrap(); let _ = std::fs::remove_dir_all(root); assert!( @@ -13074,7 +13114,7 @@ mod tests { let shutdown = std::thread::spawn({ let mux = mux.clone(); move || { - mux.shutdown(); + mux.shutdown().unwrap(); shutdown_done_tx.send(()).unwrap(); } }); @@ -13102,7 +13142,7 @@ mod tests { let shutdown = std::thread::spawn({ let mux = mux.clone(); move || { - mux.shutdown(); + mux.shutdown().unwrap(); shutdown_done_tx.send(()).unwrap(); } }); @@ -13130,18 +13170,15 @@ mod tests { let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let (failing, _attempts) = surface.set_recovering_server_shutdown_for_test(); let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); - let shutdown = std::thread::spawn({ - let mux = mux.clone(); - move || { - shutdown_done_tx.send(mux.shutdown()).unwrap(); - } + let shutdown = std::thread::spawn(move || { + shutdown_done_tx.send(mux.shutdown()).unwrap(); }); let completed_before_deadline = shutdown_done_rx.recv_timeout(Duration::from_millis(400)).is_ok(); failing.store(false, Ordering::Release); if !completed_before_deadline { - shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let _ = shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); } shutdown.join().unwrap(); @@ -13184,7 +13221,7 @@ mod tests { std::thread::sleep(Duration::from_millis(10)); } mux.request_daemon_shutdown(); - mux.shutdown(); + mux.shutdown().unwrap(); assert!( slot_available, @@ -13591,7 +13628,7 @@ mod tests { }) .collect::>(); assert!(layouts.is_empty()); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -14161,7 +14198,7 @@ mod tests { .unwrap(), Some(2) ); - mux.shutdown(); + mux.shutdown().unwrap(); } let recovered = Mux::open_persistent_provider_managed( @@ -14179,7 +14216,7 @@ mod tests { }); let error = recovered.close_workspace_at_revision(1, None).unwrap_err(); assert!(error.to_string().contains("provider-managed workspace directly")); - recovered.shutdown(); + recovered.shutdown().unwrap(); drop(recovered); std::fs::remove_dir_all(root).unwrap(); } @@ -14243,7 +14280,7 @@ mod tests { assert_eq!(resolved.terminal.lifecycle, TerminalLifecycle::Exited); assert_eq!(resolved.terminal.exit.unwrap()["reason"], "missing-host-record"); assert_eq!(resolved.terminal_revision, 2); - mux.shutdown(); + mux.shutdown().unwrap(); drop(mux); std::fs::remove_dir_all(root).unwrap(); } @@ -14330,14 +14367,14 @@ mod tests { let closed = mux.resolve_terminal(TERMINAL).unwrap().unwrap(); assert_eq!(closed.surface, None); assert_eq!(closed.terminal.lifecycle, TerminalLifecycle::Tombstoned); - mux.shutdown(); + mux.shutdown().unwrap(); drop(mux); let reopened = Mux::open_persistent("recover-exited", options, &root).unwrap(); let closed = reopened.resolve_terminal(TERMINAL).unwrap().unwrap(); assert_eq!(closed.surface, None); assert_eq!(closed.terminal.lifecycle, TerminalLifecycle::Tombstoned); - reopened.shutdown(); + reopened.shutdown().unwrap(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); } @@ -14920,7 +14957,7 @@ mod tests { assert_eq!(second_surface.spawn_cwd().as_deref(), Some("/tmp")); *mux.terminal_create_after_empty_check.lock().unwrap() = None; *mux.terminal_create_after_materialization_lock.lock().unwrap() = None; - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15000,7 +15037,7 @@ mod tests { assert!(replay.replayed); *mux.terminal_create_after_workspace_reservation.lock().unwrap() = None; surface.kill(); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15067,7 +15104,7 @@ mod tests { *mux.terminal_create_after_workspace_reservation.lock().unwrap() = None; initial.kill(); created.kill(); - mux.shutdown(); + mux.shutdown().unwrap(); } } @@ -15158,7 +15195,7 @@ mod tests { Some((replacement.workspace, key, replacement.revision + 1)) ); surface.kill(); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15219,7 +15256,7 @@ mod tests { assert!(!mux.close_workspace(managed.workspace)); *mux.workspace_close_after_selector_resolution.lock().unwrap() = None; - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15291,7 +15328,7 @@ mod tests { assert_eq!(state.workspaces[0].screens.len(), 1); assert_eq!(state.workspace_revision, 1); }); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15332,7 +15369,7 @@ mod tests { .expect_err("duplicate stable key must fail"); assert!(duplicate.to_string().contains("already exists")); mux.with_state(|state| assert_eq!(state.workspaces.len(), 1)); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15348,7 +15385,7 @@ mod tests { assert_eq!(state.pane_of(surface.id), Some(state.workspaces[0].screens[0].active_pane)); assert_eq!(state.workspace_revision, 1); }); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15377,7 +15414,7 @@ mod tests { let pane = workspace.screens[0].active_pane; assert_eq!(state.panes[&pane].tabs.len(), surfaces.len()); }); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15457,7 +15494,7 @@ mod tests { assert_eq!(state.pane_of(browser.id), Some(pane)); assert_eq!(state.pane_of(terminal.surface), Some(pane)); }); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -15604,7 +15641,7 @@ mod tests { wait_for_text(&surface, "authority-test:after"); assert_eq!(surface.process_id(), process_id); assert!(!surface.is_dead()); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 42456fde472d..072f46c00828 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -589,7 +589,7 @@ fn socket_browser_attach_streams_frames_input_and_cell_pixels() { assert_eq!(failed, "net::ERR_NAME_NOT_RESOLVED"); mux.close_surface(surface).unwrap(); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); server.join().unwrap(); } @@ -726,7 +726,7 @@ fn wedged_browser_navigate_does_not_block_same_socket_connection() { "wedged browser close blocked for {:?}", close_started.elapsed() ); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); server.join().unwrap(); } @@ -857,7 +857,7 @@ fn queued_back_and_forward_do_not_collapse_while_worker_is_blocked() { ); mux.close_surface(surface).unwrap(); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); server.join().unwrap(); } @@ -974,7 +974,7 @@ fn control_command_reports_backpressure_when_worker_queue_is_full() { ); mux.close_surface(surface).unwrap(); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); server.join().unwrap(); } @@ -1051,7 +1051,7 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { assert_eq!(mouse["params"]["x"], 50.0); assert_eq!(mouse["params"]["y"], 50.0); - mux.shutdown(); + mux.shutdown().unwrap(); server.join().unwrap(); } @@ -1131,7 +1131,7 @@ fn stalled_external_browser_nudges_target_once_before_interaction() { assert_eq!(second_mouse["method"], "Input.dispatchMouseEvent"); assert_eq!(second_mouse["params"]["x"], 13.0); - mux.shutdown(); + mux.shutdown().unwrap(); server.join().unwrap(); } @@ -1174,5 +1174,5 @@ fn browser_tab_creation_is_async_and_surfaces_bootstrap_failure() { || status.contains("timed out"), "{status}" ); - mux.shutdown(); + mux.shutdown().unwrap(); } diff --git a/cmux-tui/crates/cmux-tui-core/tests/pty.rs b/cmux-tui/crates/cmux-tui-core/tests/pty.rs index 169cb8c850da..f06417950315 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/pty.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/pty.rs @@ -225,7 +225,7 @@ fn headless_creation_uses_legacy_default_then_latest_client_size() { .unwrap(); assert_vt_state_size(&mut writer, &mut reader, 11, clamped, (1, 1)); - mux.shutdown(); + mux.shutdown().unwrap(); cmux_tui_core::server::cleanup(&sock_path); } @@ -247,7 +247,7 @@ fn shared_surface_uses_smallest_viewer_size_until_that_viewer_detaches() { mux.remove_surface_size_client(surface, 1); assert_eq!(mux.surface(surface).unwrap().size(), (120, 40)); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/tests/websocket_transport.rs b/cmux-tui/crates/cmux-tui-core/tests/websocket_transport.rs index a8f8b5eafdb2..c3c2a465faab 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/websocket_transport.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/websocket_transport.rs @@ -90,7 +90,7 @@ fn websocket_server_allows_pairing_without_a_static_token() { assert!(server.is_ok(), "WebSocket listener rejected pairing mode"); } - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -104,7 +104,7 @@ fn websocket_server_rejects_tokens_that_cannot_fit_the_auth_limit() { ); assert!(result.is_err(), "WebSocket listener accepted an unusably large token"); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -125,7 +125,7 @@ fn websocket_rejects_oversized_authentication_frames() { "oversized pre-authentication frame remained accepted" ); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -161,7 +161,7 @@ fn websocket_auth_accepts_exact_preamble_and_rejects_missing_or_wrong_tokens() { assert_eq!(identify["data"]["protocol"], server::PROTOCOL_VERSION); assert_eq!(identify["data"]["session"], "ws-auth"); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -204,7 +204,7 @@ fn websocket_pairing_is_approved_over_trusted_unix_and_credential_reconnects() { send_json(&mut reconnect, json!({"id": 4, "cmd": "identify"})); assert_eq!(read_until(&mut reconnect, |value| value["id"] == 4)["ok"], true); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); } @@ -265,7 +265,7 @@ fn websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect() { assert_eq!(identify["ok"], true); assert_eq!(identify["data"]["protocol"], server::PROTOCOL_VERSION); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -322,7 +322,7 @@ fn websocket_subscriber_receives_cross_connection_event_without_poll_delay() { ); assert_eq!(read_until(&mut trigger, |value| value["id"] == 3)["ok"], true); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] @@ -449,7 +449,7 @@ fn clients_list_identify_resize_and_detach_across_transports() { let mut eof = String::new(); assert_eq!(unix_reader.read_line(&mut eof).unwrap(), 0); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); } @@ -480,5 +480,5 @@ fn websocket_non_loopback_bind_requires_and_accepts_explicit_insecure_opt_in() { assert_eq!(identify["ok"], true); assert_eq!(identify["data"]["protocol"], server::PROTOCOL_VERSION); - mux.shutdown(); + mux.shutdown().unwrap(); } diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 57ffc6a9e4a1..48d7823f7db4 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -2527,7 +2527,7 @@ mod tests { assert_eq!(colors.selection_fg, Some(Rgb { r: 0xfe, g: 0xfe, b: 0xfe })); mux.close_surface(surface.id).unwrap(); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket); } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 74ea3a331710..dc30e768ff77 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -59,7 +59,11 @@ static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); #[cfg(unix)] const MACHINE_PROVIDER_TOKEN_ENV: &str = "CMUX_MACHINE_PROVIDER_TOKEN"; const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORITY"; -const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(5); +// One second beyond the core cleanup budget lets `run_server` report a +// bounded cleanup error before the last-resort watchdog exits the process. +const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = + cmux_tui_core::server::SERVER_SHUTDOWN_TIMEOUT + .saturating_add(std::time::Duration::from_secs(1)); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -964,11 +968,11 @@ impl ServerProcessShutdownGuard { completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) ) { - // A successful shutdown request drains every surface and - // attempts its confirmed response before setting - // shutdown_requested. If the interactive driver cannot - // return after that boundary, it owns no remaining work - // that can justify retaining the process or control socket. + // Full shutdown drains every surface before requesting + // exit. Daemon handoff fences new work and gives normal + // cleanup the same bounded grace. If the interactive + // driver cannot return after either accepted boundary, + // the old process must release its control socket. cmux_tui_core::server::cleanup(&worker_socket_path); std::process::exit(0); } @@ -1161,8 +1165,9 @@ fn run_server( run_tui(Session::Local(mux.clone()), args.session, None) }; drop(websocket_server); - mux.shutdown(); + let shutdown_result = mux.shutdown(); server_process.complete(); + shutdown_result?; result } From b7e84cdfb290ac74a67e169c001906766d1baf63 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:15:45 -0700 Subject: [PATCH 114/521] test(tui): expose lifecycle ownership leaks --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 62 ++++++++++++++ .../cmux-tui-core/src/process_session.rs | 42 ++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 36 ++++++-- .../src/terminal_host_runtime.rs | 84 +++++++++++++++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 27 ++++-- 5 files changed, 237 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index ea7f3d13fafd..c095a8d08384 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -62,10 +62,22 @@ pub mod transport { use super::Stream; + #[cfg(test)] + type SocketCreatedHook = std::sync::Arc; + #[cfg(test)] + static SOCKET_CREATED_HOOK: std::sync::OnceLock< + std::sync::Mutex>, + > = std::sync::OnceLock::new(); + pub(super) struct Listener { inner: UnixListener, } + #[cfg(test)] + pub(super) fn set_socket_created_hook(hook: Option) { + *SOCKET_CREATED_HOOK.get_or_init(Default::default).lock().unwrap() = hook; + } + pub(super) fn listen(path: &Path) -> io::Result { UnixListener::bind(path).map(|inner| Listener { inner }) } @@ -90,6 +102,12 @@ pub mod transport { // SAFETY: descriptor is a fresh successful socket result and this // OwnedFd takes its sole ownership. let descriptor = unsafe { OwnedFd::from_raw_fd(descriptor) }; + #[cfg(test)] + if let Some(hook) = + SOCKET_CREATED_HOOK.get_or_init(Default::default).lock().unwrap().clone() + { + hook(descriptor.as_raw_fd()); + } // SAFETY: F_GETFD only reads flags from this valid descriptor. let descriptor_flags = unsafe { libc::fcntl(descriptor.as_raw_fd(), libc::F_GETFD) }; if descriptor_flags < 0 { @@ -619,6 +637,9 @@ pub mod transport { mod tests { use super::*; + #[cfg(unix)] + static SOCKET_CREATED_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + #[test] fn expired_connect_deadline_fails_before_socket_resolution() { let error = connect_until( @@ -652,6 +673,47 @@ pub mod transport { std::fs::remove_file(path).unwrap(); } + #[cfg(unix)] + #[test] + fn deadline_connect_is_close_on_exec_at_socket_creation() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let _guard = SOCKET_CREATED_TEST_LOCK.lock().unwrap(); + let path = std::env::temp_dir().join(format!( + "cmux-platform-cloexec-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let close_on_exec = Arc::new(AtomicBool::new(false)); + imp::set_socket_created_hook(Some(Arc::new({ + let close_on_exec = close_on_exec.clone(); + move |descriptor| { + // SAFETY: the hook receives the fresh live socket + // descriptor before connect_until performs any fallback. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFD) }; + close_on_exec + .store(flags >= 0 && flags & libc::FD_CLOEXEC != 0, Ordering::Release); + } + }))); + + let stream = connect_until(&path, Instant::now() + Duration::from_secs(1)).unwrap(); + imp::set_socket_created_hook(None); + let (_accepted, _) = listener.accept().unwrap(); + + drop(stream); + drop(listener); + std::fs::remove_file(path).unwrap(); + assert!( + close_on_exec.load(Ordering::Acquire), + "deadline connector exposed an inheritable descriptor before setting close-on-exec" + ); + } + #[cfg(target_os = "linux")] #[test] fn deadline_connect_times_out_while_listener_backlog_is_saturated() { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 501033ceddd7..5a5025798c72 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1247,6 +1247,48 @@ mod tests { command.spawn().unwrap() } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn natural_cleanup_does_not_wait_for_pty_drain() { + let signal_lock = Mutex::new(()); + let pty_drained = AtomicBool::new(false); + let termination_started = AtomicBool::new(false); + let cleanup_complete = AtomicBool::new(false); + let child_reaped = AtomicBool::new(false); + let reap_called = AtomicBool::new(false); + let sync = || ReservedChildReap { + signal_lock: &signal_lock, + pty_drained: &pty_drained, + termination_started: &termination_started, + cleanup_complete: &cleanup_complete, + child_reaped: &child_reaped, + }; + + assert!( + reserved_child_needs_cleanup(sync()), + "a descendant holding the PTY open prevented its own cleanup" + ); + assert!( + !poll_reserved_session_leader(sync(), true, || { + reap_called.store(true, Ordering::Release); + }), + "the session leader was reaped before the PTY reader drained" + ); + assert!( + cleanup_complete.load(Ordering::Acquire), + "successful descendant cleanup was not recorded before PTY drain" + ); + assert!(!child_reaped.load(Ordering::Acquire)); + assert!(!reap_called.load(Ordering::Acquire)); + + pty_drained.store(true, Ordering::Release); + assert!(poll_reserved_session_leader(sync(), false, || { + reap_called.store(true, Ordering::Release); + })); + assert!(child_reaped.load(Ordering::Acquire)); + assert!(reap_called.load(Ordering::Acquire)); + } + #[cfg(target_os = "linux")] fn enter_syscall_blocked_subprocess( child_env: &str, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 147f605786f4..5118413ae4ef 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -3376,14 +3376,21 @@ mod tests { .join(format!("cmux-local-reap-{}", crate::workspace_registry::new_uuid_v4())); std::fs::create_dir_all(&root).unwrap(); let pid_path = root.join("descendant.pid"); + let ready_path = root.join("descendant.ready"); + let release_path = root.join("leader.release"); let mux = Mux::new_for_test("local-reap-descendants", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ "/bin/sh".into(), "-c".into(), format!( - "trap '' HUP TERM; (exec sleep 30) /dev/null 2>&1 & echo $! > {}; exit 0", - pid_path.display() + "trap '' HUP TERM; terminal=$(tty); \ + sleep 30 3<>\"$terminal\" & child=$!; \ + : > {ready}; echo $child > {pid}; \ + while [ ! -e {release} ]; do sleep 0.01; done; exit 0", + ready = ready_path.display(), + pid = pid_path.display(), + release = release_path.display(), ), ]), ..SurfaceOptions::default() @@ -3394,14 +3401,22 @@ mod tests { LocalProcess::Owned(process) => process.clone(), LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), }; - let deadline = Instant::now() + Duration::from_secs(3); - while (!pid_path.exists() || !process.child_reaped.load(Ordering::Acquire)) - && Instant::now() < deadline - { + let start_deadline = Instant::now() + Duration::from_secs(1); + while !pid_path.exists() && Instant::now() < start_deadline { std::thread::sleep(Duration::from_millis(10)); } let descendant = std::fs::read_to_string(&pid_path).unwrap().trim().parse::().unwrap(); + // SAFETY: signal 0 only probes the test-owned PID. + assert_eq!(unsafe { libc::kill(descendant, 0) }, 0); + std::fs::write(&release_path, b"ready").unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while !process.child_reaped.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let naturally_reaped = process.child_reaped.load(Ordering::Acquire); + // SAFETY: signal 0 only probes the test-owned PID. + let descendant_alive_before_cleanup = unsafe { libc::kill(descendant, 0) } == 0; let terminated = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); @@ -3422,11 +3437,14 @@ mod tests { } let _ = std::fs::remove_dir_all(root); - assert!(process.child_reaped.load(Ordering::Acquire)); + assert!( + naturally_reaped, + "normal PTY exit waited for EOF from a same-session descendant before cleanup" + ); assert!(terminated); assert!( - !descendant_alive, - "normal PTY exit reaped its leader while a same-session descendant remained alive" + !descendant_alive_before_cleanup, + "normal PTY exit left a same-session descendant holding the slave open" ); } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 524ffe3833ad..36dd607aaa2b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4128,6 +4128,90 @@ mod unix { assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); } + #[test] + fn hosted_child_exit_cleans_descendants_that_keep_the_slave_open() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + let root = std::env::temp_dir() + .join(format!("cmux-host-reap-{}", crate::workspace_registry::new_uuid_v4())); + fs::create_dir_all(&root).unwrap(); + let pid_path = root.join("descendant.pid"); + let ready_path = root.join("descendant.ready"); + let release_path = root.join("leader.release"); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: root.join("host.sock").to_string_lossy().into_owned(), + record_path: root.join("host.json").to_string_lossy().into_owned(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec![ + "/bin/sh".into(), + "-c".into(), + format!( + "trap '' HUP TERM; terminal=$(tty); \ + sleep 30 3<>\"$terminal\" & child=$!; \ + : > {ready}; echo $child > {pid}; \ + while [ ! -e {release} ]; do sleep 0.01; done; exit 0", + ready = ready_path.display(), + pid = pid_path.display(), + release = release_path.display(), + ), + ], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + let host = spawn_host_runtime(&launch, &bootstrapped).unwrap(); + let start_deadline = Instant::now() + Duration::from_secs(1); + while !pid_path.exists() && Instant::now() < start_deadline { + thread::sleep(Duration::from_millis(10)); + } + let descendant = + fs::read_to_string(&pid_path).unwrap().trim().parse::().unwrap(); + // SAFETY: signal 0 only probes the test-owned PID. + assert_eq!(unsafe { libc::kill(descendant, 0) }, 0); + fs::write(&release_path, b"ready").unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + while !host.child_reaped.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let naturally_reaped = host.child_reaped.load(Ordering::Acquire); + // SAFETY: signal 0 only probes the test-owned PID. + let descendant_alive_before_cleanup = unsafe { libc::kill(descendant, 0) } == 0; + + host.request_termination(); + let _ = host.wait_for_child_exit(Duration::from_secs(1)); + if unsafe { libc::kill(descendant, 0) } == 0 { + // SAFETY: this PID was written by the test-owned descendant. + unsafe { + libc::kill(descendant, libc::SIGKILL); + } + } + let _ = fs::remove_dir_all(root); + + assert!( + naturally_reaped, + "hosted PTY exit waited for EOF from a same-session descendant before cleanup" + ); + assert!( + !descendant_alive_before_cleanup, + "hosted PTY exit left a same-session descendant holding the slave open" + ); + } + #[test] fn detached_host_process_retains_reap_ownership_when_worker_spawn_fails() { let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 85b86e55de20..4e3dcf27882b 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1386,7 +1386,7 @@ mod tests { #[cfg(unix)] #[test] - fn legacy_helper_timeout_is_bounded_without_force_killing_the_helper() { + fn legacy_helper_timeout_retains_reap_ownership() { use std::io::BufRead as _; let mut helper = Command::new("/bin/sh") @@ -1399,21 +1399,38 @@ mod tests { let mut ready = String::new(); BufReader::new(helper.stdout.take().unwrap()).read_line(&mut ready).unwrap(); assert_eq!(ready, "ready\n"); + let helper_pid = libc::pid_t::try_from(helper.id()).unwrap(); let started = Instant::now(); let error = wait_for_child_until(&mut helper, Instant::now() + Duration::from_millis(50)) .unwrap_err(); let elapsed = started.elapsed(); - let still_running = helper.try_wait().unwrap().is_none(); - let _ = helper.kill(); - let _ = helper.wait(); + drop(helper); + // SAFETY: helper_pid names this exact test-owned child. + unsafe { + libc::kill(helper_pid, libc::SIGKILL); + } + std::thread::sleep(Duration::from_millis(250)); + let mut status = 0; + // SAFETY: this probes whether another owner retained and reaped the + // exact test child after wait_for_child_until returned. + let reaped = unsafe { libc::waitpid(helper_pid, &raw mut status, libc::WNOHANG) }; + let reaped_error = (reaped == -1).then(io::Error::last_os_error); + if reaped == 0 { + // SAFETY: the child is still owned by this test process. + unsafe { + libc::kill(helper_pid, libc::SIGKILL); + libc::waitpid(helper_pid, &raw mut status, 0); + } + } assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_cleanup_failed); assert!( elapsed < Duration::from_millis(250), "helper wait exceeded its bound: {elapsed:?}" ); - assert!(still_running, "parent force-killed the helper after its deadline"); + assert_eq!(reaped, -1, "timed-out helper lost its reaping owner"); + assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); } #[cfg(unix)] From a785a26f739b57f495b3d2fd2361c7e1d1a743e2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:30:53 -0700 Subject: [PATCH 115/521] fix(tui): retain lifecycle cleanup ownership --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 198 +++++++++++++++++- .../cmux-tui-core/src/process_session.rs | 33 +-- .../crates/cmux-tui/src/server_lifecycle.rs | 151 +++++++++++-- 3 files changed, 328 insertions(+), 54 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index c095a8d08384..58e8464f3a44 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -78,21 +78,113 @@ pub mod transport { *SOCKET_CREATED_HOOK.get_or_init(Default::default).lock().unwrap() = hook; } - pub(super) fn listen(path: &Path) -> io::Result { - UnixListener::bind(path).map(|inner| Listener { inner }) + #[cfg(target_os = "macos")] + static SOCKET_FORK_BARRIER_REGISTRATION: std::sync::OnceLock = + std::sync::OnceLock::new(); + #[cfg(target_os = "macos")] + static mut SOCKET_FORK_BARRIER: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER; + + #[cfg(target_os = "macos")] + unsafe extern "C" fn socket_fork_prepare() { + // SAFETY: pthread_atfork calls this handler before fork. Blocking + // here serializes fork with the short descriptor setup window. + let _ = unsafe { libc::pthread_mutex_lock(&raw mut SOCKET_FORK_BARRIER) }; } - pub(super) fn connect(path: &Path) -> io::Result> { - Ok(Box::new(UnixStream::connect(path)?)) + #[cfg(target_os = "macos")] + unsafe extern "C" fn socket_fork_parent() { + // SAFETY: the prepare handler acquired this mutex in the forking + // thread, and the parent handler releases that same acquisition. + let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; } - pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { - connect_unix_until(path, deadline).map(|stream| Box::new(stream) as Box) + #[cfg(target_os = "macos")] + unsafe extern "C" fn socket_fork_child() { + // SAFETY: the child inherits the prepare handler's locked mutex + // and releases it before any post-fork work can continue. + let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; } - pub(super) fn connect_unix_until(path: &Path, deadline: Instant) -> io::Result { - ensure_connect_time_remaining(deadline)?; - let (address, address_len) = unix_socket_address(path)?; + #[cfg(target_os = "macos")] + struct SocketForkBarrierGuard; + + #[cfg(target_os = "macos")] + impl SocketForkBarrierGuard { + fn acquire() -> io::Result { + let registration = *SOCKET_FORK_BARRIER_REGISTRATION.get_or_init(|| { + // SAFETY: callbacks have static lifetimes and use one + // statically initialized pthread mutex. + unsafe { + libc::pthread_atfork( + Some(socket_fork_prepare), + Some(socket_fork_parent), + Some(socket_fork_child), + ) + } + }); + if registration != 0 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "cannot register close-on-exec fork barrier: {}", + io::Error::from_raw_os_error(registration) + ), + )); + } + // SAFETY: this mutex is statically initialized and remains + // live for the process lifetime. + let locked = unsafe { libc::pthread_mutex_lock(&raw mut SOCKET_FORK_BARRIER) }; + if locked != 0 { + return Err(io::Error::from_raw_os_error(locked)); + } + Ok(Self) + } + } + + #[cfg(target_os = "macos")] + impl Drop for SocketForkBarrierGuard { + fn drop(&mut self) { + // SAFETY: each guard exists only after one successful lock. + let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; + } + } + + #[cfg(target_os = "linux")] + fn create_close_on_exec_socket() -> io::Result { + // SAFETY: socket has no pointer arguments and returns a new owned + // descriptor on success. SOCK_CLOEXEC sets the inheritance flag + // atomically with descriptor creation. + let descriptor = + unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if descriptor < 0 { + let error = io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(libc::EINVAL) | Some(libc::EPROTONOSUPPORT)) + { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("atomic close-on-exec sockets are unavailable: {error}"), + )); + } + return Err(error); + } + // SAFETY: descriptor is a fresh successful socket result and this + // OwnedFd takes its sole ownership. + let descriptor = unsafe { OwnedFd::from_raw_fd(descriptor) }; + #[cfg(test)] + if let Some(hook) = + SOCKET_CREATED_HOOK.get_or_init(Default::default).lock().unwrap().clone() + { + hook(descriptor.as_raw_fd()); + } + Ok(descriptor) + } + + #[cfg(target_os = "macos")] + fn create_close_on_exec_socket() -> io::Result { + let _fork_barrier = SocketForkBarrierGuard::acquire()?; + // macOS has no SOCK_CLOEXEC socket flag. The registered atfork + // barrier prevents every fork in this process until fcntl marks + // the fresh descriptor close-on-exec. // SAFETY: socket has no pointer arguments and returns a new owned // descriptor on success. let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; @@ -113,7 +205,8 @@ pub mod transport { if descriptor_flags < 0 { return Err(io::Error::last_os_error()); } - // SAFETY: F_SETFD updates flags on this valid descriptor. + // SAFETY: F_SETFD updates flags while the fork barrier excludes + // process creation from this setup window. if unsafe { libc::fcntl( descriptor.as_raw_fd(), @@ -124,6 +217,47 @@ pub mod transport { { return Err(io::Error::last_os_error()); } + Ok(descriptor) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + fn create_close_on_exec_socket() -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "deadline sockets require atomic close-on-exec setup", + )) + } + + #[cfg(all(target_os = "macos", test))] + pub(super) fn socket_fork_barrier_is_locked_for_test() -> bool { + // SAFETY: trylock only inspects the statically initialized mutex. + let result = unsafe { libc::pthread_mutex_trylock(&raw mut SOCKET_FORK_BARRIER) }; + if result == 0 { + // SAFETY: a successful trylock created one balanced test + // acquisition that must be released here. + let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; + false + } else { + result == libc::EBUSY + } + } + + pub(super) fn listen(path: &Path) -> io::Result { + UnixListener::bind(path).map(|inner| Listener { inner }) + } + + pub(super) fn connect(path: &Path) -> io::Result> { + Ok(Box::new(UnixStream::connect(path)?)) + } + + pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { + connect_unix_until(path, deadline).map(|stream| Box::new(stream) as Box) + } + + pub(super) fn connect_unix_until(path: &Path, deadline: Instant) -> io::Result { + ensure_connect_time_remaining(deadline)?; + let (address, address_len) = unix_socket_address(path)?; + let descriptor = create_close_on_exec_socket()?; let stream = UnixStream::from(descriptor); stream.set_nonblocking(true)?; loop { @@ -673,7 +807,7 @@ pub mod transport { std::fs::remove_file(path).unwrap(); } - #[cfg(unix)] + #[cfg(target_os = "linux")] #[test] fn deadline_connect_is_close_on_exec_at_socket_creation() { use std::sync::Arc; @@ -714,6 +848,48 @@ pub mod transport { ); } + #[cfg(target_os = "macos")] + #[test] + fn deadline_connect_holds_fork_barrier_until_close_on_exec() { + use std::os::fd::AsRawFd as _; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let _guard = SOCKET_CREATED_TEST_LOCK.lock().unwrap(); + let path = std::path::PathBuf::from("/tmp").join(format!( + "cmux-cxfb-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let fork_barrier_held = Arc::new(AtomicBool::new(false)); + imp::set_socket_created_hook(Some(Arc::new({ + let fork_barrier_held = fork_barrier_held.clone(); + move |_| { + fork_barrier_held + .store(imp::socket_fork_barrier_is_locked_for_test(), Ordering::Release); + } + }))); + + let stream = + connect_unix_until(&path, Instant::now() + Duration::from_secs(1)).unwrap(); + imp::set_socket_created_hook(None); + let flags = unsafe { libc::fcntl(stream.as_raw_fd(), libc::F_GETFD) }; + let (_accepted, _) = listener.accept().unwrap(); + + drop(stream); + drop(listener); + std::fs::remove_file(path).unwrap(); + assert!( + fork_barrier_held.load(Ordering::Acquire), + "deadline connector exposed its socket outside the fork barrier" + ); + assert!(flags >= 0 && flags & libc::FD_CLOEXEC != 0); + } + #[cfg(target_os = "linux")] #[test] fn deadline_connect_times_out_while_listener_backlog_is_saturated() { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 5a5025798c72..7ebfb820ec4a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -194,7 +194,6 @@ pub(crate) fn reserved_child_needs_cleanup(sync: ReservedChildReap<'_>) -> bool let _signal = sync.signal_lock.lock().unwrap(); !sync.cleanup_complete.load(Ordering::Acquire) && !sync.termination_started.load(Ordering::Acquire) - && sync.pty_drained.load(Ordering::Acquire) } pub(crate) fn poll_reserved_session_leader( @@ -202,29 +201,19 @@ pub(crate) fn poll_reserved_session_leader( cleanup_succeeded: bool, reap: impl FnOnce(), ) -> bool { - let should_attempt_cleanup = { - let _signal = sync.signal_lock.lock().unwrap(); - if sync.cleanup_complete.load(Ordering::Acquire) { - reap(); - sync.child_reaped.store(true, Ordering::Release); - return true; - } - !sync.termination_started.load(Ordering::Acquire) - && sync.pty_drained.load(Ordering::Acquire) - }; - - if should_attempt_cleanup { - let _signal = sync.signal_lock.lock().unwrap(); - let cleanup_claimed = sync.cleanup_complete.load(Ordering::Acquire); - let termination_started = sync.termination_started.load(Ordering::Acquire); - if cleanup_claimed || (cleanup_succeeded && !termination_started) { - sync.cleanup_complete.store(true, Ordering::Release); - reap(); - sync.child_reaped.store(true, Ordering::Release); - return true; + let _signal = sync.signal_lock.lock().unwrap(); + if !sync.cleanup_complete.load(Ordering::Acquire) { + if sync.termination_started.load(Ordering::Acquire) || !cleanup_succeeded { + return false; } + sync.cleanup_complete.store(true, Ordering::Release); + } + if !sync.pty_drained.load(Ordering::Acquire) { + return false; } - false + reap(); + sync.child_reaped.store(true, Ordering::Release); + true } fn run_natural_reaper( diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 4e3dcf27882b..23116bf99900 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -3,9 +3,11 @@ use std::ffi::OsStr; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; #[cfg(unix)] -use std::process::{Command, Stdio}; +use std::process::{Child, Command, ExitStatus, Stdio}; #[cfg(unix)] -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +#[cfg(unix)] +use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; @@ -47,6 +49,8 @@ const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); const LEGACY_HELPER_POLL_INTERVAL: Duration = Duration::from_millis(10); #[cfg(unix)] const LEGACY_HELPER_CANCEL_MARGIN: Duration = Duration::from_millis(100); +#[cfg(unix)] +const LEGACY_HELPER_REAPER_CAPACITY: usize = 64; const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; @@ -54,6 +58,103 @@ type TransportReader = BufReader>; #[cfg(unix)] static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); +#[cfg(unix)] +static LEGACY_HELPER_REAPER: OnceLock> = OnceLock::new(); + +#[cfg(unix)] +struct LegacyHelperReaper { + pending: Arc<(Mutex>, Condvar)>, + active: Arc, + _worker: std::thread::JoinHandle<()>, +} + +#[cfg(unix)] +struct LegacyHelperReap { + child: Child, + _lease: LegacyHelperReaperLease, +} + +#[cfg(unix)] +struct LegacyHelperReaperLease { + pending: Arc<(Mutex>, Condvar)>, + active: Arc, +} + +#[cfg(unix)] +impl Drop for LegacyHelperReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +#[cfg(unix)] +impl LegacyHelperReaperLease { + fn retain_reap_ownership(self, child: Child) { + let pending = self.pending.clone(); + pending + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(LegacyHelperReap { child, _lease: self }); + pending.1.notify_one(); + } +} + +#[cfg(unix)] +impl LegacyHelperReaper { + fn start() -> std::io::Result { + let pending = Arc::new((Mutex::new(Vec::new()), Condvar::new())); + let active = Arc::new(AtomicUsize::new(0)); + let worker_pending = pending.clone(); + let worker = std::thread::Builder::new() + .name("cmux-legacy-helper-reaper".into()) + .spawn(move || run_legacy_helper_reaper(worker_pending))?; + Ok(Self { pending, active, _worker: worker }) + } + + fn reserve(&self) -> Option { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < LEGACY_HELPER_REAPER_CAPACITY).then_some(active + 1) + }) + .ok()?; + Some(LegacyHelperReaperLease { pending: self.pending.clone(), active: self.active.clone() }) + } +} + +#[cfg(unix)] +fn reserve_legacy_helper_reaper() -> anyhow::Result { + let reaper = LEGACY_HELPER_REAPER.get_or_init(|| LegacyHelperReaper::start().ok()); + reaper + .as_ref() + .and_then(LegacyHelperReaper::reserve) + .ok_or_else(|| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed)) +} + +#[cfg(unix)] +fn run_legacy_helper_reaper(pending: Arc<(Mutex>, Condvar)>) { + let mut pending_children = pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + while pending_children.is_empty() { + pending_children = + pending.1.wait(pending_children).unwrap_or_else(std::sync::PoisonError::into_inner); + } + let mut index = 0; + while index < pending_children.len() { + match pending_children[index].child.try_wait() { + Ok(Some(_)) => drop(pending_children.swap_remove(index)), + Ok(None) | Err(_) => index += 1, + } + } + if !pending_children.is_empty() { + let waited = pending + .1 + .wait_timeout(pending_children, LEGACY_HELPER_POLL_INTERVAL) + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending_children = waited.0; + } + } +} #[cfg(unix)] extern "C" fn handle_legacy_helper_signal(_: libc::c_int) { @@ -587,6 +688,7 @@ fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::Result<()> { use std::os::unix::process::CommandExt; + let reaper = reserve_legacy_helper_reaper()?; let executable = std::env::current_exe().map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; @@ -608,11 +710,11 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R Ok(()) }); } - let mut helper = command.spawn().map_err(|_| { + let helper = command.spawn().map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; let deadline = Instant::now() + LEGACY_SHUTDOWN_TIMEOUT + LEGACY_HELPER_WAIT_MARGIN; - let status = wait_for_child_until(&mut helper, deadline)?; + let status = wait_for_child_until(helper, reaper, deadline)?; if status.success() { return Ok(()); } @@ -621,14 +723,18 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R #[cfg(unix)] fn wait_for_child_until( - child: &mut std::process::Child, + mut child: Child, + reaper: LegacyHelperReaperLease, deadline: Instant, -) -> anyhow::Result { +) -> anyhow::Result { loop { - if let Some(status) = child.try_wait().map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })? { - return Ok(status); + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) => {} + Err(_) => { + reaper.retain_reap_ownership(child); + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } } if Instant::now() >= deadline { let pid = libc::pid_t::try_from(child.id()).ok(); @@ -639,14 +745,15 @@ fn wait_for_child_until( } let cancel_deadline = Instant::now() + LEGACY_HELPER_CANCEL_MARGIN; while Instant::now() < cancel_deadline { - if child - .try_wait() - .map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })? - .is_some() - { - break; + match child.try_wait() { + Ok(Some(_)) => { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + Ok(None) => {} + Err(_) => { + reaper.retain_reap_ownership(child); + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } } std::thread::sleep( cancel_deadline @@ -654,6 +761,7 @@ fn wait_for_child_until( .min(LEGACY_HELPER_POLL_INTERVAL), ); } + reaper.retain_reap_ownership(child); anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } std::thread::sleep( @@ -1401,11 +1509,12 @@ mod tests { assert_eq!(ready, "ready\n"); let helper_pid = libc::pid_t::try_from(helper.id()).unwrap(); let started = Instant::now(); + let reaper = reserve_legacy_helper_reaper().unwrap(); - let error = wait_for_child_until(&mut helper, Instant::now() + Duration::from_millis(50)) - .unwrap_err(); + let error = + wait_for_child_until(helper, reaper, Instant::now() + Duration::from_millis(50)) + .unwrap_err(); let elapsed = started.elapsed(); - drop(helper); // SAFETY: helper_pid names this exact test-owned child. unsafe { libc::kill(helper_pid, libc::SIGKILL); From 22566d5f9285ae37738cdc4cda9c2c54eebfed4c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:59:57 -0700 Subject: [PATCH 116/521] test(tui): expose exceptional shutdown ownership gaps --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 19 +++++++- .../crates/cmux-tui/src/server_lifecycle.rs | 43 +++++++++++++++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 33 ++++++++++++-- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 5118413ae4ef..406cfe339e3b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -813,6 +813,14 @@ struct LocalPtyProcess { #[cfg(unix)] type LocalChildReaperLease = crate::process_session::ReservedChildReaperLease; +#[cfg(all(unix, test))] +static DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS: AtomicUsize = AtomicUsize::new(0); + +#[cfg(all(unix, test))] +fn dedicated_local_child_observer_spawns_for_test() -> usize { + DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS.load(Ordering::Acquire) +} + #[cfg(not(unix))] struct LocalChildReaperLease; @@ -870,6 +878,8 @@ impl LocalPtyProcess { reaper: LocalChildReaperLease, ) -> std::io::Result<()> { let process = self.clone(); + #[cfg(all(unix, test))] + DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS.fetch_add(1, Ordering::AcqRel); std::thread::Builder::new().name("surface-child".into()).spawn(move || { #[cfg(unix)] { @@ -3491,7 +3501,7 @@ mod tests { #[cfg(unix)] #[test] - fn persistent_natural_cleanup_uses_one_shared_reaper() { + fn local_child_observation_uses_the_reserved_shared_reaper() { const CHILDREN: usize = 8; let root = std::env::temp_dir() @@ -3500,6 +3510,7 @@ mod tests { let release_path = root.join("release"); let mux = Mux::new_for_test("local-reap-bound", SurfaceOptions::default()); let baseline = crate::process_session::dedicated_natural_reap_workers_for_test(); + let observer_baseline = dedicated_local_child_observer_spawns_for_test(); let mut surfaces = Vec::new(); let mut processes = Vec::new(); for index in 0..CHILDREN { @@ -3526,6 +3537,8 @@ mod tests { surfaces.push(surface); processes.push(process); } + let dedicated_observers = + dedicated_local_child_observer_spawns_for_test().saturating_sub(observer_baseline); std::fs::write(&release_path, b"ready").unwrap(); let deadline = Instant::now() + Duration::from_secs(3); @@ -3549,6 +3562,10 @@ mod tests { dedicated <= 1, "persistent natural cleanup retained {dedicated} dedicated retry workers" ); + assert_eq!( + dedicated_observers, 0, + "local child observation bypassed the reserved shared reaper" + ); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 23116bf99900..ee37eb003b6b 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -60,6 +60,8 @@ type TransportReader = BufReader>; static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); #[cfg(unix)] static LEGACY_HELPER_REAPER: OnceLock> = OnceLock::new(); +#[cfg(all(unix, test))] +static LEGACY_HELPER_REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); #[cfg(unix)] struct LegacyHelperReaper { @@ -131,6 +133,14 @@ fn reserve_legacy_helper_reaper() -> anyhow::Result { .ok_or_else(|| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed)) } +#[cfg(all(unix, test))] +fn active_legacy_helper_reapers_for_test() -> usize { + LEGACY_HELPER_REAPER + .get() + .and_then(Option::as_ref) + .map_or(0, |reaper| reaper.active.load(Ordering::Acquire)) +} + #[cfg(unix)] fn run_legacy_helper_reaper(pending: Arc<(Mutex>, Condvar)>) { let mut pending_children = pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1497,6 +1507,7 @@ mod tests { fn legacy_helper_timeout_retains_reap_ownership() { use std::io::BufRead as _; + let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); let mut helper = Command::new("/bin/sh") .arg("-c") .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") @@ -1542,6 +1553,38 @@ mod tests { assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); } + #[cfg(unix)] + #[test] + fn legacy_reaper_releases_children_with_lost_wait_ownership() { + let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); + let child = Command::new("/usr/bin/true") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let child_pid = libc::pid_t::try_from(child.id()).unwrap(); + let mut status = 0; + // SAFETY: child_pid is this test's exact child. Reaping it here + // simulates another library consuming the wait status first. + assert_eq!(unsafe { libc::waitpid(child_pid, &raw mut status, 0) }, child_pid); + + let baseline = active_legacy_helper_reapers_for_test(); + let reaper = reserve_legacy_helper_reaper().unwrap(); + assert_eq!(active_legacy_helper_reapers_for_test(), baseline + 1); + reaper.retain_reap_ownership(child); + + let deadline = Instant::now() + Duration::from_millis(250); + while active_legacy_helper_reapers_for_test() != baseline && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + + assert_eq!( + active_legacy_helper_reapers_for_test(), + baseline, + "legacy reaper retained a child after the kernel reported no wait ownership" + ); + } + #[cfg(unix)] #[test] fn legacy_stop_terminates_only_the_identified_process() { diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 36c3332ad237..29834bc1514f 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1208,19 +1208,30 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { #[cfg(unix)] #[test] -fn daemon_handoff_exits_when_the_interactive_driver_cannot_progress() { +fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_to_exit() { let dir = unique_temp_dir("daemon-handoff-blocked-interactive-driver"); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); - let state = dir.join("state"); + let pid_file = dir.join("local-pty.pid"); let mut server = PtyChild::start_with_env( - &["--socket", socket.to_str().unwrap(), "--state", state.to_str().unwrap()], + &["--socket", socket.to_str().unwrap(), "--ephemeral"], &[ ("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER", std::ffi::OsStr::new("1")), ("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS", std::ffi::OsStr::new("100")), ], ); wait_for_socket_path(&socket); + let command = + format!("trap '' HUP TERM; echo $$ > {}; while :; do sleep 1; done", pid_file.display()); + let run = Command::new(bin()) + .args(["--socket"]) + .arg(&socket) + .args(["run", "--new-workspace", "--command", &command]) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_success(&run); + let local_pid = wait_for_pid_file(&pid_file, Duration::from_secs(5)); let identity = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "identify"})); let response = json_socket_request( @@ -1247,8 +1258,24 @@ fn daemon_handoff_exits_when_the_interactive_driver_cannot_progress() { }; assert!(status.success(), "server exited with {status}"); assert!(transport::connect(&socket).is_err()); + let cleanup_deadline = Instant::now() + Duration::from_secs(3); + while (process_exists(local_pid) || process_group_exists(local_pid)) + && Instant::now() < cleanup_deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + let cleaned = !process_exists(local_pid) && !process_group_exists(local_pid); + if !cleaned { + let local_pid = libc::pid_t::try_from(local_pid).unwrap(); + // SAFETY: local_pid names the isolated PTY session created by this test. + unsafe { + libc::kill(-local_pid, libc::SIGKILL); + libc::kill(local_pid, libc::SIGKILL); + } + } drop(server); fs::remove_dir_all(dir).unwrap(); + assert!(cleaned, "forced interactive-driver exit abandoned its local PTY session"); } #[cfg(unix)] From 021f28248c96cf43097673a8f0f53918785ee36c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:09:57 -0700 Subject: [PATCH 117/521] fix(tui): centralize exceptional shutdown ownership --- .../cmux-tui-core/src/process_session.rs | 36 +++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 176 ++++++++++-------- .../src/terminal_host_runtime.rs | 1 + cmux-tui/crates/cmux-tui/src/main.rs | 96 ++++++++-- .../crates/cmux-tui/src/server_lifecycle.rs | 49 +++-- 5 files changed, 253 insertions(+), 105 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 7ebfb820ec4a..0e553ea39bfc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -83,6 +83,8 @@ enum NaturalReaperCommand { struct NaturalReapRequest { session: libc::pid_t, cleanup_timeout: Duration, + observe_child: Box io::Result + Send>, + child_observed: bool, needs_cleanup: Box bool + Send>, prepare_cleanup: Box bool + Send>, finish: Box bool + Send>, @@ -169,6 +171,7 @@ pub(crate) fn enqueue_reserved_session_leader( lease: ReservedChildReaperLease, session: libc::pid_t, cleanup_timeout: Duration, + observe_child: impl FnMut() -> io::Result + Send + 'static, needs_cleanup: impl Fn() -> bool + Send + 'static, prepare_cleanup: impl FnMut() -> bool + Send + 'static, finish: impl FnMut(bool) -> bool + Send + 'static, @@ -178,6 +181,8 @@ pub(crate) fn enqueue_reserved_session_leader( .send(NaturalReaperCommand::Add(NaturalReapRequest { session, cleanup_timeout, + observe_child: Box::new(observe_child), + child_observed: false, needs_cleanup: Box::new(needs_cleanup), prepare_cleanup: Box::new(prepare_cleanup), finish: Box::new(finish), @@ -268,11 +273,26 @@ fn run_natural_reaper( continue; } - let needs_cleanup = due.iter().map(|request| (request.needs_cleanup)()).collect::>(); + let child_observed = due + .iter_mut() + .map(|request| { + if !request.child_observed { + request.child_observed = (request.observe_child)().unwrap_or(false); + } + request.child_observed + }) + .collect::>(); + let needs_cleanup = due + .iter() + .zip(&child_observed) + .map(|(request, observed)| *observed && (request.needs_cleanup)()) + .collect::>(); let prepared = due .iter_mut() - .zip(&needs_cleanup) - .map(|(request, needed)| !needed || (request.prepare_cleanup)()) + .zip(child_observed.iter().zip(&needs_cleanup)) + .map(|(request, (observed, needed))| { + !observed || !needed || (request.prepare_cleanup)() + }) .collect::>(); let sessions = due .iter() @@ -302,7 +322,15 @@ fn run_natural_reaper( .unwrap_or_default() }; - for ((mut request, needed), prepared) in due.into_iter().zip(needs_cleanup).zip(prepared) { + for (((mut request, observed), needed), prepared) in + due.into_iter().zip(child_observed).zip(needs_cleanup).zip(prepared) + { + if !observed { + request.next_attempt = Instant::now() + request.retry_delay; + request.retry_delay = (request.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + pending.push(request); + continue; + } let cleanup_succeeded = needed && prepared && cleaned.contains(&request.session); let done = (request.finish)(cleanup_succeeded); if done { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 406cfe339e3b..69fa8262033e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -877,81 +877,103 @@ impl LocalPtyProcess { mut child: Box, reaper: LocalChildReaperLease, ) -> std::io::Result<()> { - let process = self.clone(); - #[cfg(all(unix, test))] - DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS.fetch_add(1, Ordering::AcqRel); - std::thread::Builder::new().name("surface-child".into()).spawn(move || { - #[cfg(unix)] - { - let observed_without_reaping = process - .pid - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .is_some_and(|pid| wait_for_local_child_without_reaping(pid).is_ok()); - if observed_without_reaping { - let session = - libc::pid_t::try_from(process.pid.expect("observed child has a PID")) - .expect("portable-pty child PID fits pid_t"); - process.child_waitable.store(true, Ordering::Release); - process.exited.1.notify_all(); - let cleanup_process = process.clone(); - let prepare_process = process.clone(); - let reap_process = process.clone(); - let mut child = Some(child); - crate::process_session::enqueue_reserved_session_leader( - reaper, - session, - NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, - move || { - crate::process_session::reserved_child_needs_cleanup( - crate::process_session::ReservedChildReap { - signal_lock: &cleanup_process.child_signal_lock, - pty_drained: &cleanup_process.pty_drained, - termination_started: &cleanup_process.termination_started, - cleanup_complete: &cleanup_process.group_escalation_complete, - child_reaped: &cleanup_process.child_reaped, - }, - ) + #[cfg(unix)] + { + let process = self.clone(); + let Some(session) = process.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + let _ = child.kill(); + let _ = child.wait(); + process.child_reaped.store(true, Ordering::Release); + process.child_waitable.store(true, Ordering::Release); + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "native PTY child did not expose a process ID", + )); + }; + let observe_process = process.clone(); + let cleanup_process = process.clone(); + let prepare_process = process.clone(); + let reap_process = process; + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, + NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, + move || { + let observed = local_child_is_waitable(session)?; + if observed { + observe_process.child_waitable.store(true, Ordering::Release); + observe_process.exited.1.notify_all(); + } + Ok(observed) + }, + move || { + crate::process_session::reserved_child_needs_cleanup( + crate::process_session::ReservedChildReap { + signal_lock: &cleanup_process.child_signal_lock, + pty_drained: &cleanup_process.pty_drained, + termination_started: &cleanup_process.termination_started, + cleanup_complete: &cleanup_process.group_escalation_complete, + child_reaped: &cleanup_process.child_reaped, }, - move || prepare_process.prepare_natural_cleanup(), - move |cleanup_succeeded| { - let done = crate::process_session::poll_reserved_session_leader( - crate::process_session::ReservedChildReap { - signal_lock: &reap_process.child_signal_lock, - pty_drained: &reap_process.pty_drained, - termination_started: &reap_process.termination_started, - cleanup_complete: &reap_process.group_escalation_complete, - child_reaped: &reap_process.child_reaped, - }, - cleanup_succeeded, - || { - let mut child = - child.take().expect("reserved child is reaped once"); - let _ = child.wait(); - }, - ); - if done { - *reap_process.exited.0.lock().unwrap() = true; - reap_process.exited.1.notify_all(); - } - done + ) + }, + move || prepare_process.prepare_natural_cleanup(), + move |cleanup_succeeded| { + let done = crate::process_session::poll_reserved_session_leader( + crate::process_session::ReservedChildReap { + signal_lock: &reap_process.child_signal_lock, + pty_drained: &reap_process.pty_drained, + termination_started: &reap_process.termination_started, + cleanup_complete: &reap_process.group_escalation_complete, + child_reaped: &reap_process.child_reaped, + }, + cleanup_succeeded, + || { + let mut child = child.take().expect("reserved child is reaped once"); + let _ = child.wait(); }, ); - return; - } else { + if done { + *reap_process.exited.0.lock().unwrap() = true; + reap_process.exited.1.notify_all(); + } + done + }, + ); + Ok(()) + } + + #[cfg(not(unix))] + { + let process = self.clone(); + let (child_sender, child_receiver) = sync_channel(1); + let observer = + std::thread::Builder::new().name("surface-child".into()).spawn(move || { + let Ok(mut child) = child_receiver.recv() else { return }; let _ = child.wait(); - process.child_reaped.store(true, Ordering::Release); - process.child_waitable.store(true, Ordering::Release); - } + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + }); + if let Err(error) = observer { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); } - #[cfg(not(unix))] - { - let _ = reaper; + if let Err(error) = child_sender.send(child) { + let mut child = error.0; + let _ = child.kill(); let _ = child.wait(); + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "surface child observer exited before accepting ownership", + )); } - *process.exited.0.lock().unwrap() = true; - process.exited.1.notify_all(); - })?; - Ok(()) + let _ = reaper; + Ok(()) + } } fn request_hangup(&self) -> bool { @@ -1180,24 +1202,30 @@ impl Drop for LocalPtyProcess { } #[cfg(unix)] -fn wait_for_local_child_without_reaping(pid: libc::pid_t) -> std::io::Result<()> { +fn local_child_is_waitable(pid: libc::pid_t) -> std::io::Result { loop { - let mut status = std::mem::MaybeUninit::::uninit(); + let mut status = std::mem::MaybeUninit::::zeroed(); let result = unsafe { libc::waitid( libc::P_PID, pid as libc::id_t, status.as_mut_ptr(), - libc::WEXITED | libc::WNOWAIT, + libc::WEXITED | libc::WNOWAIT | libc::WNOHANG, ) }; if result == 0 { - return Ok(()); + // SAFETY: waitid initialized status on success; zeroed storage + // keeps si_signo at zero when WNOHANG finds no waitable child. + return Ok(unsafe { status.assume_init() }.si_signo != 0); } let error = std::io::Error::last_os_error(); - if error.kind() != std::io::ErrorKind::Interrupted { - return Err(error); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + if error.raw_os_error() == Some(libc::ECHILD) { + return Ok(true); } + return Err(error); } } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 36dd607aaa2b..113ebb13f874 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3173,6 +3173,7 @@ mod unix { reaper, session, HOST_KILL_WAIT, + || Ok(true), move || { crate::process_session::reserved_child_needs_cleanup( crate::process_session::ReservedChildReap { diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index dc30e768ff77..d8e0b251f684 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -59,11 +59,9 @@ static SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false); #[cfg(unix)] const MACHINE_PROVIDER_TOKEN_ENV: &str = "CMUX_MACHINE_PROVIDER_TOKEN"; const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORITY"; -// One second beyond the core cleanup budget lets `run_server` report a -// bounded cleanup error before the last-resort watchdog exits the process. -const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = - cmux_tui_core::server::SERVER_SHUTDOWN_TIMEOUT - .saturating_add(std::time::Duration::from_secs(1)); +// Once core cleanup owns no external processes, give the interactive driver +// one final second to unwind before releasing the socket and exiting. +const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -945,9 +943,60 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result), +} + +struct ServerShutdownCleanup { + mux: Arc, + state: std::sync::Mutex, + changed: std::sync::Condvar, +} + +impl ServerShutdownCleanup { + fn new(mux: Arc) -> Self { + Self { + mux, + state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending), + changed: std::sync::Condvar::new(), + } + } + + fn run(&self) -> anyhow::Result<()> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + match &*state { + ServerShutdownCleanupState::Pending => { + *state = ServerShutdownCleanupState::Running; + break; + } + ServerShutdownCleanupState::Running => { + state = + self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } + ServerShutdownCleanupState::Finished(result) => { + return result.clone().map_err(anyhow::Error::msg); + } + } + } + drop(state); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.mux.shutdown())) + .map_err(|_| "server shutdown cleanup panicked".to_string()) + .and_then(|result| result.map_err(|error| format!("{error:#}"))); + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *state = ServerShutdownCleanupState::Finished(result.clone()); + self.changed.notify_all(); + result.map_err(anyhow::Error::msg) + } +} + struct ServerProcessShutdownGuard { socket_path: PathBuf, shutdown_watch: cmux_tui_core::ShutdownRequestWatch, + cleanup: Arc, completion: Option>, worker: Option>, } @@ -957,6 +1006,8 @@ impl ServerProcessShutdownGuard { let shutdown_watch = mux.watch_shutdown_request(); let worker_watch = shutdown_watch.clone(); let worker_socket_path = socket_path.clone(); + let cleanup = Arc::new(ServerShutdownCleanup::new(mux.clone())); + let worker_cleanup = cleanup.clone(); let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::Builder::new().name("cmux-server-exit-watchdog".into()).spawn( move || { @@ -964,21 +1015,36 @@ impl ServerProcessShutdownGuard { return; } + // A blocked frontend cannot own process cleanup. Complete the + // same bounded cleanup path that run_server joins below, and + // retain the socket indefinitely if cleanup cannot prove that + // every external owner was released. + if worker_cleanup.run().is_err() { + return; + } if matches!( completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) ) { - // Full shutdown drains every surface before requesting - // exit. Daemon handoff fences new work and gives normal - // cleanup the same bounded grace. If the interactive - // driver cannot return after either accepted boundary, - // the old process must release its control socket. + // Cleanup already proved that no external ownership + // remains. The frontend alone is stuck, so the old + // process can now release its control socket. cmux_tui_core::server::cleanup(&worker_socket_path); std::process::exit(0); } }, )?; - Ok(Self { socket_path, shutdown_watch, completion: Some(completion), worker: Some(worker) }) + Ok(Self { + socket_path, + shutdown_watch, + cleanup, + completion: Some(completion), + worker: Some(worker), + }) + } + + fn shutdown(&self) -> anyhow::Result<()> { + self.cleanup.run() } fn complete(mut self) { @@ -1160,12 +1226,12 @@ fn run_server( let result = if args.headless { run_headless(&mux, &socket_path) } else if let Some(runtime) = machine_runtime { - run_machine_client(runtime, mux.clone()) + run_machine_client(runtime, mux) } else { - run_tui(Session::Local(mux.clone()), args.session, None) + run_tui(Session::Local(mux), args.session, None) }; drop(websocket_server); - let shutdown_result = mux.shutdown(); + let shutdown_result = server_process.shutdown(); server_process.complete(); shutdown_result?; result @@ -1435,6 +1501,7 @@ mod tests { let original = UnixListener::bind(&socket_path).unwrap(); let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); + let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); let (completion, completed) = std::sync::mpsc::channel(); let (replacement_tx, replacement_rx) = std::sync::mpsc::sync_channel(1); let replacement_path = socket_path.clone(); @@ -1446,6 +1513,7 @@ mod tests { let guard = ServerProcessShutdownGuard { socket_path: socket_path.clone(), shutdown_watch, + cleanup, completion: Some(completion), worker: Some(worker), }; diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index ee37eb003b6b..2ab60d4f38cc 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -51,6 +51,8 @@ const LEGACY_HELPER_POLL_INTERVAL: Duration = Duration::from_millis(10); const LEGACY_HELPER_CANCEL_MARGIN: Duration = Duration::from_millis(100); #[cfg(unix)] const LEGACY_HELPER_REAPER_CAPACITY: usize = 64; +#[cfg(unix)] +const LEGACY_HELPER_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; @@ -74,6 +76,8 @@ struct LegacyHelperReaper { struct LegacyHelperReap { child: Child, _lease: LegacyHelperReaperLease, + next_attempt: Instant, + retry_delay: Duration, } #[cfg(unix)] @@ -93,11 +97,14 @@ impl Drop for LegacyHelperReaperLease { impl LegacyHelperReaperLease { fn retain_reap_ownership(self, child: Child) { let pending = self.pending.clone(); - pending - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .push(LegacyHelperReap { child, _lease: self }); + pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push( + LegacyHelperReap { + child, + _lease: self, + next_attempt: Instant::now(), + retry_delay: LEGACY_HELPER_POLL_INTERVAL, + }, + ); pending.1.notify_one(); } } @@ -149,20 +156,36 @@ fn run_legacy_helper_reaper(pending: Arc<(Mutex>, Condvar) pending_children = pending.1.wait(pending_children).unwrap_or_else(std::sync::PoisonError::into_inner); } + let now = Instant::now(); + let next_attempt = + pending_children.iter().map(|entry| entry.next_attempt).min().unwrap_or(now); + if next_attempt > now { + let waited = pending + .1 + .wait_timeout(pending_children, next_attempt.saturating_duration_since(now)) + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending_children = waited.0; + continue; + } let mut index = 0; while index < pending_children.len() { + if pending_children[index].next_attempt > now { + index += 1; + continue; + } match pending_children[index].child.try_wait() { Ok(Some(_)) => drop(pending_children.swap_remove(index)), - Ok(None) | Err(_) => index += 1, + Err(error) if error.raw_os_error() == Some(libc::ECHILD) => { + drop(pending_children.swap_remove(index)); + } + Ok(None) | Err(_) => { + let entry = &mut pending_children[index]; + entry.next_attempt = Instant::now() + entry.retry_delay; + entry.retry_delay = (entry.retry_delay * 2).min(LEGACY_HELPER_REAPER_RETRY_MAX); + index += 1; + } } } - if !pending_children.is_empty() { - let waited = pending - .1 - .wait_timeout(pending_children, LEGACY_HELPER_POLL_INTERVAL) - .unwrap_or_else(std::sync::PoisonError::into_inner); - pending_children = waited.0; - } } } From 8c3461936237562fc2b17f4a769d722aceb25848 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:12:14 -0700 Subject: [PATCH 118/521] test(tui): model browser shutdown discovery --- cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 072f46c00828..b66321cf1588 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -1010,6 +1010,9 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { | "Input.dispatchMouseEvent" => { write_json(&mut ws, json!({"id": id, "result": {}})); } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); break; @@ -1088,6 +1091,9 @@ fn stalled_external_browser_nudges_target_once_before_interaction() { | "Input.dispatchMouseEvent" => { write_json(&mut ws, json!({"id": id, "result": {}})); } + "Target.getTargets" => { + write_target_list(&mut ws, id, &["target-1"]); + } "Target.closeTarget" => { write_json(&mut ws, json!({"id": id, "result": {"success": true}})); break; From d2e018ea6f67d4c7cdc430dfc8dbbe73b5079983 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:15:08 -0700 Subject: [PATCH 119/521] fix(tui): preserve non-Unix child handoff typing --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 69fa8262033e..67122c3ec83f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -949,7 +949,8 @@ impl LocalPtyProcess { #[cfg(not(unix))] { let process = self.clone(); - let (child_sender, child_receiver) = sync_channel(1); + let (child_sender, child_receiver) = + sync_channel::>(1); let observer = std::thread::Builder::new().name("surface-child".into()).spawn(move || { let Ok(mut child) = child_receiver.recv() else { return }; From d32ff0ba457dd99d3c2f8f82e0252ae8e6effc5d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:40:41 -0700 Subject: [PATCH 120/521] test(tui): retain sockets through cleanup failure --- cmux-tui/crates/cmux-tui/src/main.rs | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index d8e0b251f684..fa45d9d62c04 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1502,6 +1502,7 @@ mod tests { let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); + *cleanup.state.lock().unwrap() = ServerShutdownCleanupState::Finished(Ok(())); let (completion, completed) = std::sync::mpsc::channel(); let (replacement_tx, replacement_rx) = std::sync::mpsc::sync_channel(1); let replacement_path = socket_path.clone(); @@ -1529,6 +1530,40 @@ mod tests { cmux_tui_core::server::cleanup(&socket_path); } + #[cfg(unix)] + #[test] + fn incomplete_server_guard_retains_its_control_socket() { + use std::os::unix::net::{UnixListener, UnixStream}; + use std::time::{SystemTime, UNIX_EPOCH}; + + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let socket_path = std::env::temp_dir() + .join(format!("cg-incomplete-{}-{suffix:x}.sock", std::process::id())); + let listener = UnixListener::bind(&socket_path).unwrap(); + let mux = Mux::new("incomplete-server-guard-test", SurfaceOptions::default()); + let shutdown_watch = mux.watch_shutdown_request(); + let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); + let (completion, completed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let _ = completed.recv(); + }); + let guard = ServerProcessShutdownGuard { + socket_path: socket_path.clone(), + shutdown_watch, + cleanup, + completion: Some(completion), + worker: Some(worker), + }; + + drop(guard); + + let client = UnixStream::connect(&socket_path) + .expect("incomplete server guard released its control socket"); + drop(client); + drop(listener); + cmux_tui_core::server::cleanup(&socket_path); + } + #[test] fn surface_only_attach_does_not_publish_session_default_colors() { let mux = Mux::new("surface-only-color-test", SurfaceOptions::default()); From e22036f3bcf718be0f171ea77fd8844bf2944c8e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:43:18 -0700 Subject: [PATCH 121/521] fix(tui): retain server ownership through cleanup failure --- cmux-tui/crates/cmux-tui/src/main.rs | 96 +++++++++++++++++++--------- 1 file changed, 65 insertions(+), 31 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index fa45d9d62c04..cff08a8a4b97 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -62,6 +62,8 @@ const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORIT // Once core cleanup owns no external processes, give the interactive driver // one final second to unwind before releasing the socket and exiting. const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1); +const SERVER_SHUTDOWN_RETRY_INITIAL: std::time::Duration = std::time::Duration::from_millis(100); +const SERVER_SHUTDOWN_RETRY_MAX: std::time::Duration = std::time::Duration::from_secs(5); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -943,10 +945,12 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result), +struct ServerShutdownCleanupState { + running: bool, + complete: bool, + blocked: bool, + next_attempt: std::time::Instant, + retry_delay: std::time::Duration, } struct ServerShutdownCleanup { @@ -959,37 +963,67 @@ impl ServerShutdownCleanup { fn new(mux: Arc) -> Self { Self { mux, - state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending), + state: std::sync::Mutex::new(ServerShutdownCleanupState { + running: false, + complete: false, + blocked: false, + next_attempt: std::time::Instant::now(), + retry_delay: SERVER_SHUTDOWN_RETRY_INITIAL, + }), changed: std::sync::Condvar::new(), } } - fn run(&self) -> anyhow::Result<()> { + fn run_until_complete(&self) { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); loop { - match &*state { - ServerShutdownCleanupState::Pending => { - *state = ServerShutdownCleanupState::Running; - break; + if state.complete { + return; + } + if state.blocked || state.running { + state = self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + continue; + } + let now = std::time::Instant::now(); + if state.next_attempt > now { + let wait = state.next_attempt.saturating_duration_since(now); + let waited = self + .changed + .wait_timeout(state, wait) + .unwrap_or_else(std::sync::PoisonError::into_inner); + state = waited.0; + continue; + } + state.running = true; + drop(state); + + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.mux.shutdown())); + state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.running = false; + match result { + Ok(Ok(())) => { + state.complete = true; + self.changed.notify_all(); + return; } - ServerShutdownCleanupState::Running => { - state = - self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + Ok(Err(_)) => { + state.next_attempt = std::time::Instant::now() + state.retry_delay; + state.retry_delay = (state.retry_delay * 2).min(SERVER_SHUTDOWN_RETRY_MAX); } - ServerShutdownCleanupState::Finished(result) => { - return result.clone().map_err(anyhow::Error::msg); + Err(_) => { + // A panic means cleanup invariants are unknown. Keep the + // socket and process ownership reserved without invoking + // the damaged path again. + state.blocked = true; } } + self.changed.notify_all(); } - drop(state); + } - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.mux.shutdown())) - .map_err(|_| "server shutdown cleanup panicked".to_string()) - .and_then(|result| result.map_err(|error| format!("{error:#}"))); - let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - *state = ServerShutdownCleanupState::Finished(result.clone()); - self.changed.notify_all(); - result.map_err(anyhow::Error::msg) + fn is_complete(&self) -> bool { + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).complete } } @@ -1019,9 +1053,7 @@ impl ServerProcessShutdownGuard { // same bounded cleanup path that run_server joins below, and // retain the socket indefinitely if cleanup cannot prove that // every external owner was released. - if worker_cleanup.run().is_err() { - return; - } + worker_cleanup.run_until_complete(); if matches!( completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) @@ -1043,8 +1075,8 @@ impl ServerProcessShutdownGuard { }) } - fn shutdown(&self) -> anyhow::Result<()> { - self.cleanup.run() + fn shutdown(&self) { + self.cleanup.run_until_complete(); } fn complete(mut self) { @@ -1052,6 +1084,9 @@ impl ServerProcessShutdownGuard { } fn finish(&mut self) { + if !self.cleanup.is_complete() { + return; + } let Some(completion) = self.completion.take() else { return }; cmux_tui_core::server::cleanup(&self.socket_path); let _ = completion.send(()); @@ -1231,9 +1266,8 @@ fn run_server( run_tui(Session::Local(mux), args.session, None) }; drop(websocket_server); - let shutdown_result = server_process.shutdown(); + server_process.shutdown(); server_process.complete(); - shutdown_result?; result } @@ -1502,7 +1536,7 @@ mod tests { let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); - *cleanup.state.lock().unwrap() = ServerShutdownCleanupState::Finished(Ok(())); + cleanup.state.lock().unwrap().complete = true; let (completion, completed) = std::sync::mpsc::channel(); let (replacement_tx, replacement_rx) = std::sync::mpsc::sync_channel(1); let replacement_path = socket_path.clone(); From 74125a8eafb6adf7ac01da03205153808d5343ee Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:17:03 -0700 Subject: [PATCH 122/521] test(tui): expose process inheritance during socket setup --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 58e8464f3a44..bf599ddf02a2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -890,6 +890,114 @@ pub mod transport { assert!(flags >= 0 && flags & libc::FD_CLOEXEC != 0); } + #[cfg(target_os = "macos")] + #[test] + fn deadline_connect_excludes_concurrent_browser_process_creation() { + use std::os::unix::fs::PermissionsExt as _; + use std::sync::Arc; + use std::sync::mpsc; + + let _guard = SOCKET_CREATED_TEST_LOCK.lock().unwrap(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = std::path::PathBuf::from("/tmp") + .join(format!("cmux-cxspawn-{}-{nonce}", std::process::id())); + std::fs::create_dir(&root).unwrap(); + let socket_path = root.join("server.sock"); + let listener = std::os::unix::net::UnixListener::bind(&socket_path).unwrap(); + let marker_path = root.join("descriptor-state"); + let browser_path = root.join("browser"); + let profile_path = root.join("profile"); + let (descriptor_sender, descriptor_receiver) = mpsc::sync_channel(1); + let (release_sender, release_receiver) = mpsc::sync_channel(1); + let release_receiver = Arc::new(std::sync::Mutex::new(release_receiver)); + imp::set_socket_created_hook(Some(Arc::new({ + let release_receiver = release_receiver.clone(); + move |descriptor| { + descriptor_sender.send(descriptor).unwrap(); + release_receiver + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(5)) + .expect("test did not release descriptor setup"); + } + }))); + + let connector = std::thread::spawn({ + let socket_path = socket_path.clone(); + move || connect_unix_until(&socket_path, Instant::now() + Duration::from_secs(5)) + }); + let descriptor = descriptor_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("connector did not expose its descriptor setup window"); + std::fs::write( + &browser_path, + format!( + "#!/bin/sh\n\ + if [ -e /dev/fd/{descriptor} ]; then\n\ + printf inherited > {}\n\ + else\n\ + printf closed > {}\n\ + fi\n\ + printf 'DevTools listening on ws://127.0.0.1:1/devtools/browser/test\\n' >&2\n\ + exec /bin/sleep 30\n", + marker_path.display(), + marker_path.display() + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&browser_path).unwrap().permissions(); + permissions.set_mode(0o700); + std::fs::set_permissions(&browser_path, permissions).unwrap(); + + let (launch_started_sender, launch_started_receiver) = mpsc::sync_channel(1); + let (launch_result_sender, launch_result_receiver) = mpsc::sync_channel(1); + let launcher = std::thread::spawn(move || { + launch_started_sender.send(()).unwrap(); + let result = + cmux_tui_cdp::Chrome::launch_with(&cmux_tui_cdp::ChromeLaunchOptions { + binary: browser_path, + mode: cmux_tui_cdp::BrowserMode::Headless, + user_data_dir: Some(profile_path), + ephemeral: false, + }) + .map(|browser| drop(browser)) + .map_err(|error| error.to_string()); + launch_result_sender.send(result).unwrap(); + }); + launch_started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + let launch_result_before_close_on_exec = + launch_result_receiver.recv_timeout(Duration::from_millis(500)).ok(); + let launched_before_close_on_exec = launch_result_before_close_on_exec.is_some(); + + release_sender.send(()).unwrap(); + let stream = connector.join().unwrap().unwrap(); + imp::set_socket_created_hook(None); + let (_accepted, _) = listener.accept().unwrap(); + let launch_result = launch_result_before_close_on_exec.unwrap_or_else(|| { + launch_result_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("browser process did not launch after descriptor setup completed") + }); + launcher.join().unwrap(); + let descriptor_state = std::fs::read_to_string(&marker_path).unwrap(); + + drop(stream); + drop(listener); + std::fs::remove_dir_all(&root).unwrap(); + launch_result.unwrap(); + assert!( + !launched_before_close_on_exec, + "browser process launched while the connector descriptor was still inheritable" + ); + assert_eq!( + descriptor_state, "closed", + "browser process inherited the connector descriptor" + ); + } + #[cfg(target_os = "linux")] #[test] fn deadline_connect_times_out_while_listener_backlog_is_saturated() { From 182aa2951f1112648c3ca69f2e53d214782d4d6d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:22:53 -0700 Subject: [PATCH 123/521] fix(tui): serialize process creation with socket setup --- cmux-tui/Cargo.lock | 7 ++ cmux-tui/Cargo.toml | 2 + cmux-tui/crates/cmux-tui-cdp/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 13 +- cmux-tui/crates/cmux-tui-core/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-core/src/platform.rs | 118 +++--------------- cmux-tui/crates/cmux-tui-core/src/surface.rs | 5 +- .../src/terminal_host_runtime.rs | 7 +- cmux-tui/crates/cmux-tui-process/Cargo.toml | 11 ++ cmux-tui/crates/cmux-tui-process/src/lib.rs | 41 ++++++ cmux-tui/crates/cmux-tui/Cargo.toml | 1 + cmux-tui/crates/cmux-tui/src/config.rs | 2 +- .../cmux-tui/src/machine_agent/transport.rs | 7 +- .../src/machine_provider_transport.rs | 2 +- .../crates/cmux-tui/src/machine_runtime.rs | 4 +- .../crates/cmux-tui/src/plugin_manager.rs | 7 +- .../crates/cmux-tui/src/server_lifecycle.rs | 2 +- 17 files changed, 109 insertions(+), 122 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-process/Cargo.toml create mode 100644 cmux-tui/crates/cmux-tui-process/src/lib.rs diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index dfba41dea6b2..088ace438444 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -230,6 +230,7 @@ dependencies = [ "cmux-tui-core", "cmux-tui-machine-agent-protocol", "cmux-tui-machine-protocol", + "cmux-tui-process", "crossterm", "getrandom 0.3.4", "ghostty-vt", @@ -251,6 +252,7 @@ version = "0.1.0" dependencies = [ "anyhow", "base64", + "cmux-tui-process", "hickory-resolver", "serde_json", "tokio", @@ -264,6 +266,7 @@ dependencies = [ "anyhow", "base64", "cmux-tui-cdp", + "cmux-tui-process", "fs4", "getrandom 0.3.4", "ghostty-vt", @@ -301,6 +304,10 @@ dependencies = [ "zeroize", ] +[[package]] +name = "cmux-tui-process" +version = "0.1.0" + [[package]] name = "combine" version = "4.6.7" diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index 5763d693d41e..9c5ec3d5e7d3 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/ghostty-vt", "crates/cmux-tui-cdp", "crates/cmux-tui-core", + "crates/cmux-tui-process", "crates/cmux-tui-machine-agent-protocol", "crates/cmux-tui-machine-protocol", "crates/cmux-tui", @@ -32,6 +33,7 @@ ghostty-vt-sys = { path = "crates/ghostty-vt-sys" } ghostty-vt = { path = "crates/ghostty-vt" } cmux-tui-core = { path = "crates/cmux-tui-core" } cmux-tui-cdp = { path = "crates/cmux-tui-cdp" } +cmux-tui-process = { path = "crates/cmux-tui-process" } cmux-tui-machine-agent-protocol = { path = "crates/cmux-tui-machine-agent-protocol" } cmux-tui-machine-protocol = { path = "crates/cmux-tui-machine-protocol" } anyhow = "1" diff --git a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml index d5296679d2ae..9cd588de1fd6 100644 --- a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] anyhow.workspace = true base64.workspace = true +cmux-tui-process.workspace = true hickory-resolver.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 14f735ce3239..c9f3cc0036ed 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -79,15 +79,14 @@ impl Chrome { let (profile_dir, profile_ephemeral) = profile_dir_for(options)?; std::fs::create_dir_all(&profile_dir)?; let mut command = Command::new(&options.binary); - command.args(chrome_args_for(&profile_dir, options.mode)); - let mut child = command + command + .args(chrome_args_for(&profile_dir, options.mode)) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| { - anyhow::anyhow!("failed to launch Chrome at {}: {e}", options.binary.display()) - })?; + .stderr(Stdio::piped()); + let mut child = cmux_tui_process::spawn(&mut command).map_err(|e| { + anyhow::anyhow!("failed to launch Chrome at {}: {e}", options.binary.display()) + })?; let stderr = child .stderr diff --git a/cmux-tui/crates/cmux-tui-core/Cargo.toml b/cmux-tui/crates/cmux-tui-core/Cargo.toml index a382992a8e7d..036b3ca76642 100644 --- a/cmux-tui/crates/cmux-tui-core/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-core/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] ghostty-vt.workspace = true cmux-tui-cdp.workspace = true +cmux-tui-process.workspace = true portable-pty.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index bf599ddf02a2..f48eb6d74684 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -78,77 +78,6 @@ pub mod transport { *SOCKET_CREATED_HOOK.get_or_init(Default::default).lock().unwrap() = hook; } - #[cfg(target_os = "macos")] - static SOCKET_FORK_BARRIER_REGISTRATION: std::sync::OnceLock = - std::sync::OnceLock::new(); - #[cfg(target_os = "macos")] - static mut SOCKET_FORK_BARRIER: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER; - - #[cfg(target_os = "macos")] - unsafe extern "C" fn socket_fork_prepare() { - // SAFETY: pthread_atfork calls this handler before fork. Blocking - // here serializes fork with the short descriptor setup window. - let _ = unsafe { libc::pthread_mutex_lock(&raw mut SOCKET_FORK_BARRIER) }; - } - - #[cfg(target_os = "macos")] - unsafe extern "C" fn socket_fork_parent() { - // SAFETY: the prepare handler acquired this mutex in the forking - // thread, and the parent handler releases that same acquisition. - let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; - } - - #[cfg(target_os = "macos")] - unsafe extern "C" fn socket_fork_child() { - // SAFETY: the child inherits the prepare handler's locked mutex - // and releases it before any post-fork work can continue. - let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; - } - - #[cfg(target_os = "macos")] - struct SocketForkBarrierGuard; - - #[cfg(target_os = "macos")] - impl SocketForkBarrierGuard { - fn acquire() -> io::Result { - let registration = *SOCKET_FORK_BARRIER_REGISTRATION.get_or_init(|| { - // SAFETY: callbacks have static lifetimes and use one - // statically initialized pthread mutex. - unsafe { - libc::pthread_atfork( - Some(socket_fork_prepare), - Some(socket_fork_parent), - Some(socket_fork_child), - ) - } - }); - if registration != 0 { - return Err(io::Error::new( - io::ErrorKind::Unsupported, - format!( - "cannot register close-on-exec fork barrier: {}", - io::Error::from_raw_os_error(registration) - ), - )); - } - // SAFETY: this mutex is statically initialized and remains - // live for the process lifetime. - let locked = unsafe { libc::pthread_mutex_lock(&raw mut SOCKET_FORK_BARRIER) }; - if locked != 0 { - return Err(io::Error::from_raw_os_error(locked)); - } - Ok(Self) - } - } - - #[cfg(target_os = "macos")] - impl Drop for SocketForkBarrierGuard { - fn drop(&mut self) { - // SAFETY: each guard exists only after one successful lock. - let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; - } - } - #[cfg(target_os = "linux")] fn create_close_on_exec_socket() -> io::Result { // SAFETY: socket has no pointer arguments and returns a new owned @@ -181,10 +110,10 @@ pub mod transport { #[cfg(target_os = "macos")] fn create_close_on_exec_socket() -> io::Result { - let _fork_barrier = SocketForkBarrierGuard::acquire()?; - // macOS has no SOCK_CLOEXEC socket flag. The registered atfork - // barrier prevents every fork in this process until fcntl marks - // the fresh descriptor close-on-exec. + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); + // macOS has no SOCK_CLOEXEC socket flag. Every cmux-tui process + // launch shares this barrier, so no child can start until fcntl + // marks the fresh descriptor close-on-exec. // SAFETY: socket has no pointer arguments and returns a new owned // descriptor on success. let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; @@ -205,7 +134,7 @@ pub mod transport { if descriptor_flags < 0 { return Err(io::Error::last_os_error()); } - // SAFETY: F_SETFD updates flags while the fork barrier excludes + // SAFETY: F_SETFD updates flags while the process barrier excludes // process creation from this setup window. if unsafe { libc::fcntl( @@ -228,20 +157,6 @@ pub mod transport { )) } - #[cfg(all(target_os = "macos", test))] - pub(super) fn socket_fork_barrier_is_locked_for_test() -> bool { - // SAFETY: trylock only inspects the statically initialized mutex. - let result = unsafe { libc::pthread_mutex_trylock(&raw mut SOCKET_FORK_BARRIER) }; - if result == 0 { - // SAFETY: a successful trylock created one balanced test - // acquisition that must be released here. - let _ = unsafe { libc::pthread_mutex_unlock(&raw mut SOCKET_FORK_BARRIER) }; - false - } else { - result == libc::EBUSY - } - } - pub(super) fn listen(path: &Path) -> io::Result { UnixListener::bind(path).map(|inner| Listener { inner }) } @@ -850,7 +765,7 @@ pub mod transport { #[cfg(target_os = "macos")] #[test] - fn deadline_connect_holds_fork_barrier_until_close_on_exec() { + fn deadline_connect_confines_inheritable_descriptor_to_process_barrier() { use std::os::fd::AsRawFd as _; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -865,12 +780,15 @@ pub mod transport { .as_nanos() )); let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - let fork_barrier_held = Arc::new(AtomicBool::new(false)); + let descriptor_was_inheritable = Arc::new(AtomicBool::new(false)); imp::set_socket_created_hook(Some(Arc::new({ - let fork_barrier_held = fork_barrier_held.clone(); - move |_| { - fork_barrier_held - .store(imp::socket_fork_barrier_is_locked_for_test(), Ordering::Release); + let descriptor_was_inheritable = descriptor_was_inheritable.clone(); + move |descriptor| { + // SAFETY: the hook receives the fresh live socket before + // the process barrier is released. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFD) }; + descriptor_was_inheritable + .store(flags >= 0 && flags & libc::FD_CLOEXEC == 0, Ordering::Release); } }))); @@ -884,8 +802,8 @@ pub mod transport { drop(listener); std::fs::remove_file(path).unwrap(); assert!( - fork_barrier_held.load(Ordering::Acquire), - "deadline connector exposed its socket outside the fork barrier" + descriptor_was_inheritable.load(Ordering::Acquire), + "test did not observe the non-atomic macOS descriptor setup window" ); assert!(flags >= 0 && flags & libc::FD_CLOEXEC != 0); } @@ -914,7 +832,6 @@ pub mod transport { let (release_sender, release_receiver) = mpsc::sync_channel(1); let release_receiver = Arc::new(std::sync::Mutex::new(release_receiver)); imp::set_socket_created_hook(Some(Arc::new({ - let release_receiver = release_receiver.clone(); move |descriptor| { descriptor_sender.send(descriptor).unwrap(); release_receiver @@ -926,7 +843,6 @@ pub mod transport { }))); let connector = std::thread::spawn({ - let socket_path = socket_path.clone(); move || connect_unix_until(&socket_path, Instant::now() + Duration::from_secs(5)) }); let descriptor = descriptor_receiver @@ -963,7 +879,7 @@ pub mod transport { user_data_dir: Some(profile_path), ephemeral: false, }) - .map(|browser| drop(browser)) + .map(drop) .map_err(|error| error.to_string()); launch_result_sender.send(result).unwrap(); }); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 67122c3ec83f..16f120856de6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1345,7 +1345,10 @@ impl Surface { cmd.cwd(cwd); } - let child = pty.slave.spawn_command(cmd)?; + let child = { + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); + pty.slave.spawn_command(cmd)? + }; let pid = child.process_id(); drop(pty.slave); let killer = child.clone_killer(); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 113ebb13f874..9d0501bd1e93 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1167,7 +1167,7 @@ mod unix { } let reaper = reserve_host_process_reaper() .context("reserve bounded terminal-host process cleanup")?; - let child = command.spawn().context("spawn terminal-host process")?; + let child = cmux_tui_process::spawn(&mut command).context("spawn terminal-host process")?; let mut process = SpawnedHostProcess::with_reaper(child, reaper); let host_pid = process.child_mut().id(); let stdin = @@ -3012,7 +3012,10 @@ mod unix { if let Some(cwd) = launch.cwd.as_deref() { command.cwd(cwd); } - let mut child = pty.slave.spawn_command(command)?; + let mut child = { + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); + pty.slave.spawn_command(command)? + }; let pid = child.process_id(); drop(pty.slave); let killer = child.clone_killer(); diff --git a/cmux-tui/crates/cmux-tui-process/Cargo.toml b/cmux-tui/crates/cmux-tui-process/Cargo.toml new file mode 100644 index 000000000000..6c4dae12fb3b --- /dev/null +++ b/cmux-tui/crates/cmux-tui-process/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "cmux-tui-process" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "Process creation coordination for cmux-tui" + +[lints] +workspace = true diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs new file mode 100644 index 000000000000..073d7f511f38 --- /dev/null +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -0,0 +1,41 @@ +//! Process creation coordination shared by every cmux-tui runtime crate. + +use std::io; +use std::process::{Child, Command}; + +#[cfg(target_os = "macos")] +static PROCESS_CREATION_BARRIER: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Holds the process-wide child-creation barrier. +/// +/// macOS cannot create Unix sockets with close-on-exec set atomically. Socket +/// setup holds this guard until `FD_CLOEXEC` is set, while every child launch +/// holds the same guard until the child exists. Library-backed process launch +/// paths, such as PTY creation, must acquire this guard around their spawn +/// call. +#[must_use = "dropping the guard releases process creation"] +pub struct ProcessCreationGuard { + #[cfg(target_os = "macos")] + _guard: std::sync::MutexGuard<'static, ()>, +} + +impl ProcessCreationGuard { + pub fn acquire() -> Self { + #[cfg(target_os = "macos")] + { + let guard = + PROCESS_CREATION_BARRIER.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + Self { _guard: guard } + } + #[cfg(not(target_os = "macos"))] + { + Self {} + } + } +} + +/// Spawn a child while excluding non-atomic close-on-exec descriptor setup. +pub fn spawn(command: &mut Command) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + command.spawn() +} diff --git a/cmux-tui/crates/cmux-tui/Cargo.toml b/cmux-tui/crates/cmux-tui/Cargo.toml index 4981868ab4f3..c1b21a0e27e0 100644 --- a/cmux-tui/crates/cmux-tui/Cargo.toml +++ b/cmux-tui/crates/cmux-tui/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] ghostty-vt.workspace = true cmux-tui-core.workspace = true +cmux-tui-process.workspace = true cmux-tui-machine-agent-protocol.workspace = true cmux-tui-machine-protocol.workspace = true ratatui.workspace = true diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 48d7823f7db4..fa7098b743aa 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -2074,7 +2074,7 @@ fn run_ghostty_show_config(installation: &platform::GhosttyInstallation) -> Opti if let Some(resources_dir) = installation.resources_dir.as_deref() { command.env("GHOSTTY_RESOURCES_DIR", resources_dir); } - let mut child = command.spawn().ok()?; + let mut child = cmux_tui_process::spawn(&mut command).ok()?; let deadline = Instant::now() + Duration::from_secs(2); let status = loop { match child.try_wait().ok()? { diff --git a/cmux-tui/crates/cmux-tui/src/machine_agent/transport.rs b/cmux-tui/crates/cmux-tui/src/machine_agent/transport.rs index 930f92fb57b7..c74f8f8e7a93 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_agent/transport.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_agent/transport.rs @@ -101,12 +101,13 @@ impl SshCloudConnector { impl CloudConnector for SshCloudConnector { fn connect(&self) -> io::Result { - let mut child = Command::new("ssh") + let mut command = Command::new("ssh"); + command .args(self.command_args()) .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .spawn()?; + .stderr(Stdio::inherit()); + let mut child = cmux_tui_process::spawn(&mut command)?; let writer = child.stdin.take().ok_or_else(|| io::Error::other("OpenSSH did not expose stdin"))?; let reader = diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs index 0dd5e7c8ab82..9c0ca3978f79 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs @@ -509,7 +509,7 @@ fn spawn_command( .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let mut child = command.spawn().map_err(|error| { + let mut child = cmux_tui_process::spawn(&mut command).map_err(|error| { io::Error::new(error.kind(), format!("failed to start machine-provider command: {error}")) })?; let stdin = child diff --git a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs index a03932b06912..e72c1e673b19 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs @@ -275,8 +275,8 @@ fn spawn_transport_process( .map_err(|error| anyhow::anyhow!("cannot create ssh diagnostics cancellation: {error}"))?; #[cfg(unix)] command.process_group(0); - let mut child = - command.spawn().map_err(|error| anyhow::anyhow!("cannot start ssh: {error}"))?; + let mut child = cmux_tui_process::spawn(command) + .map_err(|error| anyhow::anyhow!("cannot start ssh: {error}"))?; #[cfg(unix)] let process_group = match libc::pid_t::try_from(child.id()) { Ok(process_group) => process_group, diff --git a/cmux-tui/crates/cmux-tui/src/plugin_manager.rs b/cmux-tui/crates/cmux-tui/src/plugin_manager.rs index f7482f8860f6..7e50a7b10826 100644 --- a/cmux-tui/crates/cmux-tui/src/plugin_manager.rs +++ b/cmux-tui/crates/cmux-tui/src/plugin_manager.rs @@ -358,8 +358,9 @@ fn installed_name( fn run_build_if_needed(manifest: &PluginManifest, dir: &Path) -> anyhow::Result<()> { let Some(build) = &manifest.build else { return Ok(()) }; - let status = - Command::new(&build.command[0]).args(&build.command[1..]).current_dir(dir).status()?; + let mut command = Command::new(&build.command[0]); + command.args(&build.command[1..]).current_dir(dir); + let status = cmux_tui_process::spawn(&mut command)?.wait()?; if !status.success() { anyhow::bail!("build command failed with status {status}"); } @@ -406,7 +407,7 @@ fn run_git( if let Some(dir) = current_dir { command.current_dir(dir); } - let status = command.status()?; + let status = cmux_tui_process::spawn(&mut command)?.wait()?; if !status.success() { anyhow::bail!("git failed with status {status}"); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 2ab60d4f38cc..3df18d469513 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -743,7 +743,7 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R Ok(()) }); } - let helper = command.spawn().map_err(|_| { + let helper = cmux_tui_process::spawn(&mut command).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; let deadline = Instant::now() + LEGACY_SHUTDOWN_TIMEOUT + LEGACY_HELPER_WAIT_MARGIN; From 868c25d68e1740d900fe44b67747c6f7cdbf0569 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:05:48 -0700 Subject: [PATCH 124/521] test(tui): expose concurrent legacy surface creation --- cmux-tui/crates/cmux-tui/tests/cli.rs | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 29834bc1514f..0fc3a157f25b 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -334,6 +334,7 @@ fn legacy_server_process_helper() { | "applied-close-disconnect" | "exit-after-close" | "persistent-close-error" + | "concurrent-client" | "browser-surface" ) { let mut command = Command::new("yes"); @@ -515,6 +516,31 @@ fn legacy_server_process_helper() { } } let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); + if list_request["cmd"].as_str() == Some("list-clients") { + let mut clients = vec![serde_json::json!({ + "client": 1, + "transport": "unix", + "self": true, + })]; + if scenario == "concurrent-client" { + clients.push(serde_json::json!({ + "client": 2, + "transport": "unix", + "self": false, + })); + } + writeln!( + stream, + "{}", + serde_json::json!({ + "id": list_request["id"], + "ok": true, + "data": clients, + }) + ) + .unwrap(); + continue; + } assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); if scenario == "cleanup-failure" { writeln!( @@ -539,6 +565,7 @@ fn legacy_server_process_helper() { | "applied-close-disconnect" | "exit-after-close" | "persistent-close-error" + | "concurrent-client" | "zombie-child" | "reparent-on-close" | "unowned-surface" @@ -1582,6 +1609,29 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert!(!process_exists(descendant_pid)); } +#[cfg(unix)] +#[test] +fn server_stop_fails_closed_when_another_legacy_client_can_create_surfaces() { + let mut server = + LegacyServerProcess::start("legacy-server-concurrent-client", "concurrent-client", None); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(1)); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("could not close pane processes before stopping the older server") + ); + assert!(server.child.try_wait().unwrap().is_none()); + assert!(process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn server_stop_closes_exited_legacy_placeholders_without_a_pid() { From f35207dff54c4a07a95da0adc2e57f435a41dd15 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:15:05 -0700 Subject: [PATCH 125/521] fix(tui): fence legacy surface creation during stop --- .../crates/cmux-tui/src/server_lifecycle.rs | 352 +++++++++++++++--- .../src/server_lifecycle/legacy_process.rs | 44 ++- cmux-tui/crates/cmux-tui/tests/cli.rs | 4 +- 3 files changed, 335 insertions(+), 65 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 3df18d469513..050bfd47955a 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -24,8 +24,8 @@ mod legacy_process; use legacy_process::terminate_process_tree; #[cfg(unix)] use legacy_process::{ - CapturedProcessSession, CapturedProcessTree, ProcessIdentity, capture_process_session, - capture_process_tree_until, + CapturedProcessSession, CapturedProcessTree, FrozenProcessTree, ProcessIdentity, + capture_process_session, capture_process_tree_until, freeze_process_tree_until, }; const PROBE_REQUEST_ID: u64 = 0; @@ -61,10 +61,176 @@ type TransportReader = BufReader>; #[cfg(unix)] static LEGACY_HELPER_CANCELLED: AtomicBool = AtomicBool::new(false); #[cfg(unix)] +static LEGACY_SOCKET_QUARANTINE_SEQUENCE: AtomicUsize = AtomicUsize::new(1); +#[cfg(unix)] static LEGACY_HELPER_REAPER: OnceLock> = OnceLock::new(); #[cfg(all(unix, test))] static LEGACY_HELPER_REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); +#[cfg(unix)] +struct LegacySocketQuarantine { + original: PathBuf, + quarantined: PathBuf, + device: u64, + inode: u64, + armed: bool, +} + +#[cfg(unix)] +impl LegacySocketQuarantine { + fn acquire(original: &Path) -> std::io::Result { + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; + + let metadata = std::fs::symlink_metadata(original)?; + if !metadata.file_type().is_socket() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy control path is not a socket", + )); + } + let parent = original.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "legacy control socket has no parent directory", + ) + })?; + for _ in 0..64 { + let sequence = LEGACY_SOCKET_QUARANTINE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let quarantined = parent.join(format!(".cs-{:x}-{sequence:x}", std::process::id())); + match std::fs::hard_link(original, &quarantined) { + Ok(()) => { + if let Err(error) = std::fs::remove_file(original) { + let _ = std::fs::remove_file(&quarantined); + return Err(error); + } + let moved = std::fs::symlink_metadata(&quarantined)?; + if moved.dev() != metadata.dev() + || moved.ino() != metadata.ino() + || !moved.file_type().is_socket() + { + let _ = std::fs::hard_link(&quarantined, original); + let _ = std::fs::remove_file(&quarantined); + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy control socket identity changed during quarantine", + )); + } + return Ok(Self { + original: original.to_path_buf(), + quarantined, + device: metadata.dev(), + inode: metadata.ino(), + armed: true, + }); + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error), + } + } + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not reserve a private legacy control socket path", + )) + } + + fn adopt( + original: PathBuf, + quarantined: PathBuf, + device: u64, + inode: u64, + ) -> std::io::Result { + let quarantine = Self { original, quarantined, device, inode, armed: true }; + if quarantine.matches(&quarantine.quarantined)? != Some(true) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "private legacy control socket identity changed", + )); + } + Ok(quarantine) + } + + fn path(&self) -> &Path { + &self.quarantined + } + + fn matches(&self, path: &Path) -> std::io::Result> { + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; + + match std::fs::symlink_metadata(path) { + Ok(metadata) => Ok(Some( + metadata.file_type().is_socket() + && metadata.dev() == self.device + && metadata.ino() == self.inode, + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error), + } + } + + fn restore(&mut self) -> std::io::Result<()> { + if !self.armed { + return Ok(()); + } + match (self.matches(&self.original)?, self.matches(&self.quarantined)?) { + (Some(true), Some(true)) => { + std::fs::remove_file(&self.quarantined)?; + } + (Some(true), None) => {} + (None, Some(true)) => { + std::fs::hard_link(&self.quarantined, &self.original)?; + std::fs::remove_file(&self.quarantined)?; + } + (Some(false), _) | (_, Some(false)) => { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "legacy control socket path was replaced during shutdown", + )); + } + (None, None) => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "legacy control socket disappeared during shutdown", + )); + } + } + self.armed = false; + Ok(()) + } + + fn commit(&mut self) -> std::io::Result<()> { + if !self.armed { + return Ok(()); + } + for path in [&self.quarantined, &self.original] { + match self.matches(path)? { + Some(true) => std::fs::remove_file(path)?, + Some(false) => { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "legacy control socket path was replaced during shutdown", + )); + } + None => {} + } + } + self.armed = false; + Ok(()) + } +} + +#[cfg(unix)] +impl Drop for LegacySocketQuarantine { + fn drop(&mut self) { + let _ = self.restore(); + } +} + +#[cfg(unix)] +struct LegacySurfaceCleanup { + owners: Vec, + frozen_server: Option, +} + #[cfg(unix)] struct LegacyHelperReaper { pending: Arc<(Mutex>, Condvar)>, @@ -500,9 +666,15 @@ impl ServerLifecycle { #[cfg(unix)] fn stop_legacy_server(self) -> anyhow::Result<()> { let process = verified_legacy_process(self.probe.identity.pid, self.peer_process_id)?; - let result = run_detached_legacy_stop(&self.path, process); + // Preserve the exact socket inode at a private sibling path, then + // remove the public name before starting the detached helper. Existing + // streams stay valid, while no new caller can enter the legacy + // mutation protocol through the advertised path. + let quarantine = LegacySocketQuarantine::acquire(&self.path).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; drop(self.reader); - result + run_detached_legacy_stop(quarantine, process) } #[cfg(not(unix))] @@ -516,7 +688,7 @@ impl ServerLifecycle { expected: ProcessIdentity, captured: &CapturedProcessTree, deadline: Instant, - ) -> anyhow::Result> { + ) -> anyhow::Result { let mut next_request_id = LEGACY_LIST_REQUEST_ID; let mut consecutive_empty_scans = 0; let mut owners = Vec::::new(); @@ -527,6 +699,7 @@ impl ServerLifecycle { if Instant::now() >= deadline { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } + self.require_exclusive_legacy_client(&mut next_request_id, deadline)?; let closed = match self.close_legacy_surface_snapshot( &mut next_request_id, expected, @@ -546,7 +719,7 @@ impl ServerLifecycle { let verified_exit = phase == LegacyConnectionPhase::ClosingVerifiedSnapshot || (phase == LegacyConnectionPhase::Listing && previous_snapshot_completed); if verified_exit && legacy_server_retired(expected, &self.path)? { - return Ok(owners); + return Ok(LegacySurfaceCleanup { owners, frozen_server: None }); } if let Err(reconnect_error) = self.reconnect_legacy_server(expected, deadline) { if verified_exit @@ -554,7 +727,7 @@ impl ServerLifecycle { expected, &self.path, deadline, )? { - return Ok(owners); + return Ok(LegacySurfaceCleanup { owners, frozen_server: None }); } return Err(reconnect_error); } @@ -565,9 +738,20 @@ impl ServerLifecycle { }; previous_snapshot_completed = true; if closed == 0 { + self.require_exclusive_legacy_client(&mut next_request_id, deadline)?; consecutive_empty_scans += 1; if consecutive_empty_scans == LEGACY_STABLE_EMPTY_SCANS { - return Ok(owners); + // Stop the verified server before accepting the final + // empty snapshot. The frozen tree records every remaining + // PTY session while no connection handler can create or + // reparent another surface. + let frozen_server = + freeze_process_tree_until(expected, deadline).map_err(|_| { + anyhow::anyhow!( + crate::localization::catalog().server.legacy_cleanup_failed + ) + })?; + return Ok(LegacySurfaceCleanup { owners, frozen_server: Some(frozen_server) }); } } else { consecutive_empty_scans = 0; @@ -576,6 +760,39 @@ impl ServerLifecycle { anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) } + #[cfg(unix)] + fn require_exclusive_legacy_client( + &mut self, + next_request_id: &mut u64, + deadline: Instant, + ) -> anyhow::Result<()> { + let request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + write_json_line(self.reader.get_mut(), &json!({"id": request_id, "cmd": "list-clients"})) + .map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let response = + read_response_until(&mut self.reader, request_id, deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let clients = response_data(&response)?.as_array().ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let exclusive = clients.len() == 1 + && clients[0].get("self").and_then(Value::as_bool) == Some(true) + && clients[0].get("transport").and_then(Value::as_str) == Some("unix"); + // The public listener name is already quarantined. Refusing every + // previously accepted peer leaves this helper as the only process + // capable of mutating legacy topology before the final freeze. + if !exclusive { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + Ok(()) + } + #[cfg(unix)] fn reconnect_legacy_server( &mut self, @@ -718,7 +935,10 @@ fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { } #[cfg(unix)] -fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::Result<()> { +fn run_detached_legacy_stop( + mut quarantine: LegacySocketQuarantine, + expected: ProcessIdentity, +) -> anyhow::Result<()> { use std::os::unix::process::CommandExt; let reaper = reserve_legacy_helper_reaper()?; @@ -728,9 +948,12 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R let mut command = Command::new(executable); command .arg("__legacy-stop-helper") - .arg(path) + .arg(&quarantine.original) + .arg(&quarantine.quarantined) .arg(expected.pid().to_string()) .arg(expected.started_at().to_string()) + .arg(quarantine.device.to_string()) + .arg(quarantine.inode.to_string()) .arg(LEGACY_SHUTDOWN_TIMEOUT.as_millis().to_string()) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -749,8 +972,14 @@ fn run_detached_legacy_stop(path: &Path, expected: ProcessIdentity) -> anyhow::R let deadline = Instant::now() + LEGACY_SHUTDOWN_TIMEOUT + LEGACY_HELPER_WAIT_MARGIN; let status = wait_for_child_until(helper, reaper, deadline)?; if status.success() { + quarantine.commit().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; return Ok(()); } + quarantine.restore().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) } @@ -806,7 +1035,16 @@ fn wait_for_child_until( #[cfg(unix)] pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { install_legacy_helper_signal_handler(); - let [path, expected_pid, expected_started_at, timeout_ms] = args else { + let [ + original_path, + quarantined_path, + expected_pid, + expected_started_at, + device, + inode, + timeout_ms, + ] = args + else { anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); }; let expected_pid = @@ -816,6 +1054,12 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { let expected_started_at = expected_started_at .parse::() .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; + let device = device + .parse::() + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; + let inode = inode + .parse::() + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; let timeout = timeout_ms .parse::() .ok() @@ -826,31 +1070,61 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { })?; let deadline = Instant::now() + timeout; ensure_legacy_helper_active()?; + let mut quarantine = LegacySocketQuarantine::adopt( + PathBuf::from(original_path), + PathBuf::from(quarantined_path), + device, + inode, + ) + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed))?; let expected = ProcessIdentity::from_parts(expected_pid, expected_started_at); - let mut lifecycle = ServerLifecycle::connect_until(PathBuf::from(path), deadline)?; - if lifecycle.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { - anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); - } - let actual = verified_legacy_process(lifecycle.probe.identity.pid, lifecycle.peer_process_id)?; - if actual != expected { - anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); - } - let captured = capture_legacy_process_tree(actual, deadline)?; + let result = (|| { + let mut lifecycle = + ServerLifecycle::connect_until(quarantine.path().to_path_buf(), deadline)?; + if lifecycle.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + } + let actual = + verified_legacy_process(lifecycle.probe.identity.pid, lifecycle.peer_process_id)?; + if actual != expected { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + } + let captured = capture_legacy_process_tree(actual, deadline)?; - ensure_legacy_helper_active()?; - let owners = - lifecycle.close_legacy_surfaces_until_stable(actual, &captured, deadline).map_err( - |_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed), - )?; - for owner in owners { ensure_legacy_helper_active()?; - owner.terminate_until(deadline).map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; + let cleanup = + lifecycle.close_legacy_surfaces_until_stable(actual, &captured, deadline).map_err( + |_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed), + )?; + drop(captured); + for owner in cleanup.owners { + ensure_legacy_helper_active()?; + owner.terminate_until(deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + } + if let Some(server) = cleanup.frozen_server { + ensure_legacy_helper_active()?; + server.terminate_until(deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed) + })?; + } + wait_for_disconnect_until(&mut lifecycle.reader, &lifecycle.path, deadline) + })(); + match result { + Ok(()) => { + quarantine.commit().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + Ok(()) + } + Err(error) => { + quarantine.restore().map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + Err(error) + } } - ensure_legacy_helper_active()?; - terminate_captured_legacy_process_tree(captured, deadline)?; - wait_for_disconnect_until(&mut lifecycle.reader, &lifecycle.path, deadline) } #[cfg(unix)] @@ -886,8 +1160,8 @@ fn legacy_process_exited(expected: ProcessIdentity) -> anyhow::Result { } #[cfg(unix)] -fn legacy_server_retired(expected: ProcessIdentity, path: &Path) -> anyhow::Result { - Ok(!path.exists() && legacy_process_exited(expected)?) +fn legacy_server_retired(expected: ProcessIdentity, _path: &Path) -> anyhow::Result { + legacy_process_exited(expected) } #[cfg(unix)] @@ -922,16 +1196,6 @@ fn capture_legacy_process_tree( .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) } -#[cfg(unix)] -fn terminate_captured_legacy_process_tree( - process: CapturedProcessTree, - deadline: Instant, -) -> anyhow::Result<()> { - process - .terminate_until(deadline) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_signal_failed)) -} - pub(crate) fn validate_local_identity(data: &Value, path: &Path) -> anyhow::Result<()> { let probe = ServerProbe { identity: ServerIdentity::from_protocol_data(data)? }; probe.require_compatible(path) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 21e1292f58cb..46fb9abdcf4f 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -165,16 +165,18 @@ pub(super) fn terminate_process_tree(process: ProcessIdentity) -> io::Result<()> terminate_process_tree_until(process, Instant::now() + PROCESS_TREE_RETRY_TIMEOUT) } +#[cfg(test)] pub(super) fn terminate_process_tree_until( process: ProcessIdentity, deadline: Instant, ) -> io::Result<()> { retry_process_tree_termination(process, deadline, |_| { let tree = FrozenProcessTree::freeze(process, deadline)?; - tree.kill() + tree.terminate_until(deadline) }) } +#[cfg(test)] fn retry_process_tree_termination( process: ProcessIdentity, deadline: Instant, @@ -209,13 +211,13 @@ pub(super) fn capture_process_tree_until( process: ProcessIdentity, deadline: Instant, ) -> io::Result { - let tree = freeze_process_tree(process, deadline)?; - let sessions = tree.captured_sessions()?; + let tree = freeze_process_tree_until(process, deadline)?; + let sessions = tree.sessions.clone(); drop(tree); - Ok(CapturedProcessTree { root: process, sessions }) + Ok(CapturedProcessTree { sessions }) } -fn freeze_process_tree( +pub(super) fn freeze_process_tree_until( process: ProcessIdentity, deadline: Instant, ) -> io::Result { @@ -248,7 +250,6 @@ fn freeze_process_tree( } pub(super) struct CapturedProcessTree { - root: ProcessIdentity, sessions: Vec, } @@ -256,17 +257,6 @@ impl CapturedProcessTree { fn contains_process(&self, process: ProcessIdentity) -> bool { self.sessions.iter().any(|session| session.members.contains(&process)) } - - pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { - for session in &self.sessions { - if !session.kill_until_empty(deadline)? { - return Err(io::Error::other( - "captured PTY session did not exit before the legacy shutdown deadline", - )); - } - } - terminate_process_tree_until(self.root, deadline) - } } pub(super) struct CapturedProcessSession(CapturedSession); @@ -346,6 +336,7 @@ pub(super) fn capture_process_session( captured.map(Some) } +#[derive(Clone)] struct CapturedSession { id: libc::pid_t, members: Vec, @@ -416,9 +407,10 @@ impl SessionProbe { } } -struct FrozenProcessTree { +pub(super) struct FrozenProcessTree { root: ProcessFence, descendants: Vec, + sessions: Vec, } impl FrozenProcessTree { @@ -430,7 +422,7 @@ impl FrozenProcessTree { "server exited before its process tree was fenced", )); }; - let mut tree = Self { root, descendants: Vec::new() }; + let mut tree = Self { root, descendants: Vec::new(), sessions: Vec::new() }; let helper_pid = libc::pid_t::try_from(std::process::id()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid helper process id"))?; @@ -469,13 +461,14 @@ impl FrozenProcessTree { } } if !added { + tree.sessions = tree.current_sessions()?; return Ok(tree); } } Err(io::Error::other("legacy process tree did not stabilize")) } - fn captured_sessions(&self) -> io::Result> { + fn current_sessions(&self) -> io::Result> { // SAFETY: getsid only queries live, exact processes held stopped by // this FrozenProcessTree. let root_session = unsafe { libc::getsid(self.root.identity.pid) }; @@ -512,6 +505,17 @@ impl FrozenProcessTree { Ok(sessions) } + pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { + for session in &self.sessions { + if !session.kill_until_empty(deadline)? { + return Err(io::Error::other( + "frozen PTY session did not exit before the legacy shutdown deadline", + )); + } + } + self.kill() + } + fn kill(mut self) -> io::Result<()> { for process in self.descendants.iter_mut().rev() { process.kill()?; diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 0fc3a157f25b..8e5fb99b780c 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -658,7 +658,7 @@ fn legacy_server_process_helper() { drop(reader); drop(stream); drop(listener); - fs::remove_file(&socket).unwrap(); + let _ = fs::remove_file(&socket); return; } if scenario == "applied-close-disconnect" { @@ -1630,6 +1630,7 @@ fn server_stop_fails_closed_when_another_legacy_client_can_create_surfaces() { ); assert!(server.child.try_wait().unwrap().is_none()); assert!(process_exists(descendant_pid)); + assert!(transport::connect(&server.socket).is_ok(), "failed shutdown did not restore socket"); } #[cfg(unix)] @@ -1871,6 +1872,7 @@ fn server_stop_refuses_when_a_legacy_close_error_leaves_the_surface_present() { ); assert!(server.child.try_wait().unwrap().is_none()); assert!(process_exists(descendant_pid)); + assert!(transport::connect(&server.socket).is_ok(), "failed shutdown did not restore socket"); } #[cfg(unix)] From 1cb3b949a417492c84380f522359db69ef97f904 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:52:59 -0700 Subject: [PATCH 126/521] test(tui): expose legacy socket ownership races --- .../crates/cmux-tui/src/server_lifecycle.rs | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 050bfd47955a..b458a4a0ab75 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -79,6 +79,13 @@ struct LegacySocketQuarantine { #[cfg(unix)] impl LegacySocketQuarantine { fn acquire(original: &Path) -> std::io::Result { + Self::acquire_with_transfer_hook(original, || {}) + } + + fn acquire_with_transfer_hook( + original: &Path, + transfer_hook: impl FnOnce(), + ) -> std::io::Result { use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; let metadata = std::fs::symlink_metadata(original)?; @@ -99,6 +106,7 @@ impl LegacySocketQuarantine { let quarantined = parent.join(format!(".cs-{:x}-{sequence:x}", std::process::id())); match std::fs::hard_link(original, &quarantined) { Ok(()) => { + transfer_hook(); if let Err(error) = std::fs::remove_file(original) { let _ = std::fs::remove_file(&quarantined); return Err(error); @@ -1840,6 +1848,95 @@ mod tests { assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); } + #[cfg(unix)] + #[test] + fn legacy_quarantine_never_unlinks_a_replacement_socket() { + use std::os::unix::net::{UnixListener, UnixStream}; + + let path = PathBuf::from("/tmp").join(format!( + "cq-race-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + let original = UnixListener::bind(&path).unwrap(); + let mut replacement = None; + + let quarantine = LegacySocketQuarantine::acquire_with_transfer_hook(&path, || { + std::fs::remove_file(&path).unwrap(); + replacement = Some(UnixListener::bind(&path).unwrap()); + }); + let rejected_replacement = quarantine.is_err(); + let replacement_still_reachable = UnixStream::connect(&path).is_ok(); + + drop(quarantine); + drop(replacement); + drop(original); + let _ = std::fs::remove_file(&path); + + assert!(rejected_replacement, "quarantine accepted a replacement socket"); + assert!( + replacement_still_reachable, + "quarantine unlinked the replacement socket while transferring the verified inode" + ); + } + + #[cfg(unix)] + #[test] + fn legacy_helper_timeout_keeps_the_socket_quarantined_until_helper_exit() { + use std::io::BufRead as _; + use std::os::unix::net::{UnixListener, UnixStream}; + + let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); + let path = PathBuf::from("/tmp").join(format!( + "cq-help-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + let listener = UnixListener::bind(&path).unwrap(); + let quarantine = LegacySocketQuarantine::acquire(&path).unwrap(); + let mut helper = Command::new("/bin/sh") + .arg("-c") + .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut ready = String::new(); + BufReader::new(helper.stdout.take().unwrap()).read_line(&mut ready).unwrap(); + assert_eq!(ready, "ready\n"); + let helper_pid = libc::pid_t::try_from(helper.id()).unwrap(); + let baseline = active_legacy_helper_reapers_for_test(); + let reaper = reserve_legacy_helper_reaper().unwrap(); + + let error = + wait_for_child_until(helper, reaper, Instant::now() + Duration::from_millis(50)) + .unwrap_err(); + drop(quarantine); + let hidden_while_helper_alive = UnixStream::connect(&path).is_err(); + // SAFETY: helper_pid names this exact test-owned child. + unsafe { + libc::kill(helper_pid, libc::SIGKILL); + } + let deadline = Instant::now() + Duration::from_secs(1); + while active_legacy_helper_reapers_for_test() != baseline && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let restored_after_helper_exit = UnixStream::connect(&path).is_ok(); + + drop(listener); + let _ = std::fs::remove_file(&path); + + assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_cleanup_failed); + assert!( + hidden_while_helper_alive, + "timed-out helper released its admission fence before exit" + ); + assert!( + restored_after_helper_exit, + "helper reaper did not restore the socket after exact helper exit" + ); + } + #[cfg(unix)] #[test] fn legacy_reaper_releases_children_with_lost_wait_ownership() { From 4e4bdf03fd62845b1aaebb9c18bee5d3690caaab Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 07:59:58 -0700 Subject: [PATCH 127/521] fix(tui): transfer legacy socket ownership atomically --- .../crates/cmux-tui/src/server_lifecycle.rs | 291 +++++++++++++----- 1 file changed, 208 insertions(+), 83 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index b458a4a0ab75..4088368c6d91 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1,4 +1,6 @@ use std::collections::HashSet; +#[cfg(unix)] +use std::ffi::CString; use std::ffi::OsStr; use std::io::{BufRead, BufReader, Write}; use std::path::{Path, PathBuf}; @@ -67,6 +69,47 @@ static LEGACY_HELPER_REAPER: OnceLock> = OnceLock::ne #[cfg(all(unix, test))] static LEGACY_HELPER_REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); +#[cfg(unix)] +fn atomic_rename_noreplace(from: &Path, to: &Path) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt as _; + + let from = CString::new(from.as_os_str().as_bytes()).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "source path contains a NUL byte") + })?; + let to = CString::new(to.as_os_str().as_bytes()).map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "target path contains a NUL byte") + })?; + #[cfg(target_vendor = "apple")] + let result = unsafe { + libc::renameatx_np( + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_EXCL, + ) + }; + #[cfg(target_os = "linux")] + let result = unsafe { + libc::syscall( + libc::SYS_renameat2, + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + #[cfg(not(any(target_vendor = "apple", target_os = "linux")))] + let result = { + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "atomic no-replace rename is unavailable on this platform", + )); + }; + if result == 0 { Ok(()) } else { Err(std::io::Error::last_os_error()) } +} + #[cfg(unix)] struct LegacySocketQuarantine { original: PathBuf, @@ -101,44 +144,58 @@ impl LegacySocketQuarantine { "legacy control socket has no parent directory", ) })?; + let mut quarantined = None; for _ in 0..64 { let sequence = LEGACY_SOCKET_QUARANTINE_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let quarantined = parent.join(format!(".cs-{:x}-{sequence:x}", std::process::id())); - match std::fs::hard_link(original, &quarantined) { - Ok(()) => { - transfer_hook(); - if let Err(error) = std::fs::remove_file(original) { - let _ = std::fs::remove_file(&quarantined); - return Err(error); - } - let moved = std::fs::symlink_metadata(&quarantined)?; - if moved.dev() != metadata.dev() - || moved.ino() != metadata.ino() - || !moved.file_type().is_socket() - { - let _ = std::fs::hard_link(&quarantined, original); - let _ = std::fs::remove_file(&quarantined); - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "legacy control socket identity changed during quarantine", - )); - } - return Ok(Self { - original: original.to_path_buf(), - quarantined, - device: metadata.dev(), - inode: metadata.ino(), - armed: true, - }); + let candidate = parent.join(format!(".cs-{:x}-{sequence:x}", std::process::id())); + match std::fs::symlink_metadata(&candidate) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + quarantined = Some(candidate); + break; } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Ok(_) => {} Err(error) => return Err(error), } } - Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "could not reserve a private legacy control socket path", - )) + let quarantined = quarantined.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "could not reserve a private legacy control socket path", + ) + })?; + transfer_hook(); + atomic_rename_noreplace(original, &quarantined)?; + let moved = match std::fs::symlink_metadata(&quarantined) { + Ok(moved) => moved, + Err(error) => { + let _ = atomic_rename_noreplace(&quarantined, original); + return Err(error); + } + }; + if moved.dev() != metadata.dev() + || moved.ino() != metadata.ino() + || !moved.file_type().is_socket() + { + atomic_rename_noreplace(&quarantined, original).map_err(|restore_error| { + std::io::Error::new( + restore_error.kind(), + format!( + "legacy control socket identity changed and its replacement could not be restored: {restore_error}" + ), + ) + })?; + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy control socket identity changed during quarantine", + )); + } + Ok(Self { + original: original.to_path_buf(), + quarantined, + device: metadata.dev(), + inode: metadata.ino(), + armed: true, + }) } fn adopt( @@ -185,8 +242,7 @@ impl LegacySocketQuarantine { } (Some(true), None) => {} (None, Some(true)) => { - std::fs::hard_link(&self.quarantined, &self.original)?; - std::fs::remove_file(&self.quarantined)?; + atomic_rename_noreplace(&self.quarantined, &self.original)?; } (Some(false), _) | (_, Some(false)) => { return Err(std::io::Error::new( @@ -209,18 +265,30 @@ impl LegacySocketQuarantine { if !self.armed { return Ok(()); } - for path in [&self.quarantined, &self.original] { - match self.matches(path)? { - Some(true) => std::fs::remove_file(path)?, - Some(false) => { - return Err(std::io::Error::new( - std::io::ErrorKind::AlreadyExists, - "legacy control socket path was replaced during shutdown", - )); - } - None => {} + let original = self.matches(&self.original)?; + let quarantined = self.matches(&self.quarantined)?; + if original == Some(false) || quarantined == Some(false) { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "legacy control socket path was replaced during shutdown", + )); + } + if original == Some(true) { + if quarantined == Some(true) { + std::fs::remove_file(&self.quarantined)?; + } + atomic_rename_noreplace(&self.original, &self.quarantined)?; + if self.matches(&self.quarantined)? != Some(true) { + let _ = atomic_rename_noreplace(&self.quarantined, &self.original); + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "legacy control socket identity changed during removal", + )); } } + if self.matches(&self.quarantined)? == Some(true) { + std::fs::remove_file(&self.quarantined)?; + } self.armed = false; Ok(()) } @@ -247,8 +315,33 @@ struct LegacyHelperReaper { } #[cfg(unix)] -struct LegacyHelperReap { +struct LegacyHelperChild { child: Child, + quarantine: LegacySocketQuarantine, +} + +#[cfg(unix)] +impl LegacyHelperChild { + fn id(&self) -> u32 { + self.child.id() + } + + fn try_wait(&mut self) -> std::io::Result> { + self.child.try_wait() + } + + fn settle(mut self, status: ExitStatus) -> std::io::Result<()> { + if status.success() { self.quarantine.commit() } else { self.quarantine.restore() } + } + + fn restore(mut self) { + let _ = self.quarantine.restore(); + } +} + +#[cfg(unix)] +struct LegacyHelperReap { + helper: LegacyHelperChild, _lease: LegacyHelperReaperLease, next_attempt: Instant, retry_delay: Duration, @@ -269,11 +362,11 @@ impl Drop for LegacyHelperReaperLease { #[cfg(unix)] impl LegacyHelperReaperLease { - fn retain_reap_ownership(self, child: Child) { + fn retain_shutdown_ownership(self, helper: LegacyHelperChild) { let pending = self.pending.clone(); pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push( LegacyHelperReap { - child, + helper, _lease: self, next_attempt: Instant::now(), retry_delay: LEGACY_HELPER_POLL_INTERVAL, @@ -347,10 +440,14 @@ fn run_legacy_helper_reaper(pending: Arc<(Mutex>, Condvar) index += 1; continue; } - match pending_children[index].child.try_wait() { - Ok(Some(_)) => drop(pending_children.swap_remove(index)), + match pending_children[index].helper.try_wait() { + Ok(Some(status)) => { + let entry = pending_children.swap_remove(index); + let _ = entry.helper.settle(status); + } Err(error) if error.raw_os_error() == Some(libc::ECHILD) => { - drop(pending_children.swap_remove(index)); + let entry = pending_children.swap_remove(index); + entry.helper.restore(); } Ok(None) | Err(_) => { let entry = &mut pending_children[index]; @@ -944,7 +1041,7 @@ fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { #[cfg(unix)] fn run_detached_legacy_stop( - mut quarantine: LegacySocketQuarantine, + quarantine: LegacySocketQuarantine, expected: ProcessIdentity, ) -> anyhow::Result<()> { use std::os::unix::process::CommandExt; @@ -978,36 +1075,36 @@ fn run_detached_legacy_stop( anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; let deadline = Instant::now() + LEGACY_SHUTDOWN_TIMEOUT + LEGACY_HELPER_WAIT_MARGIN; - let status = wait_for_child_until(helper, reaper, deadline)?; + let status = + wait_for_child_until(LegacyHelperChild { child: helper, quarantine }, reaper, deadline)?; if status.success() { - quarantine.commit().map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; return Ok(()); } - quarantine.restore().map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed) } #[cfg(unix)] fn wait_for_child_until( - mut child: Child, + mut helper: LegacyHelperChild, reaper: LegacyHelperReaperLease, deadline: Instant, ) -> anyhow::Result { loop { - match child.try_wait() { - Ok(Some(status)) => return Ok(status), + match helper.try_wait() { + Ok(Some(status)) => { + helper.settle(status).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + return Ok(status); + } Ok(None) => {} Err(_) => { - reaper.retain_reap_ownership(child); + reaper.retain_shutdown_ownership(helper); anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } } if Instant::now() >= deadline { - let pid = libc::pid_t::try_from(child.id()).ok(); + let pid = libc::pid_t::try_from(helper.id()).ok(); if let Some(pid) = pid { // The helper installs a cooperative handler so Rust cleanup // can thaw any exact process identities currently fenced. @@ -1015,13 +1112,18 @@ fn wait_for_child_until( } let cancel_deadline = Instant::now() + LEGACY_HELPER_CANCEL_MARGIN; while Instant::now() < cancel_deadline { - match child.try_wait() { - Ok(Some(_)) => { + match helper.try_wait() { + Ok(Some(status)) => { + helper.settle(status).map_err(|_| { + anyhow::anyhow!( + crate::localization::catalog().server.legacy_cleanup_failed + ) + })?; anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } Ok(None) => {} Err(_) => { - reaper.retain_reap_ownership(child); + reaper.retain_shutdown_ownership(helper); anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } } @@ -1031,7 +1133,7 @@ fn wait_for_child_until( .min(LEGACY_HELPER_POLL_INTERVAL), ); } - reaper.retain_reap_ownership(child); + reaper.retain_shutdown_ownership(helper); anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } std::thread::sleep( @@ -1608,6 +1710,20 @@ mod tests { } } + #[cfg(unix)] + fn quarantined_test_listener( + name: &str, + ) -> (std::os::unix::net::UnixListener, LegacySocketQuarantine, PathBuf) { + let path = PathBuf::from("/tmp").join(format!( + "{name}-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let quarantine = LegacySocketQuarantine::acquire(&path).unwrap(); + (listener, quarantine, path) + } + #[test] fn mismatch_message_contains_both_release_identities_and_stop_command() { let identity = ServerIdentity { @@ -1803,6 +1919,7 @@ mod tests { use std::io::BufRead as _; let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); + let (listener, quarantine, path) = quarantined_test_listener("cq-reap"); let mut helper = Command::new("/bin/sh") .arg("-c") .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") @@ -1817,9 +1934,12 @@ mod tests { let started = Instant::now(); let reaper = reserve_legacy_helper_reaper().unwrap(); - let error = - wait_for_child_until(helper, reaper, Instant::now() + Duration::from_millis(50)) - .unwrap_err(); + let error = wait_for_child_until( + LegacyHelperChild { child: helper, quarantine }, + reaper, + Instant::now() + Duration::from_millis(50), + ) + .unwrap_err(); let elapsed = started.elapsed(); // SAFETY: helper_pid names this exact test-owned child. unsafe { @@ -1846,6 +1966,8 @@ mod tests { ); assert_eq!(reaped, -1, "timed-out helper lost its reaping owner"); assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); + drop(listener); + let _ = std::fs::remove_file(path); } #[cfg(unix)] @@ -1884,16 +2006,10 @@ mod tests { #[test] fn legacy_helper_timeout_keeps_the_socket_quarantined_until_helper_exit() { use std::io::BufRead as _; - use std::os::unix::net::{UnixListener, UnixStream}; + use std::os::unix::net::UnixStream; let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); - let path = PathBuf::from("/tmp").join(format!( - "cq-help-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() - )); - let listener = UnixListener::bind(&path).unwrap(); - let quarantine = LegacySocketQuarantine::acquire(&path).unwrap(); + let (listener, quarantine, path) = quarantined_test_listener("cq-help"); let mut helper = Command::new("/bin/sh") .arg("-c") .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") @@ -1908,10 +2024,12 @@ mod tests { let baseline = active_legacy_helper_reapers_for_test(); let reaper = reserve_legacy_helper_reaper().unwrap(); - let error = - wait_for_child_until(helper, reaper, Instant::now() + Duration::from_millis(50)) - .unwrap_err(); - drop(quarantine); + let error = wait_for_child_until( + LegacyHelperChild { child: helper, quarantine }, + reaper, + Instant::now() + Duration::from_millis(50), + ) + .unwrap_err(); let hidden_while_helper_alive = UnixStream::connect(&path).is_err(); // SAFETY: helper_pid names this exact test-owned child. unsafe { @@ -1941,6 +2059,7 @@ mod tests { #[test] fn legacy_reaper_releases_children_with_lost_wait_ownership() { let _serial = LEGACY_HELPER_REAPER_TEST_LOCK.lock().unwrap(); + let (listener, quarantine, path) = quarantined_test_listener("cq-lost"); let child = Command::new("/usr/bin/true") .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -1955,7 +2074,7 @@ mod tests { let baseline = active_legacy_helper_reapers_for_test(); let reaper = reserve_legacy_helper_reaper().unwrap(); assert_eq!(active_legacy_helper_reapers_for_test(), baseline + 1); - reaper.retain_reap_ownership(child); + reaper.retain_shutdown_ownership(LegacyHelperChild { child, quarantine }); let deadline = Instant::now() + Duration::from_millis(250); while active_legacy_helper_reapers_for_test() != baseline && Instant::now() < deadline { @@ -1967,6 +2086,12 @@ mod tests { baseline, "legacy reaper retained a child after the kernel reported no wait ownership" ); + assert!( + std::os::unix::net::UnixStream::connect(&path).is_ok(), + "lost wait ownership did not release the admission fence after helper exit" + ); + drop(listener); + let _ = std::fs::remove_file(path); } #[cfg(unix)] From eae612180dde3c0879b7f0c2e10a2bf2411fc534 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:51:09 -0700 Subject: [PATCH 128/521] test(tui): expose duplicate startup reconnect --- cmux-tui/crates/cmux-tui/tests/cli.rs | 47 +++++++++++++++------------ 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index c5fc8bdf00c6..ddd94246a258 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -997,32 +997,31 @@ fn plain_launch_attaches_to_existing_local_session() { } #[test] -fn plain_launch_explains_how_to_replace_an_incompatible_local_server() { +fn plain_launch_reports_an_incompatible_server_without_reconnecting() { let dir = unique_temp_dir("incompatible-server"); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let listener = transport::listen(&socket).unwrap(); + let (accepted_tx, accepted_rx) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { - for connection in 0..2 { - let mut stream = listener.accept().unwrap(); - if connection == 0 { - let mut request = String::new(); - BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); - let request: serde_json::Value = serde_json::from_str(&request).unwrap(); - let response = serde_json::json!({ - "id": request["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "session": "test", - "pid": 4242, - "version": "0.0.0-stale", - "protocol": 8, - }, - }); - writeln!(stream, "{response}").unwrap(); - } - } + let mut stream = listener.accept().unwrap(); + let mut request = String::new(); + BufReader::new(stream.try_clone_box().unwrap()).read_line(&mut request).unwrap(); + let request: serde_json::Value = serde_json::from_str(&request).unwrap(); + let response = serde_json::json!({ + "id": request["id"], + "ok": true, + "data": { + "app": "cmux-tui", + "session": "test", + "pid": 4242, + "version": "0.0.0-stale", + "protocol": 8, + }, + }); + writeln!(stream, "{response}").unwrap(); + let _cleanup_or_duplicate = listener.accept().unwrap(); + accepted_tx.send(()).unwrap(); }); let output = Command::new(bin()) @@ -1031,11 +1030,17 @@ fn plain_launch_explains_how_to_replace_an_incompatible_local_server() { .env_remove("CMUX_TUI_SOCKET") .output() .unwrap(); + let reconnected = accepted_rx.recv_timeout(Duration::from_millis(250)).is_ok(); + if !reconnected { + let _cleanup = transport::connect(&socket).unwrap(); + accepted_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + } server.join().unwrap(); let _ = fs::remove_file(&socket); let _ = fs::remove_dir_all(&dir); assert_eq!(output.status.code(), Some(1)); + assert!(!reconnected, "plain startup opened a second connection after identifying the server"); let stderr = String::from_utf8(output.stderr).unwrap(); assert!(stderr.contains("server: v0.0.0-stale protocol 8"), "{stderr}"); assert!( From 00dd8f25f6995dc99a54360b86e59d53ca30246f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:53:00 -0700 Subject: [PATCH 129/521] fix(tui): reuse bounded startup connection --- cmux-tui/crates/cmux-tui/src/main.rs | 39 +++++++++++-------- .../crates/cmux-tui/src/session/remote.rs | 15 +++++++ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 0e1a0ad0d7c0..2718514bb1b9 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -66,6 +66,7 @@ const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORIT const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1); const SERVER_SHUTDOWN_RETRY_INITIAL: std::time::Duration = std::time::Duration::from_millis(100); const SERVER_SHUTDOWN_RETRY_MAX: std::time::Duration = std::time::Duration::from_secs(5); +const EXISTING_SESSION_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -1142,22 +1143,17 @@ fn run_server( .socket .clone() .unwrap_or_else(|| cmux_tui_core::server::default_socket_path(&args.session)); - if args.should_attach_existing(&ws_addr, &ws_token) && socket_path.exists() { - match RemoteSession::connect(&socket_path) { - Ok(remote) => { - return run_connected_session_client( - socket_path, - args.session, - config, - Session::Remote(remote), - None, - ); - } - Err(error) if cmux_tui_core::platform::transport::connect(&socket_path).is_ok() => { - return Err(error); - } - Err(_) => {} - } + if args.should_attach_existing(&ws_addr, &ws_token) + && socket_path.exists() + && let Some(remote) = connect_existing_local_session(&socket_path)? + { + return run_connected_session_client( + socket_path, + args.session, + config, + Session::Remote(remote), + None, + ); } #[cfg(unix)] @@ -1273,6 +1269,17 @@ fn run_server( result } +fn connect_existing_local_session( + socket_path: &std::path::Path, +) -> anyhow::Result>> { + let deadline = std::time::Instant::now() + EXISTING_SESSION_CONNECT_TIMEOUT; + let stream = match cmux_tui_core::platform::transport::connect_until(socket_path, deadline) { + Ok(stream) => stream, + Err(_) => return Ok(None), + }; + RemoteSession::connect_local_stream(stream, socket_path).map(Some) +} + fn run_tui( session: Session, session_label: String, diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index 14617b8bce36..c6c7b58fe068 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -568,6 +568,21 @@ impl RemoteSession { let stream = transport::connect(path).map_err(|e| { anyhow::anyhow!("cannot connect to session socket {}: {e}", path.display()) })?; + Self::connect_local_stream_with_subscription(stream, path, subscribe) + } + + pub(crate) fn connect_local_stream( + stream: Box, + path: &Path, + ) -> anyhow::Result> { + Self::connect_local_stream_with_subscription(stream, path, true) + } + + fn connect_local_stream_with_subscription( + stream: Box, + path: &Path, + subscribe: bool, + ) -> anyhow::Result> { let transport = RemoteTransport::json_lines(stream).map_err(|error| { anyhow::anyhow!("cannot configure JSON-lines session transport: {error}") })?; From 5cfd3223636abf7b4954b8401c7973420944a68b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:56:21 -0700 Subject: [PATCH 130/521] refactor(tui): compact PTY shutdown ownership --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 57 +++++++++----------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 732e2239eb1f..784fab07637d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -566,7 +566,7 @@ pub struct PtySurface { runtime: Mutex, /// Local process ownership is separate from PTY writes so shutdown never /// waits behind a blocked writer. - local_process: Option>, + local_process: Option>, #[cfg(unix)] hosted_shutdown_owner: Option>, supports_clear_history_key_fallback: AtomicBool, @@ -1750,7 +1750,7 @@ impl Surface { stream_progress: Box::new(TerminalStreamProgress::default()), mouse_encoders: Mutex::new(mouse_encoders), runtime: Mutex::new(PtyRuntime::Local { writer, master: pty.master }), - local_process: Some(Mutex::new(LocalProcess::Owned(process.clone()))), + local_process: Some(Box::new(LocalProcess::Owned(process.clone()))), #[cfg(unix)] hosted_shutdown_owner: None, supports_clear_history_key_fallback: AtomicBool::new( @@ -2525,7 +2525,7 @@ impl Surface { }), }), }), - local_process: Some(Mutex::new(LocalProcess::untracked(Box::new(TestChildKiller)))), + local_process: Some(Box::new(LocalProcess::untracked(Box::new(TestChildKiller)))), #[cfg(unix)] hosted_shutdown_owner: None, supports_clear_history_key_fallback: AtomicBool::new(false), @@ -2560,8 +2560,7 @@ impl Surface { pub(crate) fn set_server_shutdown_failure_for_test(&self, fail: bool) { let Self::Pty(pty) = self else { return }; let Some(process) = &pty.local_process else { return }; - let process = process.lock().unwrap(); - let LocalProcess::Untracked(killer) = &*process else { return }; + let LocalProcess::Untracked(killer) = process.as_ref() else { return }; *killer.lock().unwrap() = if fail { Box::new(TestFailingChildKiller) } else { Box::new(TestChildKiller) }; } @@ -2570,8 +2569,7 @@ impl Surface { pub(crate) fn set_server_shutdown_delay_for_test(&self, delay: Duration) { let Self::Pty(pty) = self else { return }; let Some(process) = &pty.local_process else { return }; - let process = process.lock().unwrap(); - let LocalProcess::Untracked(killer) = &*process else { return }; + let LocalProcess::Untracked(killer) = process.as_ref() else { return }; *killer.lock().unwrap() = Box::new(TestSlowFailingChildKiller(delay)); } @@ -2583,8 +2581,9 @@ impl Surface { let attempts = Arc::new(AtomicUsize::new(0)); let Self::Pty(pty) = self else { return (failing, attempts) }; let Some(process) = &pty.local_process else { return (failing, attempts) }; - let process = process.lock().unwrap(); - let LocalProcess::Untracked(killer) = &*process else { return (failing, attempts) }; + let LocalProcess::Untracked(killer) = process.as_ref() else { + return (failing, attempts); + }; *killer.lock().unwrap() = Box::new(TestRecoveringChildKiller { failing: failing.clone(), attempts: attempts.clone(), @@ -3270,7 +3269,7 @@ impl Surface { match self { Surface::Pty(pty) => { if let Some(process) = &pty.local_process { - return process.lock().unwrap().request_hangup(); + return process.request_hangup(); } let mut runtime = pty.runtime.lock().unwrap(); match &mut *runtime { @@ -3325,7 +3324,7 @@ impl Surface { Surface::Pty(pty) => { if let Some(process) = &pty.local_process { return Some(SurfaceShutdownOwner { - kind: SurfaceShutdownOwnerKind::Local(process.lock().unwrap().clone()), + kind: SurfaceShutdownOwnerKind::Local(process.as_ref().clone()), }); } #[cfg(unix)] @@ -4047,11 +4046,10 @@ mod tests { ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); - let process = - match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { - LocalProcess::Owned(process) => process.clone(), - LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), - }; + let process = match surface.as_pty().unwrap().local_process.as_deref().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; let start_deadline = Instant::now() + Duration::from_secs(1); while !pid_path.exists() && Instant::now() < start_deadline { std::thread::sleep(Duration::from_millis(10)); @@ -4116,11 +4114,10 @@ mod tests { ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); - let process = - match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { - LocalProcess::Owned(process) => process.clone(), - LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), - }; + let process = match surface.as_pty().unwrap().local_process.as_deref().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; process.normal_cleanup_failures.store(1, Ordering::Release); std::fs::write(&release_path, b"ready").unwrap(); @@ -4169,11 +4166,10 @@ mod tests { Arc::downgrade(&mux), ) .unwrap(); - let process = - match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { - LocalProcess::Owned(process) => process.clone(), - LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), - }; + let process = match surface.as_pty().unwrap().local_process.as_deref().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; process.normal_cleanup_failures.store(usize::MAX, Ordering::Release); surfaces.push(surface); processes.push(process); @@ -4226,11 +4222,10 @@ mod tests { ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); - let process = - match &*surface.as_pty().unwrap().local_process.as_ref().unwrap().lock().unwrap() { - LocalProcess::Owned(process) => process.clone(), - LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), - }; + let process = match surface.as_pty().unwrap().local_process.as_deref().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; process.normal_cleanup_delay_ms.store(300, Ordering::Release); std::fs::write(&release_path, b"ready").unwrap(); let sweep_deadline = Instant::now() + Duration::from_secs(1); From a25b60c0743602f859ecf1b89d1ab1ce5489e4ce Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:13:11 -0700 Subject: [PATCH 131/521] test(tui): expose zombie session drain stall --- .../cmux-tui-core/src/process_session.rs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 0e553ea39bfc..b0c1bcaa48dd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1410,6 +1410,59 @@ mod tests { assert_eq!(stable_signals, 1, "session cleanup did not use the stable process handle"); } + #[cfg(target_os = "linux")] + #[test] + fn shutdown_treats_zombie_members_as_drained_before_reserved_parent() { + use std::io::BufRead as _; + + let mut command = Command::new("/bin/sh"); + command + .args(["-c", "trap '' HUP TERM; sleep 60 & echo $!; wait"]) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut leader = command.spawn().unwrap(); + let session = libc::pid_t::try_from(leader.id()).unwrap(); + let mut descendant = String::new(); + io::BufReader::new(leader.stdout.take().unwrap()).read_line(&mut descendant).unwrap(); + let descendant = descendant.trim().parse::().unwrap(); + let leader_handle = StableProcessHandle::capture(session).unwrap().unwrap(); + assert!(leader_handle.signal(libc::SIGSTOP).unwrap()); + let mut status = 0; + loop { + let waited = unsafe { libc::waitpid(session, &mut status, libc::WUNTRACED) }; + if waited == session { + assert!(libc::WIFSTOPPED(status)); + break; + } + assert_eq!(waited, -1); + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + panic!("could not observe reserved session leader stop: {error}"); + } + } + + let drained = + kill_until_only_reserved(session, session, Instant::now() + Duration::from_secs(1)); + + let _ = unsafe { libc::kill(descendant, libc::SIGKILL) }; + let _ = leader_handle.signal(libc::SIGCONT); + let _ = leader.kill(); + let _ = leader.wait(); + assert_eq!( + drained.unwrap(), + true, + "an unreaped zombie kept a fully terminated PTY session pending" + ); + } + #[cfg(target_os = "linux")] #[test] fn pidfd_open_denial_uses_an_exact_proc_handle() { From 3ec34c942d5ac2e6ad49908cf5032ada2449e37a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:18:09 -0700 Subject: [PATCH 132/521] fix(tui): retain fenced shutdown ownership --- .../cmux-tui-core/src/process_session.rs | 89 +++++-- .../src/server_lifecycle/legacy_process.rs | 252 ++++++++++++++---- 2 files changed, 266 insertions(+), 75 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index b0c1bcaa48dd..2a82b160eab8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -773,13 +773,79 @@ fn signal_if_still_member( } fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { + Ok(active_process_session(pid)? == Some(session)) +} + +#[cfg(not(target_os = "linux"))] +fn active_process_session(pid: libc::pid_t) -> io::Result> { // SAFETY: getsid only queries process metadata. let current_session = unsafe { libc::getsid(pid) }; if current_session >= 0 { - return Ok(current_session == session); + return Ok(Some(current_session)); } let error = io::Error::last_os_error(); - if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) } + if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) } +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct LinuxProcessMetadata { + started_at: u128, + zombie: bool, +} + +#[cfg(target_os = "linux")] +fn linux_process_metadata(pid: libc::pid_t) -> io::Result> { + let stat = match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(stat) => stat, + Err(error) + if error.kind() == io::ErrorKind::NotFound + || error.raw_os_error() == Some(libc::ESRCH) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; + let (pid_text, fields) = stat + .split_once(" (") + .and_then(|(pid_text, remainder)| { + remainder.rsplit_once(") ").map(|(_, fields)| (pid_text, fields)) + }) + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + if pid_text.parse::().ok() != Some(pid) { + return Err(io::Error::other("process metadata id mismatch")); + } + let fields = fields.split_whitespace().collect::>(); + let state = fields.first().copied().ok_or_else(|| io::Error::other("missing process state"))?; + let started_at = fields + .get(19) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| io::Error::other("invalid process birth identity"))?; + Ok(Some(LinuxProcessMetadata { started_at, zombie: state == "Z" })) +} + +#[cfg(target_os = "linux")] +fn active_process_session(pid: libc::pid_t) -> io::Result> { + let Some(before) = linux_process_metadata(pid)? else { return Ok(None) }; + if before.zombie { + return Ok(None); + } + + // SAFETY: getsid only queries process metadata. + let current_session = unsafe { libc::getsid(pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; + } + + // Bracket the reusable PID query with the kernel process birth identity. + // Zombies cannot execute or fork, so shutdown may treat them as drained + // while the reserved leader still prevents session-id reuse. + let Some(after) = linux_process_metadata(pid)? else { return Ok(None) }; + if after.zombie || after.started_at != before.started_at { + return Ok(None); + } + Ok(Some(current_session)) } fn stable_process_in_session( @@ -1128,17 +1194,12 @@ fn scan_sessions( if pid <= 1 { continue; } - // SAFETY: getsid only queries process metadata. - let current_session = unsafe { libc::getsid(pid) }; + let Some(current_session) = active_process_session(pid)? else { + continue; + }; + ensure_before_deadline(deadline)?; if let Some(session_members) = members.get_mut(¤t_session) { session_members.push(pid); - continue; - } - if current_session < 0 { - let error = io::Error::last_os_error(); - if error.raw_os_error() != Some(libc::ESRCH) { - return Err(error); - } } } for session_members in members.values_mut() { @@ -1456,11 +1517,7 @@ mod tests { let _ = leader_handle.signal(libc::SIGCONT); let _ = leader.kill(); let _ = leader.wait(); - assert_eq!( - drained.unwrap(), - true, - "an unreaped zombie kept a fully terminated PTY session pending" - ); + assert!(drained.unwrap(), "an unreaped zombie kept a fully terminated PTY session pending"); } #[cfg(target_os = "linux")] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 46fb9abdcf4f..20ccba6c8a3a 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -75,8 +75,19 @@ impl ProcessIdentity { } Ok(()) })?; - let Some(process) = StableProcessHandle::capture(self.pid)? else { - return Ok(None); + let process = match StableProcessHandle::capture(self.pid) { + Ok(Some(process)) => process, + Ok(None) => return Ok(None), + Err(error) => match Self::capture(self.pid) { + Ok(None) => return Ok(None), + Ok(Some(current)) if current != self => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "process identity changed", + )); + } + Ok(Some(_)) | Err(_) => return Err(error), + }, }; let Some(current) = Self::capture(self.pid)? else { return Ok(None); @@ -95,6 +106,7 @@ impl ProcessIdentity { struct ProcessSnapshot { identity: ProcessIdentity, parent: libc::pid_t, + stopped: bool, } #[cfg(test)] @@ -111,36 +123,74 @@ struct ProcessFence { } impl ProcessFence { - fn acquire(identity: ProcessIdentity) -> io::Result> { + fn acquire(identity: ProcessIdentity, deadline: Instant) -> io::Result> { + let Some(initial) = process_snapshot(identity.pid)? else { return Ok(None) }; + if initial.identity != identity { + return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); + } let Some(process) = identity.stable_handle()? else { return Ok(None) }; - if !process.signal(libc::SIGSTOP)? { + let armed = !initial.stopped; + if armed && !process.signal(libc::SIGSTOP)? { return Ok(None); } - Ok(Some(Self { identity, process, armed: true })) + let fence = Self { identity, process, armed }; + loop { + ensure_helper_active()?; + let Some(current) = process_snapshot(identity.pid)? else { return Ok(None) }; + if current.identity != identity { + return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); + } + if current.stopped { + return Ok(Some(fence)); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "process did not stop before the legacy shutdown deadline", + )); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(PROCESS_TREE_RETRY_INTERVAL), + ); + } } fn acquire_in_session( identity: ProcessIdentity, session: libc::pid_t, + deadline: Instant, ) -> io::Result> { - let Some(process) = identity.stable_handle()? else { return Ok(None) }; - if !process.matches_current()? { - return Ok(None); - } - // SAFETY: getsid only reads metadata. Exact-handle probes bracket the - // reusable PID so a replacement process is never stopped. + let Some(process) = Self::acquire(identity, deadline)? else { return Ok(None) }; + // SAFETY: the exact process is now confirmed stopped, so this + // reusable-PID query cannot race process exit or replacement. let current_session = unsafe { libc::getsid(identity.pid) }; if current_session < 0 { let error = io::Error::last_os_error(); return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; } - if current_session != session || !process.matches_current()? { + if current_session != session + || ProcessIdentity::capture(identity.pid)? != Some(identity) + || !process.process.matches_current()? + { return Ok(None); } - if !process.signal(libc::SIGSTOP)? { - return Ok(None); + Ok(Some(process)) + } + + fn still_in_session(&self, session: libc::pid_t) -> io::Result { + if ProcessIdentity::capture(self.identity.pid)? != Some(self.identity) { + return Ok(false); + } + // SAFETY: a confirmed stopped process cannot exit or be replaced + // without an external signal; exact identity probes bracket getsid. + let current_session = unsafe { libc::getsid(self.identity.pid) }; + if current_session < 0 { + let error = io::Error::last_os_error(); + return if error.raw_os_error() == Some(libc::ESRCH) { Ok(false) } else { Err(error) }; } - Ok(Some(Self { identity, process, armed: true })) + Ok(current_session == session + && self.process.matches_current()? + && ProcessIdentity::capture(self.identity.pid)? == Some(self.identity)) } fn kill(&mut self) -> io::Result<()> { @@ -254,20 +304,30 @@ pub(super) struct CapturedProcessTree { } impl CapturedProcessTree { - fn contains_process(&self, process: ProcessIdentity) -> bool { - self.sessions.iter().any(|session| session.members.contains(&process)) + fn session_process(&self, pid: libc::pid_t) -> Option<(&CapturedSession, ProcessIdentity)> { + self.sessions.iter().find_map(|session| { + session + .members + .iter() + .copied() + .find(|process| process.pid == pid) + .map(|process| (session, process)) + }) } } -pub(super) struct CapturedProcessSession(CapturedSession); +pub(super) struct CapturedProcessSession { + session: CapturedSession, + _fences: Vec, +} impl CapturedProcessSession { pub(super) fn id(&self) -> libc::pid_t { - self.0.id + self.session.id } pub(super) fn terminate_until(self, deadline: Instant) -> io::Result<()> { - if self.0.kill_until_empty(deadline)? { + if self.session.kill_until_empty(deadline)? { Ok(()) } else { Err(io::Error::other( @@ -289,51 +349,113 @@ pub(super) fn capture_process_session( { return Err(io::Error::new(io::ErrorKind::InvalidInput, "invalid PTY owner process")); } - let Some(process) = ProcessIdentity::capture(pid)? else { return Ok(None) }; - if !captured.contains_process(process) { + let Some((captured_session, process)) = captured.session_process(pid) else { return Err(io::Error::new( io::ErrorKind::InvalidData, "PTY owner was not captured in the verified server process tree", )); + }; + if ProcessIdentity::capture(pid)?.is_some_and(|current| current != process) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PTY owner process identity changed after the verified tree capture", + )); + } + let session = captured_session.id; + // SAFETY: the server identity remains validated by the caller. + let server_session = unsafe { libc::getsid(server.pid) }; + if server_session < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: getsid(0) only queries the detached helper. + let helper_session = unsafe { libc::getsid(0) }; + if helper_session < 0 { + return Err(io::Error::last_os_error()); + } + if session == server_session || session == helper_session { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "PTY owner shares a protected process session", + )); } - let Some(process_fence) = ProcessFence::acquire(process)? else { return Ok(None) }; - let captured = (|| { + + // Retain every exact process fence across the legacy close mutation. + // Once every live member is stopped, the session cannot fork and at + // least one fence pins its numeric ID until owned termination begins. + let mut fences = Vec::new(); + for process in &captured_session.members { ensure_helper_active()?; - // SAFETY: the retained exact process handle keeps this stopped PID - // bound to the verified process instance. - let session = unsafe { libc::getsid(pid) }; - if session <= 1 { - return Err(io::Error::last_os_error()); + if let Some(fence) = ProcessFence::acquire_in_session(*process, session, deadline)? { + fences.push(fence); } - // SAFETY: the server identity remains validated by the caller. - let server_session = unsafe { libc::getsid(server.pid) }; - // SAFETY: getsid(0) only queries the detached helper. - let helper_session = unsafe { libc::getsid(0) }; - if session == server_session || session == helper_session { + } + loop { + ensure_helper_active()?; + let mut pinned = false; + for fence in &fences { + pinned |= fence.still_in_session(session)?; + } + let member_pids = + cmux_tui_core::process_session::session_member_pids_until(session, deadline)?; + if !pinned { + if member_pids.is_empty() { + return Ok(Some(CapturedProcessSession { + session: CapturedSession { id: session, members: Vec::new() }, + _fences: fences, + })); + } return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "PTY owner shares a protected process session", + io::ErrorKind::InvalidData, + "PTY session lost every captured process before ownership could be fenced", )); } + let mut members = Vec::new(); - for member in cmux_tui_core::process_session::session_member_pids_until(session, deadline)? - { + for member in member_pids { if let Some(identity) = ProcessIdentity::capture(member)? { members.push(identity); } } - if !members.contains(&process) { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "PTY owner left its process session while being captured", - )); + for fence in &fences { + if fence.still_in_session(session)? { + members.push(fence.identity); + } } members.sort_by_key(|member| member.pid); members.dedup(); - Ok(CapturedProcessSession(CapturedSession { id: session, members })) - })(); - drop(process_fence); - captured.map(Some) + + let mut stable = !members.is_empty(); + for member in &members { + let held = fences + .iter() + .find(|fence| fence.identity == *member) + .map(|fence| fence.still_in_session(session)) + .transpose()? + .unwrap_or(false); + if held { + continue; + } + stable = false; + if let Some(fence) = ProcessFence::acquire_in_session(*member, session, deadline)? { + fences.push(fence); + } + } + if stable { + return Ok(Some(CapturedProcessSession { + session: CapturedSession { id: session, members }, + _fences: fences, + })); + } + if Instant::now() >= deadline { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "PTY session ownership did not stabilize before the legacy shutdown deadline", + )); + } + std::thread::sleep( + deadline.saturating_duration_since(Instant::now()).min(PROCESS_TREE_RETRY_INTERVAL), + ); + } } #[derive(Clone)] @@ -346,7 +468,7 @@ impl CapturedSession { fn kill_until_empty(&self, deadline: Instant) -> io::Result { loop { ensure_helper_active()?; - let error = match SessionProbe::acquire(self) { + let error = match SessionProbe::acquire(self, deadline) { Ok(Some(mut probe)) => { match cmux_tui_core::process_session::kill_until_only_reserved( self.id, @@ -388,13 +510,11 @@ struct SessionProbe { } impl SessionProbe { - fn acquire(session: &CapturedSession) -> io::Result> { + fn acquire(session: &CapturedSession, deadline: Instant) -> io::Result> { for process in &session.members { ensure_helper_active()?; - if ProcessIdentity::capture(process.pid)? != Some(*process) { - continue; - } - let Some(process) = ProcessFence::acquire_in_session(*process, session.id)? else { + let Some(process) = ProcessFence::acquire_in_session(*process, session.id, deadline)? + else { continue; }; return Ok(Some(Self { process })); @@ -416,7 +536,7 @@ pub(super) struct FrozenProcessTree { impl FrozenProcessTree { fn freeze(root: ProcessIdentity, deadline: Instant) -> io::Result { ensure_helper_active()?; - let Some(root) = ProcessFence::acquire(root)? else { + let Some(root) = ProcessFence::acquire(root, deadline)? else { return Err(io::Error::new( io::ErrorKind::NotFound, "server exited before its process tree was fenced", @@ -452,7 +572,7 @@ impl FrozenProcessTree { if snapshot.parent != parent.pid { continue; } - let Some(process) = ProcessFence::acquire(snapshot.identity)? else { + let Some(process) = ProcessFence::acquire(snapshot.identity, deadline)? else { continue; }; known.insert(pid); @@ -553,7 +673,11 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { return Ok(None); } let started_at = (u128::from(info.pbi_start_tvsec) << 64) | u128::from(info.pbi_start_tvusec); - Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent })) + Ok(Some(ProcessSnapshot { + identity: ProcessIdentity { pid, started_at }, + parent, + stopped: info.pbi_status == libc::SSTOP, + })) } #[cfg(target_os = "macos")] @@ -696,7 +820,12 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { let path = format!("/proc/{pid}/stat"); let stat = match std::fs::read_to_string(path) { Ok(stat) => stat, - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) + if error.kind() == io::ErrorKind::NotFound + || error.raw_os_error() == Some(libc::ESRCH) => + { + return Ok(None); + } Err(error) => return Err(error), }; let (pid_text, remainder) = stat @@ -709,7 +838,8 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { return Err(io::Error::other("process metadata id mismatch")); } let fields = remainder.split_whitespace().collect::>(); - if fields.first() == Some(&"Z") { + let state = fields.first().copied().ok_or_else(|| io::Error::other("missing process state"))?; + if state == "Z" { return Ok(None); } let parent = fields @@ -720,7 +850,11 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { .get(19) .and_then(|value| value.parse::().ok()) .ok_or_else(|| io::Error::other("invalid process birth identity"))?; - Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent })) + Ok(Some(ProcessSnapshot { + identity: ProcessIdentity { pid, started_at }, + parent, + stopped: matches!(state, "T" | "t"), + })) } #[cfg(target_os = "linux")] From 45a854c4522565d26b2da0741769d97e5283f1a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:11:54 -0700 Subject: [PATCH 133/521] test(tui): expose lost lifecycle ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 60 ++++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 12 ++++ 2 files changed, 72 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b116dc42e3a6..94832e39c66a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12241,6 +12241,66 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn exited_placeholder_does_not_suppress_terminal_adoption_rescan() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let root = std::env::temp_dir().join(format!( + "cmux-adoption-exited-placeholder-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let mux = Mux::new_for_test("adoption-exited-placeholder", SurfaceOptions::default()); + let workspace = + mux.create_empty_workspace(Some("adoption-placeholder".into()), None, None).unwrap(); + let terminal_id = TerminalId::random().unwrap().to_hex(); + let incarnation = crate::terminal_host::HostIncarnation::random().unwrap().to_hex(); + let placeholder = insert_running_terminal_identity_surface( + &mux, + &terminal_id, + &incarnation, + &workspace.key, + ); + assert!(placeholder.is_dead()); + + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 2, + terminal_id: terminal_id.clone(), + incarnation, + endpoint: format!( + "/tmp/cmux-th-{}/{}.sock", + std::fs::metadata(&root).unwrap().uid(), + terminal_id + ), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 1, + host_start_nonce: "01".repeat(32), + workspace_key: workspace.key, + supports_set_defaults: true, + supports_terminate_only: true, + supports_clear_history: true, + }; + let record_path = root.join(format!("{terminal_id}.json")); + std::fs::write(&record_path, serde_json::to_vec(&record).unwrap()).unwrap(); + let mut permissions = std::fs::metadata(&record_path).unwrap().permissions(); + permissions.set_mode(0o600); + std::fs::set_permissions(&record_path, permissions).unwrap(); + mux.update_surface_options(|options| options.terminal_host_root = Some(root.clone())); + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = true; + + assert!(mux.rescan_terminal_adoptions()); + let rescan_required = + mux.terminal_adoption_coordinator.state.lock().unwrap().rescan_required; + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = false; + let _ = std::fs::remove_dir_all(root); + + assert!( + rescan_required, + "an exited identity placeholder suppressed the retry for its live discovery record" + ); + } + #[cfg(unix)] #[test] fn terminal_adoption_retries_when_the_registry_read_is_uncertain() { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 784fab07637d..34ed35a198d5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4019,6 +4019,18 @@ mod tests { assert_eq!(signals.load(Ordering::Relaxed), 0); } + #[cfg(unix)] + #[test] + fn non_child_pid_is_not_reported_as_a_waitable_child() { + let current = + libc::pid_t::try_from(std::process::id()).expect("current process ID fits pid_t"); + + assert!( + !local_child_is_waitable(current).unwrap(), + "ECHILD was treated as proof that the numeric session remains reserved" + ); + } + #[cfg(unix)] #[test] fn normal_child_exit_cleans_same_session_descendants_before_reaping() { From 19ddd15a0059a6ab2b64ea4212b2747ff4cc2b11 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:23:49 -0700 Subject: [PATCH 134/521] fix(tui): fail closed on lost lifecycle ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 +-- .../cmux-tui-core/src/process_session.rs | 99 ++++++++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 154 ++++++++++++++---- .../src/terminal_host_runtime.rs | 152 ++++++++++------- 4 files changed, 321 insertions(+), 103 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 94832e39c66a..d0a4073c213e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2378,22 +2378,21 @@ impl Mux { }; terminals.into_iter().collect::>() }; - let live_counts = { + let live_incarnations = { let state = self.state.lock().unwrap(); - let mut counts = HashMap::::new(); - for terminal_id in state - .surfaces - .values() - .filter_map(|surface| surface.terminal_host_identity()) - .map(|identity| identity.terminal_id) + let mut live = HashMap::>::new(); + for identity in + state.surfaces.values().filter_map(|surface| surface.live_terminal_host_identity()) { - *counts.entry(terminal_id).or_default() += 1; + live.entry(identity.terminal_id).or_default().insert(identity.incarnation); } - counts + live }; for (record_path, record) in records { let already_live = registered.contains(&record.terminal_id) - && live_counts.get(&record.terminal_id).copied() == Some(1); + && live_incarnations + .get(&record.terminal_id) + .is_some_and(|incarnations| incarnations.contains(&record.incarnation)); if already_live { continue; } diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2a82b160eab8..e6a32c9c14dd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -73,6 +73,14 @@ pub(crate) struct ReservedChildReap<'a> { pub(crate) termination_started: &'a AtomicBool, pub(crate) cleanup_complete: &'a AtomicBool, pub(crate) child_reaped: &'a AtomicBool, + pub(crate) wait_ownership_lost: &'a AtomicBool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ChildWaitState { + Running, + Waitable, + OwnershipLost, } enum NaturalReaperCommand { @@ -151,6 +159,7 @@ impl NaturalReaper { } pub(crate) fn reserve_child_reaper() -> io::Result { + require_waitable_child_disposition()?; let mut slot = NATURAL_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); if slot.is_none() { *slot = Some(NaturalReaper::start()?); @@ -158,6 +167,59 @@ pub(crate) fn reserve_child_reaper() -> io::Result { slot.as_ref().expect("natural PTY reaper initialized").lease() } +pub(crate) fn observe_child_without_reaping( + pid: libc::pid_t, + nonblocking: bool, +) -> io::Result { + loop { + let mut status = std::mem::MaybeUninit::::zeroed(); + let mut flags = libc::WEXITED | libc::WNOWAIT; + if nonblocking { + flags |= libc::WNOHANG; + } + // SAFETY: status points to writable siginfo storage. WNOWAIT keeps + // the child and its numeric session ID reserved until the sole owner + // performs the final wait. + let result = + unsafe { libc::waitid(libc::P_PID, pid as libc::id_t, status.as_mut_ptr(), flags) }; + if result == 0 { + // SAFETY: waitid initialized status on success; zeroed storage + // keeps si_signo at zero when WNOHANG finds no waitable child. + return Ok(if unsafe { status.assume_init() }.si_signo == 0 { + ChildWaitState::Running + } else { + ChildWaitState::Waitable + }); + } + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + if error.raw_os_error() == Some(libc::ECHILD) { + return Ok(ChildWaitState::OwnershipLost); + } + return Err(error); + } +} + +fn require_waitable_child_disposition() -> io::Result<()> { + let mut action = std::mem::MaybeUninit::::zeroed(); + // SAFETY: a null replacement only reads the process-wide SIGCHLD + // disposition into the initialized output storage. + if unsafe { libc::sigaction(libc::SIGCHLD, std::ptr::null(), action.as_mut_ptr()) } < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: sigaction initialized the output on success. + let action = unsafe { action.assume_init() }; + if action.sa_sigaction == libc::SIG_IGN || action.sa_flags & libc::SA_NOCLDWAIT != 0 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "SIGCHLD disposition does not preserve PTY child wait ownership", + )); + } + Ok(()) +} + pub(crate) fn wake_child_reaper() { let Some(slot) = NATURAL_REAPER.get() else { return }; let slot = slot.lock().unwrap(); @@ -197,7 +259,8 @@ pub(crate) fn enqueue_reserved_session_leader( pub(crate) fn reserved_child_needs_cleanup(sync: ReservedChildReap<'_>) -> bool { let _signal = sync.signal_lock.lock().unwrap(); - !sync.cleanup_complete.load(Ordering::Acquire) + !sync.wait_ownership_lost.load(Ordering::Acquire) + && !sync.cleanup_complete.load(Ordering::Acquire) && !sync.termination_started.load(Ordering::Acquire) } @@ -207,6 +270,9 @@ pub(crate) fn poll_reserved_session_leader( reap: impl FnOnce(), ) -> bool { let _signal = sync.signal_lock.lock().unwrap(); + if sync.wait_ownership_lost.load(Ordering::Acquire) { + return false; + } if !sync.cleanup_complete.load(Ordering::Acquire) { if sync.termination_started.load(Ordering::Acquire) || !cleanup_succeeded { return false; @@ -1111,6 +1177,7 @@ pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::R "forced process-session preflight failure", )); } + require_waitable_child_disposition()?; let pid = libc::pid_t::try_from(std::process::id()) .map_err(|_| io::Error::new(io::ErrorKind::Unsupported, "invalid process id"))?; let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { @@ -1333,6 +1400,7 @@ mod tests { let termination_started = AtomicBool::new(false); let cleanup_complete = AtomicBool::new(false); let child_reaped = AtomicBool::new(false); + let wait_ownership_lost = AtomicBool::new(false); let reap_called = AtomicBool::new(false); let sync = || ReservedChildReap { signal_lock: &signal_lock, @@ -1340,6 +1408,7 @@ mod tests { termination_started: &termination_started, cleanup_complete: &cleanup_complete, child_reaped: &child_reaped, + wait_ownership_lost: &wait_ownership_lost, }; assert!( @@ -1367,6 +1436,34 @@ mod tests { assert!(reap_called.load(Ordering::Acquire)); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn ignored_sigchld_fails_before_reserving_a_child_reaper() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_IGNORED_SIGCHLD"; + const TEST_NAME: &str = + "process_session::tests::ignored_sigchld_fails_before_reserving_a_child_reaper"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "SIGCHLD preflight subprocess failed: {status}"); + return; + } + + // SAFETY: this isolated one-test subprocess intentionally changes its + // process-wide child disposition and exits immediately after the + // preflight assertion. + let previous = unsafe { libc::signal(libc::SIGCHLD, libc::SIG_IGN) }; + assert_ne!(previous, libc::SIG_ERR); + + let Err(error) = reserve_child_reaper() else { + panic!("ignored SIGCHLD passed the child-wait ownership preflight"); + }; + assert_eq!(error.kind(), io::ErrorKind::Unsupported); + } + #[cfg(target_os = "linux")] fn enter_syscall_blocked_subprocess( child_env: &str, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 34ed35a198d5..7fc36638cd88 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -851,6 +851,8 @@ struct LocalPtyProcess { #[cfg(unix)] child_reaped: AtomicBool, #[cfg(unix)] + child_wait_ownership_lost: AtomicBool, + #[cfg(unix)] group_escalation_complete: AtomicBool, #[cfg(all(unix, test))] normal_cleanup_failures: AtomicUsize, @@ -912,6 +914,8 @@ impl LocalPtyProcess { #[cfg(unix)] child_reaped: AtomicBool::new(false), #[cfg(unix)] + child_wait_ownership_lost: AtomicBool::new(false), + #[cfg(unix)] group_escalation_complete: AtomicBool::new(false), #[cfg(all(unix, test))] normal_cleanup_failures: AtomicUsize::new(0), @@ -944,6 +948,29 @@ impl LocalPtyProcess { "native PTY child did not expose a process ID", )); }; + let child_identity = match crate::process_session::StableProcessHandle::capture(session) + { + Ok(Some(identity)) => identity, + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + process.child_reaped.store(true, Ordering::Release); + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "native PTY child exited before its process identity was captured", + )); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + process.child_reaped.store(true, Ordering::Release); + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + return Err(error); + } + }; let observe_process = process.clone(); let cleanup_process = process.clone(); let prepare_process = process.clone(); @@ -954,12 +981,9 @@ impl LocalPtyProcess { session, NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, move || { - let observed = local_child_is_waitable(session)?; - if observed { - observe_process.child_waitable.store(true, Ordering::Release); - observe_process.exited.1.notify_all(); - } - Ok(observed) + let _signal = observe_process.child_signal_lock.lock().unwrap(); + let state = observe_process.observe_child_wait_state_locked(session)?; + Ok(state != crate::process_session::ChildWaitState::Running) }, move || { crate::process_session::reserved_child_needs_cleanup( @@ -969,11 +993,26 @@ impl LocalPtyProcess { termination_started: &cleanup_process.termination_started, cleanup_complete: &cleanup_process.group_escalation_complete, child_reaped: &cleanup_process.child_reaped, + wait_ownership_lost: &cleanup_process.child_wait_ownership_lost, }, ) }, move || prepare_process.prepare_natural_cleanup(), move |cleanup_succeeded| { + if reap_process.child_wait_ownership_lost.load(Ordering::Acquire) { + let _signal = reap_process.child_signal_lock.lock().unwrap(); + if !reap_process.pty_drained.load(Ordering::Acquire) + || !matches!(child_identity.matches_current(), Ok(false)) + { + return false; + } + drop(child.take()); + reap_process.child_reaped.store(true, Ordering::Release); + drop(_signal); + *reap_process.exited.0.lock().unwrap() = true; + reap_process.exited.1.notify_all(); + return true; + } let done = crate::process_session::poll_reserved_session_leader( crate::process_session::ReservedChildReap { signal_lock: &reap_process.child_signal_lock, @@ -981,6 +1020,7 @@ impl LocalPtyProcess { termination_started: &reap_process.termination_started, cleanup_complete: &reap_process.group_escalation_complete, child_reaped: &reap_process.child_reaped, + wait_ownership_lost: &reap_process.child_wait_ownership_lost, }, cleanup_succeeded, || { @@ -1036,6 +1076,16 @@ impl LocalPtyProcess { if self.child_reaped.load(Ordering::Acquire) { return true; } + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return false; + }; + if !matches!( + self.observe_child_wait_state_locked(session), + Ok(crate::process_session::ChildWaitState::Running + | crate::process_session::ChildWaitState::Waitable) + ) { + return false; + } return self.killer.lock().unwrap().kill().is_ok(); } #[cfg(not(unix))] @@ -1064,6 +1114,29 @@ impl LocalPtyProcess { *exited } + #[cfg(unix)] + fn observe_child_wait_state_locked( + &self, + session: libc::pid_t, + ) -> std::io::Result { + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Ok(crate::process_session::ChildWaitState::OwnershipLost); + } + let state = crate::process_session::observe_child_without_reaping(session, true)?; + match state { + crate::process_session::ChildWaitState::Waitable => { + self.child_waitable.store(true, Ordering::Release); + self.exited.1.notify_all(); + } + crate::process_session::ChildWaitState::OwnershipLost => { + self.child_wait_ownership_lost.store(true, Ordering::Release); + self.exited.1.notify_all(); + } + crate::process_session::ChildWaitState::Running => {} + } + Ok(state) + } + #[cfg(unix)] fn abandon_termination(&self) { self.termination_started.store(false, Ordering::Release); @@ -1096,6 +1169,9 @@ impl LocalPtyProcess { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } if self.child_reaped.load(Ordering::Acquire) { return match crate::process_session::session_is_empty_until(session, deadline) { Ok(true) => Ok(()), @@ -1105,6 +1181,11 @@ impl LocalPtyProcess { Err(error) => Err(error), }; } + if self.observe_child_wait_state_locked(session)? + == crate::process_session::ChildWaitState::OwnershipLost + { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } match snapshot { Some(snapshot) => { crate::process_session::signal_from_snapshot(snapshot, session, signal) @@ -1123,9 +1204,17 @@ impl LocalPtyProcess { let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } if self.child_reaped.load(Ordering::Acquire) { return crate::process_session::session_is_empty_until(leader, deadline); } + if self.observe_child_wait_state_locked(leader)? + == crate::process_session::ChildWaitState::OwnershipLost + { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } match snapshot { Some(snapshot) => crate::process_session::kill_until_only_leader_from_snapshot( snapshot, @@ -1190,13 +1279,20 @@ impl LocalPtyProcess { ) -> bool { #[cfg(unix)] { - let child_reaped = { + let (child_reaped, wait_ownership_lost) = { let _signal = self.child_signal_lock.lock().unwrap(); self.termination_started.store(true, Ordering::Release); - self.child_reaped.load(Ordering::Acquire) + ( + self.child_reaped.load(Ordering::Acquire), + self.child_wait_ownership_lost.load(Ordering::Acquire), + ) }; self.exited.1.notify_all(); crate::process_session::wake_child_reaper(); + if wait_ownership_lost { + self.abandon_termination(); + return self.wait_for_exit_until(deadline); + } if child_reaped { if self.group_escalation_complete.load(Ordering::Acquire) { return true; @@ -1246,7 +1342,7 @@ impl LocalPtyProcess { impl Drop for LocalPtyProcess { fn drop(&mut self) { #[cfg(unix)] - if !*self.child_reaped.get_mut() { + if !*self.child_reaped.get_mut() && !*self.child_wait_ownership_lost.get_mut() { let _ = self.killer.get_mut().map(|killer| killer.kill()); } #[cfg(not(unix))] @@ -1293,32 +1389,10 @@ impl Drop for NonblockingFdGuard { } } -#[cfg(unix)] +#[cfg(all(unix, test))] fn local_child_is_waitable(pid: libc::pid_t) -> std::io::Result { - loop { - let mut status = std::mem::MaybeUninit::::zeroed(); - let result = unsafe { - libc::waitid( - libc::P_PID, - pid as libc::id_t, - status.as_mut_ptr(), - libc::WEXITED | libc::WNOWAIT | libc::WNOHANG, - ) - }; - if result == 0 { - // SAFETY: waitid initialized status on success; zeroed storage - // keeps si_signo at zero when WNOHANG finds no waitable child. - return Ok(unsafe { status.assume_init() }.si_signo != 0); - } - let error = std::io::Error::last_os_error(); - if error.kind() == std::io::ErrorKind::Interrupted { - continue; - } - if error.raw_os_error() == Some(libc::ECHILD) { - return Ok(true); - } - return Err(error); - } + crate::process_session::observe_child_without_reaping(pid, true) + .map(|state| state == crate::process_session::ChildWaitState::Waitable) } #[cfg(unix)] @@ -3161,6 +3235,18 @@ impl Surface { self.as_pty().and_then(|pty| pty.host_identity.clone()) } + #[cfg(unix)] + pub(crate) fn live_terminal_host_identity( + &self, + ) -> Option { + let pty = self.as_pty()?; + let runtime = pty.runtime.lock().unwrap(); + match &*runtime { + PtyRuntime::Hosted(host) => Some(host.identity()), + PtyRuntime::ExitedHosted | PtyRuntime::Local { .. } => None, + } + } + pub fn terminal_host_connection_state(&self) -> Option { let pty = self.as_pty()?; pty.host_identity.as_ref()?; diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 3be2d957374c..d49215c11d3c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2394,6 +2394,7 @@ mod unix { termination_started: AtomicBool, child_signal_lock: Mutex<()>, child_reaped: AtomicBool, + child_wait_ownership_lost: AtomicBool, group_escalation_complete: AtomicBool, #[cfg(test)] normal_cleanup_failures: AtomicUsize, @@ -2723,6 +2724,30 @@ mod unix { self.pty_drained.load(Ordering::Acquire) } + fn observe_child_wait_state_locked( + &self, + session: libc::pid_t, + nonblocking: bool, + ) -> std::io::Result { + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Ok(crate::process_session::ChildWaitState::OwnershipLost); + } + let state = + crate::process_session::observe_child_without_reaping(session, nonblocking)?; + match state { + crate::process_session::ChildWaitState::Waitable => { + self.child_waitable.store(true, Ordering::Release); + self.child_exit.1.notify_all(); + } + crate::process_session::ChildWaitState::OwnershipLost => { + self.child_wait_ownership_lost.store(true, Ordering::Release); + self.child_exit.1.notify_all(); + } + crate::process_session::ChildWaitState::Running => {} + } + Ok(state) + } + fn signal_terminal_process_session( &self, signal: libc::c_int, @@ -2735,6 +2760,9 @@ mod unix { let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } if self.child_reaped.load(Ordering::Acquire) { return match crate::process_session::session_is_empty_until(session, deadline) { Ok(true) => Ok(()), @@ -2744,6 +2772,11 @@ mod unix { Err(error) => Err(error), }; } + if self.observe_child_wait_state_locked(session, true)? + == crate::process_session::ChildWaitState::OwnershipLost + { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } crate::process_session::signal_until(session, signal, deadline) } @@ -2752,9 +2785,17 @@ mod unix { let Some(leader) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); }; + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } if self.child_reaped.load(Ordering::Acquire) { return crate::process_session::session_is_empty_until(leader, deadline); } + if self.observe_child_wait_state_locked(leader, true)? + == crate::process_session::ChildWaitState::OwnershipLost + { + return Err(std::io::Error::other("PTY child wait ownership was lost")); + } crate::process_session::kill_until_only_leader(leader, leader, deadline, || { self.child_waitable.load(Ordering::Acquire) }) @@ -2833,13 +2874,20 @@ mod unix { } fn terminate_and_wait(&self) { - let child_reaped = { + let (child_reaped, wait_ownership_lost) = { let _signal = self.child_signal_lock.lock().unwrap(); self.termination_started.store(true, Ordering::Release); - self.child_reaped.load(Ordering::Acquire) + ( + self.child_reaped.load(Ordering::Acquire), + self.child_wait_ownership_lost.load(Ordering::Acquire), + ) }; self.child_exit.1.notify_all(); crate::process_session::wake_child_reaper(); + if wait_ownership_lost { + self.abandon_group_escalation(); + return; + } if child_reaped { let session_empty = self .pid @@ -2945,31 +2993,6 @@ mod unix { Ok(()) } - fn wait_for_child_exit_without_reaping(pid: libc::pid_t) -> std::io::Result<()> { - loop { - let mut status = std::mem::MaybeUninit::::uninit(); - // SAFETY: status points to writable siginfo storage. WNOWAIT - // observes this owned child becoming waitable without releasing - // its PID/PGID for reuse; the portable Child handle reaps it after - // acquiring child_signal_lock. - let result = unsafe { - libc::waitid( - libc::P_PID, - pid as libc::id_t, - status.as_mut_ptr(), - libc::WEXITED | libc::WNOWAIT, - ) - }; - if result == 0 { - return Ok(()); - } - let error = std::io::Error::last_os_error(); - if error.kind() != std::io::ErrorKind::Interrupted { - return Err(error); - } - } - } - struct HostLivenessLease { file: File, path: PathBuf, @@ -3222,6 +3245,26 @@ mod unix { pty.slave.spawn_command(command)? }; let pid = child.process_id(); + let Some(session) = pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + let _ = child.kill(); + let _ = child.wait(); + anyhow::bail!("native terminal-host PTY child did not expose a process ID"); + }; + let child_identity = match crate::process_session::StableProcessHandle::capture(session) { + Ok(Some(identity)) => identity, + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + anyhow::bail!( + "terminal-host PTY child exited before its process identity was captured" + ); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error).context("capture terminal-host PTY child identity"); + } + }; drop(pty.slave); let killer = child.clone_killer(); let pty_poll_fd = pty.master.as_raw_fd().context("open terminal-host PTY poll fd")?; @@ -3284,6 +3327,7 @@ mod unix { termination_started: AtomicBool::new(false), child_signal_lock: Mutex::new(()), child_reaped: AtomicBool::new(false), + child_wait_ownership_lost: AtomicBool::new(false), group_escalation_complete: AtomicBool::new(false), #[cfg(test)] normal_cleanup_failures: AtomicUsize::new( @@ -3373,14 +3417,13 @@ mod unix { })?; let child_host = shared.clone(); thread::Builder::new().name("terminal-host-child".into()).spawn(move || { - let observed_without_reaping = child_host - .pid - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .is_some_and(|pid| wait_for_child_exit_without_reaping(pid).is_ok()); - if observed_without_reaping { - let session = - libc::pid_t::try_from(child_host.pid.expect("observed child has a PID")) - .expect("portable-pty child PID fits pid_t"); + let observation = loop { + match crate::process_session::observe_child_without_reaping(session, false) { + Ok(crate::process_session::ChildWaitState::Running) => continue, + observation => break observation, + } + }; + if matches!(observation, Ok(crate::process_session::ChildWaitState::Waitable)) { child_host.child_waitable.store(true, Ordering::Release); child_host.child_exit.1.notify_all(); let cleanup_host = child_host.clone(); @@ -3400,6 +3443,7 @@ mod unix { termination_started: &cleanup_host.termination_started, cleanup_complete: &cleanup_host.group_escalation_complete, child_reaped: &cleanup_host.child_reaped, + wait_ownership_lost: &cleanup_host.child_wait_ownership_lost, }, ) }, @@ -3412,6 +3456,7 @@ mod unix { termination_started: &reap_host.termination_started, cleanup_complete: &reap_host.group_escalation_complete, child_reaped: &reap_host.child_reaped, + wait_ownership_lost: &reap_host.child_wait_ownership_lost, }, cleanup_succeeded, || { @@ -3431,30 +3476,21 @@ mod unix { }, ); } else { - // Native Unix PTYs always expose a PID and support waitid; - // retain a conservative fallback for alternate backends. - let _ = child.wait(); - child_host.child_reaped.store(true, Ordering::Release); - child_host.child_waitable.store(true, Ordering::Release); - let session_empty = child_host - .pid - .and_then(|pid| libc::pid_t::try_from(pid).ok()) - .is_some_and(|session| { - matches!( - crate::process_session::session_is_empty_until( - session, - Instant::now() + HOST_KILL_WAIT, - ), - Ok(true) - ) - }); - if session_empty { - let mut exited = child_host.child_exit.0.lock().unwrap(); - *exited = true; - child_host.child_exit.1.notify_all(); - drop(exited); - child_host.publish_exit_if_drained(); + { + let _signal = child_host.child_signal_lock.lock().unwrap(); + child_host.child_wait_ownership_lost.store(true, Ordering::Release); } + child_host.child_exit.1.notify_all(); + while !matches!(child_identity.matches_current(), Ok(false)) { + thread::sleep(Duration::from_millis(25)); + } + drop(child); + child_host.child_reaped.store(true, Ordering::Release); + let mut exited = child_host.child_exit.0.lock().unwrap(); + *exited = true; + child_host.child_exit.1.notify_all(); + drop(exited); + child_host.publish_exit_if_drained(); } })?; Ok(shared) From 8318effbcea73c1a8a09db3adccc483c75e8f9fe Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:04:52 -0700 Subject: [PATCH 135/521] test(tui): expose macOS ownership gaps --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 33 +++++++++++++ .../cmux-tui-core/src/process_session.rs | 17 +++++++ .../crates/cmux-tui/src/machine_runtime.rs | 48 +++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index f48eb6d74684..749ca4acd8a5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -914,6 +914,39 @@ pub mod transport { ); } + #[cfg(target_os = "macos")] + #[test] + fn transport_listener_creation_waits_for_the_process_barrier() { + use std::sync::mpsc; + + let _serial = SOCKET_CREATED_TEST_LOCK.lock().unwrap(); + let path = std::path::PathBuf::from("/tmp").join(format!( + "cmux-listen-barrier-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (result_sender, result_receiver) = mpsc::sync_channel(1); + let creator = std::thread::spawn({ + let path = path.clone(); + move || result_sender.send(listen(&path)).unwrap() + }); + + assert!( + result_receiver.recv_timeout(Duration::from_millis(250)).is_err(), + "Unix listener creation bypassed the process-wide descriptor barrier" + ); + drop(process_barrier); + let listener = result_receiver.recv_timeout(Duration::from_secs(2)).unwrap().unwrap(); + creator.join().unwrap(); + + drop(listener); + std::fs::remove_file(path).unwrap(); + } + #[cfg(target_os = "linux")] #[test] fn deadline_connect_times_out_while_listener_backlog_is_saturated() { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index e6a32c9c14dd..a4354dcb9402 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1547,6 +1547,23 @@ mod tests { child.wait().unwrap(); } + #[cfg(target_os = "macos")] + #[test] + fn macos_binary_does_not_strongly_import_optional_signal_api() { + let output = Command::new("/usr/bin/nm") + .arg("-u") + .arg(std::env::current_exe().unwrap()) + .output() + .unwrap(); + assert!(output.status.success()); + let imports = String::from_utf8(output.stdout).unwrap(); + + assert!( + !imports.lines().any(|line| line.contains("_proc_signal_with_audittoken")), + "an optional post-deployment-target API remained a strong dyld import" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn session_members_are_never_signaled_through_a_reusable_pid() { diff --git a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs index e72c1e673b19..c574d209b312 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs @@ -516,6 +516,54 @@ mod tests { ); } + #[cfg(target_os = "macos")] + #[test] + fn diagnostics_socket_pair_waits_for_the_process_barrier() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_DIAGNOSTICS_PAIR_BARRIER"; + const TEST_NAME: &str = + "machine_runtime::tests::diagnostics_socket_pair_waits_for_the_process_barrier"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "descriptor-barrier subprocess failed: {status}"); + return; + } + + fn descriptor_count() -> usize { + std::fs::read_dir("/dev/fd").unwrap().count() + } + + let baseline = descriptor_count(); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = std::sync::mpsc::sync_channel(1); + let worker = thread::spawn(move || { + let mut command = Command::new("/usr/bin/true"); + command.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); + started_sender.send(()).unwrap(); + spawn_transport_process(&mut command) + }); + started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + let deadline = Instant::now() + Duration::from_millis(250); + let mut observed = baseline; + while observed == baseline && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + observed = descriptor_count(); + } + + assert_eq!( + observed, baseline, + "UnixStream::pair created inheritable descriptors outside the process barrier" + ); + drop(process_barrier); + let (stdin, stdout, process) = worker.join().unwrap().unwrap(); + drop(stdin); + drop(stdout); + drop(process); + } + #[cfg(unix)] #[test] fn transport_cleanup_does_not_wait_for_descendant_inheriting_stderr() { From f11bd4121dc45da0b1ea572639cff8e21e897448 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:25:41 -0700 Subject: [PATCH 136/521] fix(tui): close macOS ownership gaps --- cmux-tui/Cargo.lock | 4 + cmux-tui/Cargo.toml | 1 + cmux-tui/crates/cmux-tui-cdp/src/client.rs | 7 +- cmux-tui/crates/cmux-tui-core/src/platform.rs | 8 +- .../cmux-tui-core/src/process_session.rs | 48 +++- cmux-tui/crates/cmux-tui-core/src/server.rs | 19 +- .../src/terminal_host_runtime.rs | 16 +- cmux-tui/crates/cmux-tui-process/Cargo.toml | 4 + cmux-tui/crates/cmux-tui-process/src/lib.rs | 229 +++++++++++++++++- .../src/machine_provider_transport.rs | 12 +- .../crates/cmux-tui/src/machine_runtime.rs | 2 +- 11 files changed, 314 insertions(+), 36 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 906aa9ff1a47..bb0fb095bacb 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -307,6 +307,10 @@ dependencies = [ [[package]] name = "cmux-tui-process" version = "0.1.0" +dependencies = [ + "libc", + "socket2", +] [[package]] name = "combine" diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index b672aa0783d4..1d83023f07d6 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -48,6 +48,7 @@ unicode-width = "0.2" unicode-segmentation = "1" base64 = "0.22" libc = "0.2" +socket2 = "0.6" tungstenite = { version = "0.29", default-features = false, features = ["handshake"] } uds_windows = "1.2" windows-sys = { version = "0.61", features = ["Win32_Networking_WinSock"] } diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index c3d9bd41197d..4ee65becada4 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -625,7 +625,7 @@ impl CdpClient { timeout: Duration, ) -> anyhow::Result { let remaining = remaining_until(deadline, "CDP connection deadline expired")?; - let stream = TcpStream::connect_timeout(&addr, remaining)?; + let stream = cmux_tui_process::tcp::connect_stream_timeout(&addr, remaining)?; let stream = DeadlineTcpStream::new(stream, deadline); stream.set_nodelay(true)?; let request = web_socket_url.into_client_request()?; @@ -1301,7 +1301,10 @@ fn fetch_json_version(host: &str, port: u16) -> anyhow::Result { let deadline = deadline_after(CDP_DISCOVERY_TIMEOUT, "CDP discovery deadline exceeded")?; let addr = resolve_socket_addr_until(host, port, deadline)?; let remaining = remaining_until(deadline, "CDP discovery deadline exceeded")?; - let mut stream = TcpStream::connect_timeout(&addr, remaining.min(Duration::from_millis(250)))?; + let mut stream = cmux_tui_process::tcp::connect_stream_timeout( + &addr, + remaining.min(Duration::from_millis(250)), + )?; stream.set_nodelay(true)?; let request = format!("GET /json/version HTTP/1.1\r\nHost: {host}:{port}\r\nConnection: close\r\n\r\n"); diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 749ca4acd8a5..4808a9521edc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -158,11 +158,11 @@ pub mod transport { } pub(super) fn listen(path: &Path) -> io::Result { - UnixListener::bind(path).map(|inner| Listener { inner }) + cmux_tui_process::unix::bind_listener(path).map(|inner| Listener { inner }) } pub(super) fn connect(path: &Path) -> io::Result> { - Ok(Box::new(UnixStream::connect(path)?)) + Ok(Box::new(cmux_tui_process::unix::connect_stream(path)?)) } pub(super) fn connect_until(path: &Path, deadline: Instant) -> io::Result> { @@ -341,14 +341,14 @@ pub mod transport { impl Listener { pub(super) fn accept(&self) -> io::Result> { - let (stream, _) = self.inner.accept()?; + let (stream, _) = cmux_tui_process::unix::accept_stream(&self.inner)?; Ok(Box::new(stream)) } } impl Stream for UnixStream { fn try_clone_box(&self) -> io::Result> { - Ok(Box::new(self.try_clone()?)) + Ok(Box::new(cmux_tui_process::unix::clone_stream(self)?)) } fn read_timeout(&self) -> io::Result> { diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index a4354dcb9402..522195ceaa3f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1011,11 +1011,12 @@ impl StableProcessHandle { /// Signal this exact process instance, returning false if it is already gone. pub fn signal(&self, signal: libc::c_int) -> io::Result { + let signal_process = resolve_proc_signal_with_audittoken()?; let mut token = self.audit_token; loop { // SAFETY: the audit token came from TASK_AUDIT_TOKEN for this // process instance; libproc validates its PID generation. - let result = unsafe { proc_signal_with_audittoken(&mut token, signal) }; + let result = unsafe { signal_process(&mut token, signal) }; if result == 0 { #[cfg(test)] STABLE_PROCESS_SIGNAL_COUNT.set(STABLE_PROCESS_SIGNAL_COUNT.get() + 1); @@ -1061,9 +1062,34 @@ unsafe extern "C" { } #[cfg(target_os = "macos")] -#[link(name = "proc")] -unsafe extern "C" { - fn proc_signal_with_audittoken(token: *mut MacAuditToken, signal: libc::c_int) -> libc::c_int; +type ProcSignalWithAuditToken = + unsafe extern "C" fn(*mut MacAuditToken, libc::c_int) -> libc::c_int; + +#[cfg(target_os = "macos")] +fn resolve_proc_signal_with_audittoken() -> io::Result { + static SYMBOL: OnceLock> = OnceLock::new(); + SYMBOL + .get_or_init(|| { + // SAFETY: RTLD_DEFAULT searches already loaded images. The + // resolved address is used only with the documented libproc ABI. + let address = + unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"proc_signal_with_audittoken".as_ptr()) }; + if address.is_null() { + return None; + } + // SAFETY: dlsym returned the address of the named C function. + Some(unsafe { + std::mem::transmute::<*mut libc::c_void, ProcSignalWithAuditToken>(address) + }) + }) + .as_ref() + .copied() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::Unsupported, + "exact process signaling is unavailable on this macOS version", + ) + }) } #[cfg(target_os = "linux")] @@ -1183,7 +1209,19 @@ pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::R let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { io::Error::new(io::ErrorKind::Unsupported, "current process identity is unavailable") })?; - match process.matches_current() { + #[cfg(target_os = "macos")] + let retained = { + resolve_proc_signal_with_audittoken().map_err(|error| { + io::Error::new( + io::ErrorKind::Unsupported, + format!("stable process signaling is unavailable: {error}"), + ) + })?; + process.matches_current() + }; + #[cfg(not(target_os = "macos"))] + let retained = process.signal(0); + match retained { Ok(true) => {} Ok(false) => { return Err(io::Error::new( diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 809130d3e1c8..3f821e78d8e0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -22,7 +22,9 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use std::io::{BufRead, BufReader, Read, Write}; #[cfg(unix)] use std::mem::{offset_of, size_of}; -use std::net::{Shutdown, SocketAddr, TcpListener, TcpStream}; +#[cfg(test)] +use std::net::TcpListener; +use std::net::{Shutdown, SocketAddr, TcpStream}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; @@ -2089,11 +2091,14 @@ impl SynchronizedTcpStream { } fn try_clone(&self) -> std::io::Result { - Ok(Self { stream: self.stream.try_clone()?, write_lock: self.write_lock.clone() }) + Ok(Self { + stream: cmux_tui_process::tcp::clone_stream(&self.stream)?, + write_lock: self.write_lock.clone(), + }) } fn try_clone_raw(&self) -> std::io::Result { - self.stream.try_clone() + cmux_tui_process::tcp::clone_stream(&self.stream) } fn set_read_timeout(&self, timeout: Option) -> std::io::Result<()> { @@ -2639,7 +2644,7 @@ impl Drop for WebSocketServer { for stream in self.connections.lock().unwrap().values() { let _ = stream.shutdown(Shutdown::Both); } - if let Ok(stream) = TcpStream::connect(self.local_addr) { + if let Ok(stream) = cmux_tui_process::tcp::connect_stream(self.local_addr) { let _ = stream.set_nodelay(true); } if let Some(thread) = self.thread.take() { @@ -2670,7 +2675,7 @@ pub fn serve_websocket( ); } } - let listener = TcpListener::bind(addr)?; + let listener = cmux_tui_process::tcp::bind_listener(addr)?; let local_addr = listener.local_addr()?; let shutdown = Arc::new(AtomicBool::new(false)); let connections = Arc::new(Mutex::new(HashMap::new())); @@ -2680,7 +2685,7 @@ pub fn serve_websocket( let thread_connections = connections.clone(); let thread = std::thread::Builder::new().name("mux-ws-server".into()).spawn(move || { while !thread_shutdown.load(Ordering::Acquire) { - let (stream, peer) = match listener.accept() { + let (stream, peer) = match cmux_tui_process::tcp::accept_stream(&listener) { Ok(connection) => connection, Err(_) => { if thread_shutdown.load(Ordering::Acquire) { @@ -2700,7 +2705,7 @@ pub fn serve_websocket( } let Some(permit) = claim_connection(&active_connections) else { continue }; let id = next_connection.fetch_add(1, Ordering::Relaxed); - if let Ok(tracked) = stream.try_clone() { + if let Ok(tracked) = cmux_tui_process::tcp::clone_stream(&stream) { thread_connections.lock().unwrap().insert(id, tracked); } let mux = mux.clone(); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index d49215c11d3c..00d318b59423 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -297,7 +297,9 @@ mod unix { #[cfg(test)] use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{FileTypeExt, MetadataExt, OpenOptionsExt, PermissionsExt}; - use std::os::unix::net::{UnixListener, UnixStream}; + #[cfg(test)] + use std::os::unix::net::UnixListener; + use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; @@ -2070,7 +2072,7 @@ mod unix { // forever while it sends input, mouse, resize, or Terminate. Reads are // unbounded because the dedicated reader thread is intentionally // long-lived and reconnects on any eventual EOF/protocol failure. - let reader = stream.try_clone()?; + let reader = cmux_tui_process::unix::clone_stream(&stream)?; let attachment = HostAttachment { record, record_path, @@ -3130,7 +3132,7 @@ mod unix { if let Some(parent) = endpoint.parent() { prepare_private_dir(parent)?; } - let listener = UnixListener::bind(&endpoint)?; + let listener = cmux_tui_process::unix::bind_listener(&endpoint)?; fs::set_permissions(&endpoint, fs::Permissions::from_mode(0o600))?; listener.set_nonblocking(true)?; @@ -3194,7 +3196,7 @@ mod unix { let _ = write_frame(writer, &response); while !shared.dead.load(Ordering::Acquire) { - match listener.accept() { + match cmux_tui_process::unix::accept_stream(&listener) { Ok((stream, _)) => { // Accepted sockets inherit O_NONBLOCK from the listener // on macOS. Client protocol threads use blocking framed @@ -3270,7 +3272,7 @@ mod unix { let pty_poll_fd = pty.master.as_raw_fd().context("open terminal-host PTY poll fd")?; let mut pty_reader = pty.master.try_clone_reader()?; let pty_writer = pty.master.take_writer()?; - let (pty_drain_waker, pty_drain_waiter) = UnixStream::pair()?; + let (pty_drain_waker, pty_drain_waiter) = cmux_tui_process::unix::pair_stream()?; let pending_responses = Arc::new(Mutex::new(Vec::::new())); let title_changed = Arc::new(AtomicBool::new(false)); @@ -3548,7 +3550,7 @@ mod unix { let tap = HostTap { sender, queued_bytes: Arc::new(AtomicUsize::new(0)), - shutdown: Arc::new(stream.try_clone()?), + shutdown: Arc::new(cmux_tui_process::unix::clone_stream(&stream)?), max_queued_bytes: MAX_HOST_CLIENT_QUEUED_BYTES, }; let command_sender = tap.clone(); @@ -3605,7 +3607,7 @@ mod unix { return Err(error.into()); } - let mut command_stream = stream.try_clone()?; + let mut command_stream = cmux_tui_process::unix::clone_stream(&stream)?; let command_host = host.clone(); thread::Builder::new().name("terminal-host-client-input".into()).spawn(move || { while let Ok(Some(frame)) = read_frame(&mut command_stream, MAX_FRAME_PAYLOAD) { diff --git a/cmux-tui/crates/cmux-tui-process/Cargo.toml b/cmux-tui/crates/cmux-tui-process/Cargo.toml index 6c4dae12fb3b..56f3269b8f6d 100644 --- a/cmux-tui/crates/cmux-tui-process/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-process/Cargo.toml @@ -9,3 +9,7 @@ description = "Process creation coordination for cmux-tui" [lints] workspace = true + +[target.'cfg(target_os = "macos")'.dependencies] +libc.workspace = true +socket2 = { workspace = true, features = ["all"] } diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index 073d7f511f38..182a7b0f8640 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -8,11 +8,11 @@ static PROCESS_CREATION_BARRIER: std::sync::Mutex<()> = std::sync::Mutex::new(() /// Holds the process-wide child-creation barrier. /// -/// macOS cannot create Unix sockets with close-on-exec set atomically. Socket -/// setup holds this guard until `FD_CLOEXEC` is set, while every child launch -/// holds the same guard until the child exists. Library-backed process launch -/// paths, such as PTY creation, must acquire this guard around their spawn -/// call. +/// macOS cannot create every socket descriptor with close-on-exec set +/// atomically. Socket setup holds this guard until `FD_CLOEXEC` is set, while +/// every child launch holds the same guard until the child exists. +/// Library-backed process launch paths, such as PTY creation, must acquire +/// this guard around their spawn call. #[must_use = "dropping the guard releases process creation"] pub struct ProcessCreationGuard { #[cfg(target_os = "macos")] @@ -39,3 +39,222 @@ pub fn spawn(command: &mut Command) -> io::Result { let _guard = ProcessCreationGuard::acquire(); command.spawn() } + +/// Unix socket operations whose descriptor creation is coordinated with +/// process launch on macOS. +#[cfg(unix)] +pub mod unix { + use std::io; + #[cfg(target_os = "macos")] + use std::os::fd::AsRawFd; + use std::os::unix::net::{SocketAddr, UnixListener, UnixStream}; + use std::path::Path; + + use super::ProcessCreationGuard; + + /// Bind a Unix listener without exposing its descriptor to a concurrent + /// child launch. + pub fn bind_listener(path: impl AsRef) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + UnixListener::bind(path) + } + + /// Connect a Unix stream without exposing its descriptor to a concurrent + /// child launch. + pub fn connect_stream(path: impl AsRef) -> io::Result { + #[cfg(target_os = "macos")] + { + let address = socket2::SockAddr::unix(path)?; + let socket = { + let _guard = ProcessCreationGuard::acquire(); + socket2::Socket::new(socket2::Domain::UNIX, socket2::Type::STREAM, None)? + }; + socket.connect(&address)?; + Ok(socket.into()) + } + #[cfg(not(target_os = "macos"))] + { + UnixStream::connect(path) + } + } + + /// Create a connected Unix stream pair without exposing either descriptor + /// to a concurrent child launch. + pub fn pair_stream() -> io::Result<(UnixStream, UnixStream)> { + let _guard = ProcessCreationGuard::acquire(); + UnixStream::pair() + } + + /// Clone a Unix stream without exposing the duplicate descriptor to a + /// concurrent child launch. + pub fn clone_stream(stream: &UnixStream) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + stream.try_clone() + } + + /// Clone a Unix listener without exposing the duplicate descriptor to a + /// concurrent child launch. + pub fn clone_listener(listener: &UnixListener) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + listener.try_clone() + } + + /// Accept a Unix stream without exposing the accepted descriptor to a + /// concurrent child launch. + pub fn accept_stream(listener: &UnixListener) -> io::Result<(UnixStream, SocketAddr)> { + #[cfg(target_os = "macos")] + { + accept_macos(listener.as_raw_fd(), || listener.accept()) + } + #[cfg(not(target_os = "macos"))] + { + listener.accept() + } + } + + #[cfg(target_os = "macos")] + pub(super) fn accept_macos( + descriptor: libc::c_int, + mut accept: impl FnMut() -> io::Result, + ) -> io::Result { + loop { + let blocking = descriptor_is_blocking(descriptor)?; + if blocking { + wait_until_readable(descriptor)?; + } + let _guard = ProcessCreationGuard::acquire(); + match accept() { + Err(error) + if blocking + && matches!( + error.kind(), + io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock + ) => {} + result => return result, + } + } + } + + #[cfg(target_os = "macos")] + fn descriptor_is_blocking(descriptor: libc::c_int) -> io::Result { + // SAFETY: F_GETFL only reads flags from this valid listener descriptor. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFL) }; + if flags < 0 { + return Err(io::Error::last_os_error()); + } + Ok(flags & libc::O_NONBLOCK == 0) + } + + #[cfg(target_os = "macos")] + fn wait_until_readable(descriptor: libc::c_int) -> io::Result<()> { + loop { + let mut poll_descriptor = + libc::pollfd { fd: descriptor, events: libc::POLLIN, revents: 0 }; + // SAFETY: poll_descriptor points to one initialized pollfd for + // the duration of the blocking readiness wait. + let result = unsafe { libc::poll(&raw mut poll_descriptor, 1, -1) }; + if result > 0 { + if poll_descriptor.revents & libc::POLLNVAL != 0 { + return Err(io::Error::from_raw_os_error(libc::EBADF)); + } + return Ok(()); + } + if result == 0 { + continue; + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + } + } +} + +/// TCP socket operations whose descriptor creation is coordinated with +/// process launch on macOS. +pub mod tcp { + use std::io; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::time::Duration; + + use super::ProcessCreationGuard; + + /// Bind a TCP listener without exposing its descriptor to a concurrent + /// child launch. + pub fn bind_listener(address: SocketAddr) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + TcpListener::bind(address) + } + + /// Connect a TCP stream without exposing its descriptor to a concurrent + /// child launch. + pub fn connect_stream(address: SocketAddr) -> io::Result { + #[cfg(target_os = "macos")] + { + let socket = new_stream_socket(address)?; + socket.connect(&address.into())?; + Ok(socket.into()) + } + #[cfg(not(target_os = "macos"))] + { + TcpStream::connect(address) + } + } + + /// Connect a bounded TCP stream without exposing its descriptor to a + /// concurrent child launch. + pub fn connect_stream_timeout( + address: &SocketAddr, + timeout: Duration, + ) -> io::Result { + #[cfg(target_os = "macos")] + { + let socket = new_stream_socket(*address)?; + socket.connect_timeout(&(*address).into(), timeout)?; + Ok(socket.into()) + } + #[cfg(not(target_os = "macos"))] + { + TcpStream::connect_timeout(address, timeout) + } + } + + /// Clone a TCP stream without exposing the duplicate descriptor to a + /// concurrent child launch. + pub fn clone_stream(stream: &TcpStream) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + stream.try_clone() + } + + /// Clone a TCP listener without exposing the duplicate descriptor to a + /// concurrent child launch. + pub fn clone_listener(listener: &TcpListener) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + listener.try_clone() + } + + /// Accept a TCP stream without exposing the accepted descriptor to a + /// concurrent child launch. + pub fn accept_stream(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + #[cfg(target_os = "macos")] + { + use std::os::fd::AsRawFd; + + super::unix::accept_macos(listener.as_raw_fd(), || listener.accept()) + } + #[cfg(not(target_os = "macos"))] + { + listener.accept() + } + } + + #[cfg(target_os = "macos")] + fn new_stream_socket(address: SocketAddr) -> io::Result { + let _guard = ProcessCreationGuard::acquire(); + socket2::Socket::new( + socket2::Domain::for_address(address), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + } +} diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs index 9c0ca3978f79..933c84d04768 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_transport.rs @@ -208,11 +208,13 @@ struct UnixMachineStreamConnector { impl MachineStreamConnector for UnixMachineStreamConnector { fn open(&self) -> io::Result { - let writer = UnixStream::connect(&self.socket_path)?; + let writer = cmux_tui_process::unix::connect_stream(&self.socket_path)?; writer.set_write_timeout(Some(PROVIDER_WRITE_TIMEOUT))?; - let reader = writer.try_clone()?; - let cleanup = - Arc::new(UnixCleanup { stream: writer.try_clone()?, closed: AtomicBool::new(false) }); + let reader = cmux_tui_process::unix::clone_stream(&writer)?; + let cleanup = Arc::new(UnixCleanup { + stream: cmux_tui_process::unix::clone_stream(&writer)?, + closed: AtomicBool::new(false), + }); Ok(ProviderIo::new(reader, writer, ProviderIoGuard::new(cleanup))) } } @@ -500,7 +502,7 @@ fn spawn_command( arguments: &[OsString], redactions: Arc>, ) -> io::Result { - let (stderr_cancel, stderr_cancel_worker) = UnixStream::pair()?; + let (stderr_cancel, stderr_cancel_worker) = cmux_tui_process::unix::pair_stream()?; let mut command = Command::new(program); command .args(arguments) diff --git a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs index c574d209b312..81f0b815b747 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs @@ -271,7 +271,7 @@ fn spawn_transport_process( command: &mut Command, ) -> anyhow::Result<(ChildStdin, ChildStdout, Arc)> { #[cfg(unix)] - let (stderr_cancel, stderr_cancel_worker) = UnixStream::pair() + let (stderr_cancel, stderr_cancel_worker) = cmux_tui_process::unix::pair_stream() .map_err(|error| anyhow::anyhow!("cannot create ssh diagnostics cancellation: {error}"))?; #[cfg(unix)] command.process_group(0); From 0697876775ced351566f3a276291e5bb410fb238 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:25:38 -0700 Subject: [PATCH 137/521] test(tui): expose host resolver regression --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 4ee65becada4..243aac692ce9 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -37,6 +37,12 @@ static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); +#[cfg(test)] +type TestHostResolver = Arc< + dyn Fn(&str, u16) -> std::io::Result> + Send + Sync, +>; +#[cfg(test)] +static TEST_HOST_RESOLVER: Mutex> = Mutex::new(None); static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -1772,6 +1778,28 @@ mod tests { ); } + #[test] + fn local_network_endpoint_uses_the_host_resolver() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + let called = Arc::new(AtomicBool::new(false)); + let called_from_resolver = called.clone(); + *TEST_HOST_RESOLVER.lock().unwrap() = Some(Arc::new(move |host, port| { + called_from_resolver.store(true, Ordering::Release); + assert_eq!(host, "cmux-host-resolver.invalid"); + Ok(vec![SocketAddr::from(([127, 0, 0, 1], port))]) + })); + + let result = resolve_socket_addr_until( + "cmux-host-resolver.invalid", + 9222, + Instant::now() + Duration::from_secs(2), + ); + *TEST_HOST_RESOLVER.lock().unwrap() = None; + + assert_eq!(result.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9222))); + assert!(called.load(Ordering::Acquire), "host resolver was bypassed"); + } + #[test] fn concurrent_calls_complete_while_reader_receives_events() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); From c391c7fbfb4a86ef2a36b763d66b006e78c1809f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:32:40 -0700 Subject: [PATCH 138/521] fix(tui): preserve host name resolution --- cmux-tui/Cargo.lock | 890 +-------------------- cmux-tui/Cargo.toml | 2 - cmux-tui/crates/cmux-tui-cdp/Cargo.toml | 2 - cmux-tui/crates/cmux-tui-cdp/src/client.rs | 140 ++-- 4 files changed, 76 insertions(+), 958 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index bb0fb095bacb..2f08a24cc945 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -32,17 +32,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "async-trait" -version = "0.1.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "atomic" version = "0.6.1" @@ -190,17 +179,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] - [[package]] name = "clang-sys" version = "1.8.1" @@ -253,9 +231,7 @@ dependencies = [ "anyhow", "base64", "cmux-tui-process", - "hickory-resolver", "serde_json", - "tokio", "tungstenite", ] @@ -279,7 +255,7 @@ dependencies = [ "sha2", "tungstenite", "uds_windows", - "windows-sys 0.61.2", + "windows-sys", "zeroize", ] @@ -312,16 +288,6 @@ dependencies = [ "socket2", ] -[[package]] -name = "combine" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" -dependencies = [ - "bytes", - "memchr", -] - [[package]] name = "compact_str" version = "0.9.1" @@ -345,22 +311,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -370,45 +320,12 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "critical-section" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" -[[package]] -name = "crossbeam-channel" -version = "0.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - [[package]] name = "crossterm" version = "0.29.0" @@ -538,17 +455,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "displaydoc" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "document-features" version = "0.2.12" @@ -583,7 +489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -670,15 +576,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - [[package]] name = "fs4" version = "1.1.0" @@ -686,16 +583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e72ed92b67c146290f88e9c89d60ca163ea417a446f61ffd7b72df3e7f1dfd5" dependencies = [ "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", + "windows-sys", ] [[package]] @@ -704,23 +592,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" -[[package]] -name = "futures-io" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" - -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "futures-task" version = "0.3.32" @@ -734,7 +605,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", - "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -750,17 +620,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "getrandom" version = "0.3.4" @@ -782,7 +641,6 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.1", ] [[package]] @@ -839,76 +697,6 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hickory-net" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "futures-channel", - "futures-io", - "futures-util", - "hickory-proto", - "idna", - "ipnet", - "jni", - "rand 0.10.2", - "thiserror 2.0.18", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "hickory-proto" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" -dependencies = [ - "data-encoding", - "idna", - "ipnet", - "jni", - "once_cell", - "prefix-trie", - "rand 0.10.2", - "ring", - "thiserror 2.0.18", - "tinyvec", - "tracing", - "url", -] - -[[package]] -name = "hickory-resolver" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" -dependencies = [ - "cfg-if", - "futures-util", - "hickory-net", - "hickory-proto", - "ipconfig", - "ipnet", - "jni", - "moka", - "ndk-context", - "once_cell", - "parking_lot", - "rand 0.10.2", - "resolv-conf", - "smallvec", - "system-configuration", - "thiserror 2.0.18", - "tokio", - "tracing", -] - [[package]] name = "http" version = "1.4.2" @@ -925,115 +713,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - [[package]] name = "indexmap" version = "2.14.0" @@ -1066,28 +751,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "ipconfig" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" -dependencies = [ - "socket2", - "widestring", - "windows-registry", - "windows-result", - "windows-sys 0.61.2", -] - -[[package]] -name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -dependencies = [ - "serde", -] - [[package]] name = "itertools" version = "0.13.0" @@ -1112,55 +775,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jni" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" -dependencies = [ - "cfg-if", - "combine", - "jni-macros", - "jni-sys", - "log", - "simd_cesu8", - "thiserror 2.0.18", - "walkdir", - "windows-link", -] - -[[package]] -name = "jni-macros" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn 2.0.119", -] - -[[package]] -name = "jni-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" -dependencies = [ - "jni-sys-macros", -] - -[[package]] -name = "jni-sys-macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" -dependencies = [ - "quote", - "syn 2.0.119", -] - [[package]] name = "js-sys" version = "0.3.103" @@ -1243,12 +857,6 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - [[package]] name = "litrs" version = "1.0.0" @@ -1325,32 +933,9 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys 0.61.2", + "windows-sys", ] -[[package]] -name = "moka" -version = "0.12.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" -dependencies = [ - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "parking_lot", - "portable-atomic", - "smallvec", - "tagptr", - "uuid", -] - -[[package]] -name = "ndk-context" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" - [[package]] name = "nix" version = "0.28.0" @@ -1426,10 +1011,6 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -dependencies = [ - "critical-section", - "portable-atomic", -] [[package]] name = "ordered-float" @@ -1487,12 +1068,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - [[package]] name = "pest" version = "2.8.7" @@ -1626,15 +1201,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -1650,17 +1216,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prefix-trie" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" -dependencies = [ - "either", - "ipnet", - "num-traits", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -1720,17 +1275,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core 0.10.1", -] - [[package]] name = "rand_chacha" version = "0.9.0" @@ -1756,12 +1300,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "ratatui" version = "0.30.2" @@ -1901,26 +1439,6 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "resolv-conf" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - [[package]] name = "rusqlite" version = "0.40.1" @@ -1959,7 +1477,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -1974,15 +1492,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -2055,7 +1564,7 @@ checksum = "9eb6ea5562eeaed6936b8b54e086aa0f88b9e5b1bef45beb038e2519fa1185b1" dependencies = [ "cfg-if", "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -2065,7 +1574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -2076,7 +1585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -2139,22 +1648,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd_cesu8" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" -dependencies = [ - "rustc_version", - "simdutf8", -] - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "siphasher" version = "1.0.3" @@ -2180,15 +1673,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "static_assertions" version = "1.1.0" @@ -2250,55 +1737,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags 2.13.1", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - [[package]] name = "tempfile" version = "3.27.0" @@ -2309,7 +1747,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -2322,7 +1760,7 @@ dependencies = [ "parking_lot", "rustix", "signal-hook", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -2449,57 +1887,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" -dependencies = [ - "bytes", - "libc", - "mio", - "pin-project-lite", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "toml" version = "0.8.23" @@ -2541,25 +1928,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", -] - [[package]] name = "tungstenite" version = "0.29.0" @@ -2596,7 +1964,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -2628,30 +1996,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8parse" version = "0.2.2" @@ -2691,16 +2035,6 @@ dependencies = [ "utf8parse", ] -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2833,12 +2167,6 @@ dependencies = [ "wezterm-dynamic", ] -[[package]] -name = "widestring" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" - [[package]] name = "winapi" version = "0.3.9" @@ -2855,15 +2183,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -2876,44 +2195,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-result" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -2923,70 +2204,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "winnow" version = "0.7.15" @@ -3011,35 +2228,6 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - [[package]] name = "zerocopy" version = "0.8.54" @@ -3060,66 +2248,12 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - [[package]] name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "zmij" version = "1.0.23" diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index 1d83023f07d6..800e7cf8b1cf 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -62,8 +62,6 @@ sha2 = "0.10" zeroize = "1.9" subtle = "2.6.1" parking_lot = "0.12" -hickory-resolver = { version = "0.26.1", default-features = false, features = ["system-config", "tokio"] } -tokio = { version = "1.47.1", features = ["net", "rt-multi-thread", "sync", "time"] } [patch.crates-io] # Crossterm 0.29 discards Kitty's shifted key, base-layout key, and associated diff --git a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml index 9cd588de1fd6..6c2dabbcfdf7 100644 --- a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml @@ -14,7 +14,5 @@ workspace = true anyhow.workspace = true base64.workspace = true cmux-tui-process.workspace = true -hickory-resolver.workspace = true serde_json.workspace = true -tokio.workspace = true tungstenite.workspace = true diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 243aac692ce9..241c7ad93a1f 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1,7 +1,7 @@ use std::collections::{HashMap, VecDeque}; use std::io::{Read, Write}; use std::mem::size_of; -use std::net::{IpAddr, SocketAddr, TcpStream}; +use std::net::{IpAddr, SocketAddr, TcpStream, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{ Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, @@ -9,10 +9,7 @@ use std::sync::mpsc::{ use std::sync::{Arc, Condvar, Mutex, Weak}; use std::time::{Duration, Instant}; -use hickory_resolver::TokioResolver; use serde_json::{Value, json}; -use tokio::runtime::{Builder as RuntimeBuilder, Runtime}; -use tokio::sync::Semaphore; use tungstenite::client::IntoClientRequest; use tungstenite::{Error as WsError, Message, WebSocket, client}; @@ -23,6 +20,7 @@ use tungstenite::{Error as WsError, Message, WebSocket, client}; pub const CDP_EVENT_QUEUE_CAPACITY: usize = 64; const CDP_INGRESS_EVENT_CAPACITY: usize = 1024; const CDP_RESOLVER_QUEUE_CAPACITY: usize = 64; +const CDP_RESOLVER_WORKERS: usize = 4; const CDP_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); /// Maximum estimated retained bytes in each bounded CDP event queue. /// @@ -38,9 +36,7 @@ static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] -type TestHostResolver = Arc< - dyn Fn(&str, u16) -> std::io::Result> + Send + Sync, ->; +type TestHostResolver = Arc std::io::Result> + Send + Sync>; #[cfg(test)] static TEST_HOST_RESOLVER: Mutex> = Mutex::new(None); @@ -48,15 +44,13 @@ static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); struct CdpResolver { - resolver: Arc<(Mutex, Condvar)>, - permits: Arc, - runtime: Runtime, + workers: Arc<(Mutex, Condvar)>, _initializer: std::thread::JoinHandle<()>, } enum ResolverState { Initializing, - Ready(Arc), + Ready(SyncSender), Failed(String), } @@ -71,14 +65,8 @@ struct ResolveRequest { impl CdpResolver { fn spawn() -> std::io::Result { - let runtime = RuntimeBuilder::new_multi_thread() - .worker_threads(1) - .thread_name("cmux-tui-cdp-resolver") - .enable_all() - .build()?; - let resolver = Arc::new((Mutex::new(ResolverState::Initializing), Condvar::new())); - let initializer_state = resolver.clone(); - let runtime_handle = runtime.handle().clone(); + let workers = Arc::new((Mutex::new(ResolverState::Initializing), Condvar::new())); + let initializer_state = workers.clone(); let initializer = std::thread::Builder::new() .name("cmux-tui-cdp-resolver-init".into()) .spawn(move || { @@ -90,78 +78,33 @@ impl CdpResolver { std::thread::sleep(Duration::from_millis(delay_ms)); } } - let _runtime = runtime_handle.enter(); - hickory_resolver::Resolver::builder_tokio() - .map_err(|error| error.to_string()) - .and_then(|builder| builder.build().map_err(|error| error.to_string())) + spawn_host_resolver_workers().map_err(|error| error.to_string()) })) .unwrap_or_else(|_| Err("resolver initializer panicked".to_string())); let (state, ready) = &*initializer_state; *state.lock().unwrap() = match result { - Ok(resolver) => ResolverState::Ready(Arc::new(resolver)), + Ok(sender) => ResolverState::Ready(sender), Err(error) => ResolverState::Failed(error), }; ready.notify_all(); })?; - Ok(Self { - resolver, - permits: Arc::new(Semaphore::new(CDP_RESOLVER_QUEUE_CAPACITY)), - runtime, - _initializer: initializer, - }) + Ok(Self { workers, _initializer: initializer }) } fn submit(&self, request: ResolveRequest) -> anyhow::Result<()> { - let permit = self - .permits - .clone() - .try_acquire_owned() - .map_err(|_| anyhow::anyhow!("CDP resolver queue is full"))?; - let resolver = self.resolver_until(request.deadline)?; - let _task = self.runtime.spawn(async move { - let deadline = tokio::time::Instant::from_std(request.deadline); - let result = if deadline <= tokio::time::Instant::now() { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "CDP address resolution deadline expired", - )) - } else { - let lookup = async { - #[cfg(test)] - if !request.delay.is_zero() { - tokio::time::sleep(request.delay).await; - } - resolver - .lookup_ip(request.host) - .await - .map(|addresses| { - addresses - .iter() - .map(|address| SocketAddr::new(address, request.port)) - .collect() - }) - .map_err(|error| std::io::Error::other(error.to_string())) - }; - match tokio::time::timeout_at(deadline, lookup).await { - Ok(result) => result, - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "CDP address resolution deadline expired", - )), - } - }; - let _ = request.response.send(result); - drop(permit); - }); - Ok(()) + let sender = self.workers_until(request.deadline)?; + sender.try_send(request).map_err(|error| match error { + TrySendError::Full(_) => anyhow::anyhow!("CDP resolver queue is full"), + TrySendError::Disconnected(_) => anyhow::anyhow!("CDP resolver workers stopped"), + }) } - fn resolver_until(&self, deadline: Instant) -> anyhow::Result> { - let (state, ready) = &*self.resolver; + fn workers_until(&self, deadline: Instant) -> anyhow::Result> { + let (state, ready) = &*self.workers; let mut state = state.lock().unwrap(); loop { match &*state { - ResolverState::Ready(resolver) => return Ok(resolver.clone()), + ResolverState::Ready(sender) => return Ok(sender.clone()), ResolverState::Failed(error) => { anyhow::bail!("CDP resolver initialization failed: {error}") } @@ -177,11 +120,56 @@ impl CdpResolver { } fn initialization_failed(&self) -> bool { - let state = self.resolver.0.lock().unwrap(); + let state = self.workers.0.lock().unwrap(); matches!(&*state, ResolverState::Failed(_)) } } +fn spawn_host_resolver_workers() -> std::io::Result> { + let (sender, receiver) = sync_channel(CDP_RESOLVER_QUEUE_CAPACITY); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..CDP_RESOLVER_WORKERS { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("cmux-tui-cdp-resolver-{index}")) + .spawn(move || host_resolver_worker(receiver))?; + } + Ok(sender) +} + +fn host_resolver_worker(receiver: Arc>>) { + loop { + let request = { + let receiver = receiver.lock().unwrap(); + let Ok(request) = receiver.recv() else { + return; + }; + request + }; + #[cfg(test)] + if !request.delay.is_zero() { + std::thread::sleep(request.delay); + } + let result = if request.deadline <= Instant::now() { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )) + } else { + resolve_host_addresses(&request.host, request.port) + }; + let _ = request.response.send(result); + } +} + +fn resolve_host_addresses(host: &str, port: u16) -> std::io::Result> { + #[cfg(test)] + if let Some(resolver) = TEST_HOST_RESOLVER.lock().unwrap().clone() { + return resolver(host, port); + } + (host, port).to_socket_addrs().map(Iterator::collect) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ScreencastFrame { pub session_id: String, From 3a5b9e77139d98785dcbb19656605856228f5fa8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:34:01 -0700 Subject: [PATCH 139/521] test(tui): expose remaining bounded lifecycle gaps --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 97 ++++++++++++++++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 47 +++++++++ .../src/terminal_host_runtime.rs | 72 ++++++++++++++ 3 files changed, 215 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 241c7ad93a1f..a181552c0774 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -39,6 +39,10 @@ static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); type TestHostResolver = Arc std::io::Result> + Send + Sync>; #[cfg(test)] static TEST_HOST_RESOLVER: Mutex> = Mutex::new(None); +#[cfg(test)] +type TestHostResolverCommand = Arc std::process::Command + Send + Sync>; +#[cfg(test)] +static TEST_HOST_RESOLVER_COMMAND: Mutex> = Mutex::new(None); static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -164,7 +168,9 @@ fn host_resolver_worker(receiver: Arc>>) { fn resolve_host_addresses(host: &str, port: u16) -> std::io::Result> { #[cfg(test)] - if let Some(resolver) = TEST_HOST_RESOLVER.lock().unwrap().clone() { + let test_resolver = TEST_HOST_RESOLVER.lock().unwrap().clone(); + #[cfg(test)] + if let Some(resolver) = test_resolver { return resolver(host, port); } (host, port).to_socket_addrs().map(Iterator::collect) @@ -1439,6 +1445,34 @@ mod tests { .expect("initialize test resolver"); } + fn resolver_fixture_command(host: &str, port: u16) -> std::process::Command { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "client::tests::host_resolver_process_fixture", + "--ignored", + "--nocapture", + ]) + .env( + "CMUX_TUI_TEST_HOST_RESOLVER_MODE", + if host.starts_with("cmux-blocked-") { "blocked" } else { "success" }, + ) + .env("CMUX_TUI_TEST_HOST_RESOLVER_PORT", port.to_string()); + command + } + + #[test] + #[ignore] + fn host_resolver_process_fixture() { + if std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_MODE").as_deref() == Ok("blocked") { + thread::sleep(Duration::from_secs(30)); + return; + } + let port = std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_PORT").unwrap(); + println!("127.0.0.1:{port}"); + } + #[test] fn key_event_params_omit_unavailable_physical_identity() { let params = key_event_params(CdpKeyEvent { @@ -1776,6 +1810,12 @@ mod tests { assert_eq!(host, "cmux-host-resolver.invalid"); Ok(vec![SocketAddr::from(([127, 0, 0, 1], port))]) })); + let called_from_command = called.clone(); + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(move |host, port| { + called_from_command.store(true, Ordering::Release); + assert_eq!(host, "cmux-host-resolver.invalid"); + resolver_fixture_command(host, port) + })); let result = resolve_socket_addr_until( "cmux-host-resolver.invalid", @@ -1783,11 +1823,66 @@ mod tests { Instant::now() + Duration::from_secs(2), ); *TEST_HOST_RESOLVER.lock().unwrap() = None; + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; assert_eq!(result.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9222))); assert!(called.load(Ordering::Acquire), "host resolver was bypassed"); } + #[test] + fn timed_out_host_lookups_do_not_strand_resolver_capacity() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + warm_resolver(); + let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let release = Arc::new(AtomicBool::new(false)); + let resolver_started = started.clone(); + let resolver_release = release.clone(); + *TEST_HOST_RESOLVER.lock().unwrap() = Some(Arc::new(move |_host, port| { + resolver_started.fetch_add(1, Ordering::AcqRel); + while !resolver_release.load(Ordering::Acquire) { + thread::sleep(Duration::from_millis(5)); + } + Ok(vec![SocketAddr::from(([127, 0, 0, 1], port))]) + })); + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(resolver_fixture_command)); + + let callers_ready = Arc::new(Barrier::new(CDP_RESOLVER_WORKERS + 1)); + let mut callers = Vec::new(); + for index in 0..CDP_RESOLVER_WORKERS { + let callers_ready = callers_ready.clone(); + callers.push(thread::spawn(move || { + callers_ready.wait(); + resolve_socket_addr_until( + &format!("cmux-blocked-{index}.invalid"), + 9222, + Instant::now() + Duration::from_millis(500), + ) + })); + } + callers_ready.wait(); + for caller in callers { + assert!(caller.join().unwrap().is_err()); + } + let blocked_workers = started.load(Ordering::Acquire); + + let next = resolve_socket_addr_until( + "cmux-success.invalid", + 9333, + Instant::now() + Duration::from_secs(1), + ); + + *TEST_HOST_RESOLVER.lock().unwrap() = None; + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; + release.store(true, Ordering::Release); + thread::sleep(Duration::from_millis(100)); + + assert!( + blocked_workers == 0 || blocked_workers == CDP_RESOLVER_WORKERS, + "resolver saturation setup reached only {blocked_workers} workers" + ); + assert_eq!(next.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9333))); + } + #[test] fn concurrent_calls_complete_while_reader_receives_events() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 7fc36638cd88..f83f3684bea1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4007,6 +4007,53 @@ fn terminal_scroll_position(term: &Terminal) -> (u64, bool) { mod tests { use super::*; + #[cfg(target_os = "macos")] + #[test] + fn local_pty_creation_waits_for_process_barrier() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_LOCAL_PTY_CREATION_BARRIER"; + const TEST_NAME: &str = "surface::tests::local_pty_creation_waits_for_process_barrier"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "PTY barrier subprocess failed: {status}"); + return; + } + + fn descriptor_count() -> usize { + std::fs::read_dir("/dev/fd").unwrap().count() + } + + drop(reserve_local_child_reaper().unwrap()); + let mux = Mux::new_for_test("local-pty-barrier", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec!["/usr/bin/true".into()]), + ..SurfaceOptions::default() + }; + let baseline = descriptor_count(); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = sync_channel(1); + let worker = std::thread::spawn(move || { + started_sender.send(()).unwrap(); + Surface::spawn(1, options, Arc::downgrade(&mux)) + }); + started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + let deadline = Instant::now() + Duration::from_millis(250); + let mut observed = baseline; + while observed == baseline && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + observed = descriptor_count(); + } + + drop(process_barrier); + let surface = worker.join().unwrap().unwrap(); + let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); + + assert_eq!(observed, baseline, "PTY descriptors were created outside the process barrier"); + } + #[derive(Clone, Default)] struct CapturingWriter(Arc>>); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 00d318b59423..8fb0a84fba2b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4051,6 +4051,78 @@ mod unix { mod tests { use super::*; + #[cfg(target_os = "macos")] + #[test] + fn host_pty_creation_waits_for_process_barrier() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_HOST_PTY_CREATION_BARRIER"; + const TEST_NAME: &str = + "terminal_host_runtime::unix::tests::host_pty_creation_waits_for_process_barrier"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "host PTY barrier subprocess failed: {status}"); + return; + } + + fn descriptor_count() -> usize { + fs::read_dir("/dev/fd").unwrap().count() + } + + drop(crate::process_session::reserve_child_reaper().unwrap()); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: "/tmp/cmux-host-pty-barrier.sock".into(), + record_path: "/tmp/cmux-host-pty-barrier.json".into(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec!["/usr/bin/true".into()], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + let baseline = descriptor_count(); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = sync_channel(1); + let worker = thread::spawn(move || { + started_sender.send(()).unwrap(); + spawn_host_runtime(&launch, &bootstrapped) + }); + started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + let deadline = Instant::now() + Duration::from_millis(250); + let mut observed = baseline; + while observed == baseline && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + observed = descriptor_count(); + } + + drop(process_barrier); + let host = worker.join().unwrap().unwrap(); + host.request_termination(); + let _ = host.wait_for_child_exit(Duration::from_secs(1)); + + assert_eq!( + observed, baseline, + "host PTY descriptors were created outside the process barrier" + ); + } + #[cfg(target_os = "linux")] #[test] fn saturated_listener_never_returns_an_unconnected_host_stream() { From 8121e1441417d3194c1c5782e026a4f5496a8e35 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:53:06 -0700 Subject: [PATCH 140/521] fix(tui): bound PTY and resolver creation --- cmux-tui/Cargo.lock | 1 + cmux-tui/crates/cmux-tui-cdp/src/client.rs | 228 +++++++++++++++--- cmux-tui/crates/cmux-tui-cdp/src/lib.rs | 1 + cmux-tui/crates/cmux-tui-core/src/surface.rs | 7 +- .../src/terminal_host_runtime.rs | 7 +- cmux-tui/crates/cmux-tui/Cargo.toml | 1 + cmux-tui/crates/cmux-tui/src/main.rs | 8 + 7 files changed, 206 insertions(+), 47 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 2f08a24cc945..7f2dbea8d5ce 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -205,6 +205,7 @@ version = "0.1.0" dependencies = [ "anyhow", "base64", + "cmux-tui-cdp", "cmux-tui-core", "cmux-tui-machine-agent-protocol", "cmux-tui-machine-protocol", diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index a181552c0774..3c2fce493af2 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -2,7 +2,8 @@ use std::collections::{HashMap, VecDeque}; use std::io::{Read, Write}; use std::mem::size_of; use std::net::{IpAddr, SocketAddr, TcpStream, ToSocketAddrs}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{ Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, }; @@ -21,6 +22,9 @@ pub const CDP_EVENT_QUEUE_CAPACITY: usize = 64; const CDP_INGRESS_EVENT_CAPACITY: usize = 1024; const CDP_RESOLVER_QUEUE_CAPACITY: usize = 64; const CDP_RESOLVER_WORKERS: usize = 4; +const CDP_RESOLVER_CHILD_CAPACITY: usize = CDP_RESOLVER_QUEUE_CAPACITY + CDP_RESOLVER_WORKERS; +const CDP_RESOLVER_POLL_INTERVAL: Duration = Duration::from_millis(5); +const CDP_RESOLVER_OUTPUT_LIMIT: u64 = 64 * 1024; const CDP_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); /// Maximum estimated retained bytes in each bounded CDP event queue. /// @@ -36,16 +40,14 @@ static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] -type TestHostResolver = Arc std::io::Result> + Send + Sync>; -#[cfg(test)] -static TEST_HOST_RESOLVER: Mutex> = Mutex::new(None); -#[cfg(test)] -type TestHostResolverCommand = Arc std::process::Command + Send + Sync>; +type TestHostResolverCommand = Arc Command + Send + Sync>; #[cfg(test)] static TEST_HOST_RESOLVER_COMMAND: Mutex> = Mutex::new(None); static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); +static CDP_RESOLVER_CHILD_REAPER: std::sync::OnceLock>> = + std::sync::OnceLock::new(); struct CdpResolver { workers: Arc<(Mutex, Condvar)>, @@ -67,6 +69,28 @@ struct ResolveRequest { delay: Duration, } +struct ResolverChildReaper { + sender: Sender, + active: Arc, + _worker: std::thread::JoinHandle<()>, +} + +struct ResolverChildReaperLease { + sender: Sender, + active: Arc, +} + +impl Drop for ResolverChildReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +struct ResolverChildReapRequest { + child: Child, + _lease: ResolverChildReaperLease, +} + impl CdpResolver { fn spawn() -> std::io::Result { let workers = Arc::new((Mutex::new(ResolverState::Initializing), Condvar::new())); @@ -130,6 +154,7 @@ impl CdpResolver { } fn spawn_host_resolver_workers() -> std::io::Result> { + resolver_child_reaper()?; let (sender, receiver) = sync_channel(CDP_RESOLVER_QUEUE_CAPACITY); let receiver = Arc::new(Mutex::new(receiver)); for index in 0..CDP_RESOLVER_WORKERS { @@ -160,20 +185,170 @@ fn host_resolver_worker(receiver: Arc>>) { "CDP address resolution deadline expired", )) } else { - resolve_host_addresses(&request.host, request.port) + resolve_host_addresses_until(&request.host, request.port, request.deadline) }; let _ = request.response.send(result); } } -fn resolve_host_addresses(host: &str, port: u16) -> std::io::Result> { +fn resolve_host_addresses_until( + host: &str, + port: u16, + deadline: Instant, +) -> std::io::Result> { #[cfg(test)] - let test_resolver = TEST_HOST_RESOLVER.lock().unwrap().clone(); + { + let test_command = TEST_HOST_RESOLVER_COMMAND.lock().unwrap().clone(); + if test_command.is_none() { + return (host, port).to_socket_addrs().map(Iterator::collect); + } + } + + let reaper = reserve_resolver_child_reaper()?; + let mut command = host_resolver_command(host, port)?; + command.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()); + let mut child = cmux_tui_process::spawn(&mut command)?; + loop { + match child.try_wait() { + Ok(Some(status)) => { + drop(reaper); + if !status.success() { + return Err(std::io::Error::other(format!( + "host resolver helper exited with {status}" + ))); + } + let stdout = child.stdout.take().ok_or_else(|| { + std::io::Error::other("host resolver helper stdout was unavailable") + })?; + let mut output = Vec::new(); + stdout.take(CDP_RESOLVER_OUTPUT_LIMIT + 1).read_to_end(&mut output)?; + if output.len() as u64 > CDP_RESOLVER_OUTPUT_LIMIT { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "host resolver helper output exceeded its limit", + )); + } + let addresses = String::from_utf8_lossy(&output) + .lines() + .filter_map(|line| line.parse::().ok()) + .filter(|address| address.port() == port) + .collect::>(); + if addresses.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "host resolver helper returned no addresses", + )); + } + return Ok(addresses); + } + Ok(None) => {} + Err(error) => { + handoff_resolver_child(child, reaper); + return Err(error); + } + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + handoff_resolver_child(child, reaper); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )); + }; + std::thread::sleep(remaining.min(CDP_RESOLVER_POLL_INTERVAL)); + } +} + +fn host_resolver_command(host: &str, port: u16) -> std::io::Result { #[cfg(test)] - if let Some(resolver) = test_resolver { - return resolver(host, port); + if let Some(command) = TEST_HOST_RESOLVER_COMMAND.lock().unwrap().clone() { + return Ok(command(host, port)); + } + let mut command = Command::new(std::env::current_exe()?); + command.args(["__resolve-host", host, &port.to_string()]); + Ok(command) +} + +fn resolver_child_reaper() -> std::io::Result<()> { + let mut slot = CDP_RESOLVER_CHILD_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); + if slot.is_none() { + let (sender, receiver) = channel(); + let active = Arc::new(AtomicUsize::new(0)); + let worker = std::thread::Builder::new() + .name("cmux-tui-cdp-resolver-reaper".into()) + .spawn(move || run_resolver_child_reaper(receiver))?; + *slot = Some(ResolverChildReaper { sender, active, _worker: worker }); + } + Ok(()) +} + +fn reserve_resolver_child_reaper() -> std::io::Result { + resolver_child_reaper()?; + let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); + let reaper = slot.as_ref().expect("resolver child reaper initialized"); + reaper + .active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < CDP_RESOLVER_CHILD_CAPACITY).then_some(active + 1) + }) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "host resolver child capacity exhausted", + ) + })?; + Ok(ResolverChildReaperLease { sender: reaper.sender.clone(), active: reaper.active.clone() }) +} + +fn handoff_resolver_child(mut child: Child, lease: ResolverChildReaperLease) { + let _ = child.kill(); + let sender = lease.sender.clone(); + let request = ResolverChildReapRequest { child, _lease: lease }; + if let Err(error) = sender.send(request) { + let mut request = error.0; + let _ = request.child.kill(); + let _ = request.child.wait(); + } +} + +fn run_resolver_child_reaper(receiver: Receiver) { + let mut pending = Vec::new(); + loop { + match receiver.recv_timeout(CDP_RESOLVER_POLL_INTERVAL) { + Ok(request) => pending.push(request), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) if pending.is_empty() => return, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {} + } + while let Ok(request) = receiver.try_recv() { + pending.push(request); + } + pending.retain_mut(|request| { + let _ = request.child.kill(); + !matches!(request.child.try_wait(), Ok(Some(_))) + }); } - (host, port).to_socket_addrs().map(Iterator::collect) +} + +#[doc(hidden)] +pub fn run_host_resolver_helper(args: &[String]) -> anyhow::Result<()> { + let [host, port] = args else { + anyhow::bail!("host resolver helper requires a host and port"); + }; + let port = port.parse::()?; + let addresses = (host.as_str(), port).to_socket_addrs()?.take(257).collect::>(); + if addresses.is_empty() { + anyhow::bail!("host resolver returned no addresses"); + } + if addresses.len() > 256 { + anyhow::bail!("host resolver returned too many addresses"); + } + let stdout = std::io::stdout(); + let mut output = std::io::BufWriter::new(stdout.lock()); + for address in addresses { + writeln!(output, "{address}")?; + } + output.flush()?; + Ok(()) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -1445,8 +1620,8 @@ mod tests { .expect("initialize test resolver"); } - fn resolver_fixture_command(host: &str, port: u16) -> std::process::Command { - let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + fn resolver_fixture_command(host: &str, port: u16) -> Command { + let mut command = Command::new(std::env::current_exe().unwrap()); command .args([ "--exact", @@ -1804,12 +1979,6 @@ mod tests { fn local_network_endpoint_uses_the_host_resolver() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); let called = Arc::new(AtomicBool::new(false)); - let called_from_resolver = called.clone(); - *TEST_HOST_RESOLVER.lock().unwrap() = Some(Arc::new(move |host, port| { - called_from_resolver.store(true, Ordering::Release); - assert_eq!(host, "cmux-host-resolver.invalid"); - Ok(vec![SocketAddr::from(([127, 0, 0, 1], port))]) - })); let called_from_command = called.clone(); *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(move |host, port| { called_from_command.store(true, Ordering::Release); @@ -1822,7 +1991,6 @@ mod tests { 9222, Instant::now() + Duration::from_secs(2), ); - *TEST_HOST_RESOLVER.lock().unwrap() = None; *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; assert_eq!(result.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9222))); @@ -1833,17 +2001,6 @@ mod tests { fn timed_out_host_lookups_do_not_strand_resolver_capacity() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); - let started = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let release = Arc::new(AtomicBool::new(false)); - let resolver_started = started.clone(); - let resolver_release = release.clone(); - *TEST_HOST_RESOLVER.lock().unwrap() = Some(Arc::new(move |_host, port| { - resolver_started.fetch_add(1, Ordering::AcqRel); - while !resolver_release.load(Ordering::Acquire) { - thread::sleep(Duration::from_millis(5)); - } - Ok(vec![SocketAddr::from(([127, 0, 0, 1], port))]) - })); *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(resolver_fixture_command)); let callers_ready = Arc::new(Barrier::new(CDP_RESOLVER_WORKERS + 1)); @@ -1863,7 +2020,6 @@ mod tests { for caller in callers { assert!(caller.join().unwrap().is_err()); } - let blocked_workers = started.load(Ordering::Acquire); let next = resolve_socket_addr_until( "cmux-success.invalid", @@ -1871,15 +2027,9 @@ mod tests { Instant::now() + Duration::from_secs(1), ); - *TEST_HOST_RESOLVER.lock().unwrap() = None; *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; - release.store(true, Ordering::Release); thread::sleep(Duration::from_millis(100)); - assert!( - blocked_workers == 0 || blocked_workers == CDP_RESOLVER_WORKERS, - "resolver saturation setup reached only {blocked_workers} workers" - ); assert_eq!(next.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9333))); } diff --git a/cmux-tui/crates/cmux-tui-cdp/src/lib.rs b/cmux-tui/crates/cmux-tui-cdp/src/lib.rs index 35d31556d32a..e5ca860ec23e 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/lib.rs @@ -12,4 +12,5 @@ pub use client::{ CDP_EVENT_QUEUE_CAPACITY, CDP_EVENT_QUEUE_MAX_BYTES, CdpClient, CdpEvent, CdpKeyEvent, NavigationEntry, NavigationHistory, ScreencastFrame, TargetCreated, TargetInfo, discover_browser_ws_url, event_retained_bytes, resolve_browser_ws_url, + run_host_resolver_helper, }; diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index f83f3684bea1..58fcebed71f4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1739,6 +1739,7 @@ impl Surface { } let _ = terminal_id; let reaper = reserve_local_child_reaper().context("reserve bounded PTY session cleanup")?; + let process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); let pty = native_pty_system().openpty(PtySize { rows: opts.rows, cols: opts.cols, @@ -1765,10 +1766,8 @@ impl Surface { cmd.cwd(cwd); } - let child = { - let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); - pty.slave.spawn_command(cmd)? - }; + let child = pty.slave.spawn_command(cmd)?; + drop(process_creation); let pid = child.process_id(); drop(pty.slave); let killer = child.clone_killer(); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 8fb0a84fba2b..9473a706083c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3227,6 +3227,7 @@ mod unix { ) -> anyhow::Result> { let reaper = crate::process_session::reserve_child_reaper() .context("reserve bounded PTY session cleanup")?; + let process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); let pty = native_pty_system().openpty(PtySize { rows: launch.rows, cols: launch.cols, @@ -3242,10 +3243,8 @@ mod unix { if let Some(cwd) = launch.cwd.as_deref() { command.cwd(cwd); } - let mut child = { - let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); - pty.slave.spawn_command(command)? - }; + let mut child = pty.slave.spawn_command(command)?; + drop(process_creation); let pid = child.process_id(); let Some(session) = pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { let _ = child.kill(); diff --git a/cmux-tui/crates/cmux-tui/Cargo.toml b/cmux-tui/crates/cmux-tui/Cargo.toml index c1b21a0e27e0..13ec629d0cd3 100644 --- a/cmux-tui/crates/cmux-tui/Cargo.toml +++ b/cmux-tui/crates/cmux-tui/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] ghostty-vt.workspace = true +cmux-tui-cdp.workspace = true cmux-tui-core.workspace = true cmux-tui-process.workspace = true cmux-tui-machine-agent-protocol.workspace = true diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 2718514bb1b9..e6f9137177fd 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -684,6 +684,14 @@ fn main() { } return; } + if raw_args.first().map(String::as_str) == Some("__resolve-host") { + discard_provider_secret_environment(); + if let Err(error) = cmux_tui_cdp::run_host_resolver_helper(&raw_args[1..]) { + eprintln!("cmux-tui host resolver: {error}"); + std::process::exit(1); + } + return; + } #[cfg(unix)] if raw_args.first().map(String::as_str) == Some("__legacy-stop-helper") { discard_provider_secret_environment(); From 62e3421902b1b201e0de06fc13242c77d355b991 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:33:10 -0700 Subject: [PATCH 141/521] test(tui): expose unbounded socket barrier wait --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 4808a9521edc..d2fdcbbade07 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -914,6 +914,60 @@ pub mod transport { ); } + #[cfg(target_os = "macos")] + #[test] + fn deadline_connect_bounds_process_barrier_wait() { + use std::sync::mpsc; + + let _serial = SOCKET_CREATED_TEST_LOCK.lock().unwrap(); + let path = std::path::PathBuf::from("/tmp").join(format!( + "cmux-connect-barrier-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (result_sender, result_receiver) = mpsc::sync_channel(1); + let connector = std::thread::spawn({ + let path = path.clone(); + move || { + let started = Instant::now(); + let result = + connect_unix_until(&path, started + Duration::from_millis(40)).map(drop); + result_sender.send((started.elapsed(), result)).unwrap(); + } + }); + + let before_release = result_receiver.recv_timeout(Duration::from_millis(150)); + drop(process_barrier); + let completed_before_release = before_release.is_ok(); + let (elapsed, result) = before_release.unwrap_or_else(|_| { + result_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("connector did not finish after the process barrier was released") + }); + connector.join().unwrap(); + drop(listener); + std::fs::remove_file(path).unwrap(); + + assert!( + completed_before_release, + "deadline connector waited for an unbounded process barrier" + ); + assert_eq!( + result.unwrap_err().kind(), + io::ErrorKind::TimedOut, + "barrier deadline returned the wrong transport error" + ); + assert!( + elapsed < Duration::from_millis(150), + "barrier deadline exceeded its wall-clock bound: {elapsed:?}" + ); + } + #[cfg(target_os = "macos")] #[test] fn transport_listener_creation_waits_for_the_process_barrier() { From 0d78e912ce6032b4d764df54fe15d97c891910f8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:40:22 -0700 Subject: [PATCH 142/521] fix(tui): bound socket barrier acquisition --- cmux-tui/crates/cmux-tui-core/src/platform.rs | 11 ++-- .../src/terminal_host_runtime.rs | 38 ++++++++++--- cmux-tui/crates/cmux-tui-process/src/lib.rs | 57 ++++++++++++++++++- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index d2fdcbbade07..f6c931f28f8a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -79,7 +79,7 @@ pub mod transport { } #[cfg(target_os = "linux")] - fn create_close_on_exec_socket() -> io::Result { + fn create_close_on_exec_socket(_deadline: Instant) -> io::Result { // SAFETY: socket has no pointer arguments and returns a new owned // descriptor on success. SOCK_CLOEXEC sets the inheritance flag // atomically with descriptor creation. @@ -109,8 +109,9 @@ pub mod transport { } #[cfg(target_os = "macos")] - fn create_close_on_exec_socket() -> io::Result { - let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); + fn create_close_on_exec_socket(deadline: Instant) -> io::Result { + let _process_creation = + cmux_tui_process::ProcessCreationGuard::acquire_until(deadline)?; // macOS has no SOCK_CLOEXEC socket flag. Every cmux-tui process // launch shares this barrier, so no child can start until fcntl // marks the fresh descriptor close-on-exec. @@ -150,7 +151,7 @@ pub mod transport { } #[cfg(not(any(target_os = "linux", target_os = "macos")))] - fn create_close_on_exec_socket() -> io::Result { + fn create_close_on_exec_socket(_deadline: Instant) -> io::Result { Err(io::Error::new( io::ErrorKind::Unsupported, "deadline sockets require atomic close-on-exec setup", @@ -172,7 +173,7 @@ pub mod transport { pub(super) fn connect_unix_until(path: &Path, deadline: Instant) -> io::Result { ensure_connect_time_remaining(deadline)?; let (address, address_len) = unix_socket_address(path)?; - let descriptor = create_close_on_exec_socket()?; + let descriptor = create_close_on_exec_socket(deadline)?; let stream = UnixStream::from(descriptor); stream.set_nonblocking(true)?; loop { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 9473a706083c..5796a24af89d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4212,6 +4212,24 @@ mod unix { (record_path, record, lease) } + fn accept_test_client_until( + listener: &UnixListener, + timeout: Duration, + ) -> Option { + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + timeout; + loop { + match listener.accept() { + Ok((stream, _)) => return Some(stream), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + let remaining = deadline.checked_duration_since(Instant::now())?; + thread::sleep(remaining.min(Duration::from_millis(1))); + } + Err(error) => panic!("test listener failed: {error}"), + } + } + } + #[test] fn launch_round_trip_preserves_ghostty_defaults() { let mut default_colors = DefaultColors { @@ -4755,8 +4773,11 @@ mod unix { let _ = fs::remove_file(&endpoint); let listener = UnixListener::bind(&endpoint).unwrap(); let stalled = thread::spawn(move || { - let (_stream, _) = listener.accept().unwrap(); - thread::sleep(Duration::from_millis(200)); + if let Some(_stream) = + accept_test_client_until(&listener, Duration::from_millis(500)) + { + thread::sleep(Duration::from_millis(200)); + } }); let started = Instant::now(); @@ -4784,12 +4805,15 @@ mod unix { let _ = fs::remove_file(&endpoint); let listener = UnixListener::bind(&endpoint).unwrap(); let stalled = thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - for _ in 0..crate::terminal_host_protocol::HEADER_LEN { - if stream.write_all(&[0]).is_err() { - break; + if let Some(mut stream) = + accept_test_client_until(&listener, Duration::from_millis(500)) + { + for _ in 0..crate::terminal_host_protocol::HEADER_LEN { + if stream.write_all(&[0]).is_err() { + break; + } + thread::sleep(Duration::from_millis(20)); } - thread::sleep(Duration::from_millis(20)); } }); diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index 182a7b0f8640..ac8db77773f9 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -32,6 +32,37 @@ impl ProcessCreationGuard { Self {} } } + + /// Acquire the process barrier without exceeding an operation deadline. + pub fn acquire_until(deadline: std::time::Instant) -> io::Result { + #[cfg(target_os = "macos")] + { + loop { + match PROCESS_CREATION_BARRIER.try_lock() { + Ok(guard) => return Ok(Self { _guard: guard }), + Err(std::sync::TryLockError::Poisoned(error)) => { + return Ok(Self { _guard: error.into_inner() }); + } + Err(std::sync::TryLockError::WouldBlock) => {} + } + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::TimedOut, + "process creation barrier acquisition timed out", + ) + })?; + std::thread::sleep(remaining.min(std::time::Duration::from_millis(1))); + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = deadline; + Ok(Self {}) + } + } } /// Spawn a child while excluding non-atomic close-on-exec descriptor setup. @@ -175,7 +206,7 @@ pub mod unix { pub mod tcp { use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; - use std::time::Duration; + use std::time::{Duration, Instant}; use super::ProcessCreationGuard; @@ -209,8 +240,15 @@ pub mod tcp { ) -> io::Result { #[cfg(target_os = "macos")] { - let socket = new_stream_socket(*address)?; - socket.connect_timeout(&(*address).into(), timeout)?; + let deadline = Instant::now() + timeout; + let socket = new_stream_socket_until(*address, deadline)?; + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| { + io::Error::new(io::ErrorKind::TimedOut, "TCP connection timed out") + })?; + socket.connect_timeout(&(*address).into(), remaining)?; Ok(socket.into()) } #[cfg(not(target_os = "macos"))] @@ -257,4 +295,17 @@ pub mod tcp { Some(socket2::Protocol::TCP), ) } + + #[cfg(target_os = "macos")] + fn new_stream_socket_until( + address: SocketAddr, + deadline: Instant, + ) -> io::Result { + let _guard = ProcessCreationGuard::acquire_until(deadline)?; + socket2::Socket::new( + socket2::Domain::for_address(address), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + ) + } } From d1d6cc331a9b1e8ba704c784fb2ac5dc7fc33e99 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:19:14 -0700 Subject: [PATCH 143/521] test(tui): expose resolver cleanup deadline gaps --- cmux-tui/Cargo.lock | 1 + cmux-tui/crates/cmux-tui-cdp/Cargo.toml | 3 + cmux-tui/crates/cmux-tui-cdp/src/client.rs | 128 ++++++++++++++++++++- 3 files changed, 131 insertions(+), 1 deletion(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 9ddbc898a1ce..4366f3a23625 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -232,6 +232,7 @@ dependencies = [ "anyhow", "base64", "cmux-tui-process", + "libc", "serde_json", "tungstenite", ] diff --git a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml index 6c2dabbcfdf7..6d70f794edb2 100644 --- a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml @@ -16,3 +16,6 @@ base64.workspace = true cmux-tui-process.workspace = true serde_json.workspace = true tungstenite.workspace = true + +[dev-dependencies] +libc.workspace = true diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index b666266d65a7..f4dfc2653f5a 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -44,6 +44,10 @@ static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); type TestHostResolverCommand = Arc Command + Send + Sync>; #[cfg(test)] static TEST_HOST_RESOLVER_COMMAND: Mutex> = Mutex::new(None); +#[cfg(test)] +static TEST_RESOLVER_REAPER_WAIT_ERROR: Mutex> = Mutex::new(None); +#[cfg(test)] +static TEST_RESOLVER_REAPER_WAIT_CALLS: AtomicUsize = AtomicUsize::new(0); static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -325,11 +329,24 @@ fn run_resolver_child_reaper(receiver: Receiver) { } pending.retain_mut(|request| { let _ = request.child.kill(); - !matches!(request.child.try_wait(), Ok(Some(_))) + !matches!(resolver_child_try_wait(request), Ok(Some(_))) }); } } +fn resolver_child_try_wait( + request: &mut ResolverChildReapRequest, +) -> std::io::Result> { + #[cfg(test)] + { + TEST_RESOLVER_REAPER_WAIT_CALLS.fetch_add(1, Ordering::AcqRel); + if let Some(kind) = *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() { + return Err(std::io::Error::from(kind)); + } + } + request.child.try_wait() +} + #[doc(hidden)] pub fn run_host_resolver_helper(args: &[String]) -> anyhow::Result<()> { let [host, port] = args else { @@ -2041,6 +2058,115 @@ mod tests { assert!(called.load(Ordering::Acquire), "host resolver was bypassed"); } + #[cfg(target_os = "macos")] + #[test] + fn resolver_spawn_obeys_the_absolute_deadline() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(resolver_fixture_command)); + let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (result_sender, result_receiver) = sync_channel(1); + let resolver = thread::spawn(move || { + let started = Instant::now(); + let result = resolve_host_addresses_until( + "cmux-success.invalid", + 9222, + started + Duration::from_millis(40), + ); + result_sender.send((started.elapsed(), result)).unwrap(); + }); + + let before_release = result_receiver.recv_timeout(Duration::from_millis(150)); + drop(process_barrier); + let completed_before_release = before_release.is_ok(); + let (elapsed, result) = before_release.unwrap_or_else(|_| { + result_receiver + .recv_timeout(Duration::from_secs(2)) + .expect("resolver did not finish after the process barrier was released") + }); + resolver.join().unwrap(); + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; + + assert!( + completed_before_release, + "resolver helper spawn waited past its absolute deadline" + ); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::TimedOut); + assert!( + elapsed < Duration::from_millis(150), + "resolver helper spawn exceeded its wall-clock bound: {elapsed:?}" + ); + } + + #[cfg(unix)] + #[test] + fn resolver_reaper_releases_capacity_after_child_ownership_is_lost() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + resolver_child_reaper().unwrap(); + let active = { + let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); + slot.as_ref().unwrap().active.clone() + }; + let baseline = active.load(Ordering::Acquire); + let lease = reserve_resolver_child_reaper().unwrap(); + assert_eq!(active.load(Ordering::Acquire), baseline + 1); + + let mut command = Command::new("/bin/sh"); + command.args(["-c", "exit 0"]); + let mut child = cmux_tui_process::spawn(&mut command).unwrap(); + let pid = libc::pid_t::try_from(child.id()).unwrap(); + let mut status = 0; + // SAFETY: pid identifies this live test child, status points to valid + // writable storage, and this call intentionally consumes wait ownership. + assert_eq!(unsafe { libc::waitpid(pid, &raw mut status, 0) }, pid); + assert_eq!(child.try_wait().unwrap_err().raw_os_error(), Some(libc::ECHILD)); + + handoff_resolver_child(child, lease); + let deadline = Instant::now() + Duration::from_millis(250); + while active.load(Ordering::Acquire) != baseline && Instant::now() < deadline { + thread::sleep(Duration::from_millis(5)); + } + + assert_eq!( + active.load(Ordering::Acquire), + baseline, + "terminal wait errors permanently retained resolver capacity" + ); + } + + #[test] + fn resolver_reaper_backs_off_retryable_wait_errors() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + resolver_child_reaper().unwrap(); + let active = { + let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); + slot.as_ref().unwrap().active.clone() + }; + let baseline = active.load(Ordering::Acquire); + let lease = reserve_resolver_child_reaper().unwrap(); + let mut command = Command::new("/bin/sh"); + command.args(["-c", "sleep 30"]); + let child = cmux_tui_process::spawn(&mut command).unwrap(); + + TEST_RESOLVER_REAPER_WAIT_CALLS.store(0, Ordering::Release); + *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = Some(std::io::ErrorKind::Interrupted); + handoff_resolver_child(child, lease); + let observation_deadline = Instant::now() + Duration::from_millis(250); + while TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire) < 12 + && Instant::now() < observation_deadline + { + thread::sleep(Duration::from_millis(2)); + } + let wait_calls = TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire); + *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = None; + + let cleanup_deadline = Instant::now() + Duration::from_secs(1); + while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { + thread::sleep(Duration::from_millis(5)); + } + assert_eq!(active.load(Ordering::Acquire), baseline); + assert!(wait_calls < 12, "retryable child wait errors were hot-polled {wait_calls} times"); + } + #[test] fn timed_out_host_lookups_do_not_strand_resolver_capacity() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); From 081c0648def6d72917c59c8386eb012881f44987 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:25:06 -0700 Subject: [PATCH 144/521] fix(tui): bound resolver spawn and reaping --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 92 ++++++++++++++++++--- cmux-tui/crates/cmux-tui-process/src/lib.rs | 9 ++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index f4dfc2653f5a..564a0e9aebf6 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -25,6 +25,8 @@ const CDP_RESOLVER_QUEUE_CAPACITY: usize = 64; const CDP_RESOLVER_WORKERS: usize = 4; const CDP_RESOLVER_CHILD_CAPACITY: usize = CDP_RESOLVER_QUEUE_CAPACITY + CDP_RESOLVER_WORKERS; const CDP_RESOLVER_POLL_INTERVAL: Duration = Duration::from_millis(5); +const CDP_RESOLVER_REAP_MAX_BACKOFF: Duration = Duration::from_millis(250); +const CDP_RESOLVER_REAP_RETRY_WINDOW: Duration = Duration::from_secs(2); const CDP_RESOLVER_OUTPUT_LIMIT: u64 = 64 * 1024; const CDP_DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); /// Maximum estimated retained bytes in each bounded CDP event queue. @@ -77,6 +79,7 @@ struct ResolveRequest { struct ResolverChildReaper { sender: Sender, active: Arc, + degraded: Arc, _worker: std::thread::JoinHandle<()>, } @@ -94,6 +97,9 @@ impl Drop for ResolverChildReaperLease { struct ResolverChildReapRequest { child: Child, _lease: ResolverChildReaperLease, + next_attempt: Instant, + retry_delay: Duration, + retry_deadline: Instant, } impl CdpResolver { @@ -212,7 +218,7 @@ fn resolve_host_addresses_until( let reaper = reserve_resolver_child_reaper()?; let mut command = host_resolver_command(host, port)?; command.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()); - let mut child = cmux_tui_process::spawn(&mut command)?; + let mut child = cmux_tui_process::spawn_until(&mut command, deadline)?; loop { match child.try_wait() { Ok(Some(status)) => { @@ -278,10 +284,12 @@ fn resolver_child_reaper() -> std::io::Result<()> { if slot.is_none() { let (sender, receiver) = channel(); let active = Arc::new(AtomicUsize::new(0)); + let degraded = Arc::new(AtomicBool::new(false)); + let worker_degraded = degraded.clone(); let worker = std::thread::Builder::new() .name("cmux-tui-cdp-resolver-reaper".into()) - .spawn(move || run_resolver_child_reaper(receiver))?; - *slot = Some(ResolverChildReaper { sender, active, _worker: worker }); + .spawn(move || run_resolver_child_reaper(receiver, worker_degraded))?; + *slot = Some(ResolverChildReaper { sender, active, degraded, _worker: worker }); } Ok(()) } @@ -290,6 +298,11 @@ fn reserve_resolver_child_reaper() -> std::io::Result resolver_child_reaper()?; let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); let reaper = slot.as_ref().expect("resolver child reaper initialized"); + if reaper.degraded.load(Ordering::Acquire) { + return Err(std::io::Error::other( + "host resolver child cleanup is temporarily unavailable", + )); + } reaper .active .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { @@ -301,13 +314,26 @@ fn reserve_resolver_child_reaper() -> std::io::Result "host resolver child capacity exhausted", ) })?; + if reaper.degraded.load(Ordering::Acquire) { + reaper.active.fetch_sub(1, Ordering::AcqRel); + return Err(std::io::Error::other( + "host resolver child cleanup is temporarily unavailable", + )); + } Ok(ResolverChildReaperLease { sender: reaper.sender.clone(), active: reaper.active.clone() }) } fn handoff_resolver_child(mut child: Child, lease: ResolverChildReaperLease) { let _ = child.kill(); let sender = lease.sender.clone(); - let request = ResolverChildReapRequest { child, _lease: lease }; + let now = Instant::now(); + let request = ResolverChildReapRequest { + child, + _lease: lease, + next_attempt: now, + retry_delay: CDP_RESOLVER_POLL_INTERVAL, + retry_deadline: now + CDP_RESOLVER_REAP_RETRY_WINDOW, + }; if let Err(error) = sender.send(request) { let mut request = error.0; let _ = request.child.kill(); @@ -315,22 +341,66 @@ fn handoff_resolver_child(mut child: Child, lease: ResolverChildReaperLease) { } } -fn run_resolver_child_reaper(receiver: Receiver) { +fn run_resolver_child_reaper( + receiver: Receiver, + degraded: Arc, +) { let mut pending = Vec::new(); + let mut disconnected = false; loop { - match receiver.recv_timeout(CDP_RESOLVER_POLL_INTERVAL) { - Ok(request) => pending.push(request), - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) if pending.is_empty() => return, - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {} + if pending.is_empty() { + if disconnected { + return; + } + match receiver.recv() { + Ok(request) => pending.push(request), + Err(_) => return, + } + } else { + let now = Instant::now(); + let wait = pending + .iter() + .map(|request| request.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or_default(); + if disconnected { + std::thread::sleep(wait); + } else { + match receiver.recv_timeout(wait) { + Ok(request) => pending.push(request), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + disconnected = true; + } + } + } } while let Ok(request) = receiver.try_recv() { pending.push(request); } + let now = Instant::now(); pending.retain_mut(|request| { + if request.next_attempt > now { + return true; + } let _ = request.child.kill(); - !matches!(resolver_child_try_wait(request), Ok(Some(_))) + match resolver_child_try_wait(request) { + Ok(Some(_)) => false, + Err(error) if error.kind() != std::io::ErrorKind::Interrupted => false, + Ok(None) | Err(_) => { + if now >= request.retry_deadline { + degraded.store(true, Ordering::Release); + } + request.next_attempt = now + request.retry_delay; + request.retry_delay = + request.retry_delay.saturating_mul(2).min(CDP_RESOLVER_REAP_MAX_BACKOFF); + true + } + } }); + if pending.is_empty() { + degraded.store(false, Ordering::Release); + } } } diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index ac8db77773f9..7424589efc5b 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -71,6 +71,15 @@ pub fn spawn(command: &mut Command) -> io::Result { command.spawn() } +/// Spawn a child only while the process barrier deadline remains valid. +pub fn spawn_until(command: &mut Command, deadline: std::time::Instant) -> io::Result { + let _guard = ProcessCreationGuard::acquire_until(deadline)?; + if std::time::Instant::now() >= deadline { + return Err(io::Error::new(io::ErrorKind::TimedOut, "process spawn deadline expired")); + } + command.spawn() +} + /// Unix socket operations whose descriptor creation is coordinated with /// process launch on macOS. #[cfg(unix)] From 25d0a7e2dd5587363bf95e612c91d799ffb87071 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:32:21 -0700 Subject: [PATCH 145/521] fix(tui): scope deadline imports by platform --- cmux-tui/crates/cmux-tui-process/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index 7424589efc5b..7bd043723681 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -215,7 +215,9 @@ pub mod unix { pub mod tcp { use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; - use std::time::{Duration, Instant}; + use std::time::Duration; + #[cfg(target_os = "macos")] + use std::time::Instant; use super::ProcessCreationGuard; From 5d9d60dcb462c744a5a13e710ede6e075e4e4f79 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 01:50:52 -0700 Subject: [PATCH 146/521] test(tui): exercise reconnect resolver boundary --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 52 ++++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/browser.rs | 8 +-- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 564a0e9aebf6..94ccbe12c300 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -2055,6 +2055,58 @@ mod tests { ); } + #[test] + fn reconnect_uses_the_cached_peer_without_resolving_again() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + warm_resolver(); + let listener = TcpListener::bind(("localhost", 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = thread::spawn(move || { + listener.set_nonblocking(true).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + let mut connections = 0; + while connections < 2 && Instant::now() < deadline { + match listener.accept() { + Ok((stream, _)) => { + stream.set_nonblocking(false).unwrap(); + let mut ws = accept(stream).unwrap(); + ws.close(None).unwrap(); + connections += 1; + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(1)); + } + Err(error) => panic!("reconnect test listener failed: {error}"), + } + } + connections + }); + let (initial_events, _initial_receiver) = sync_channel(1); + let client = CdpClient::connect( + &format!("ws://localhost:{port}/devtools/browser/fake"), + initial_events, + ) + .unwrap(); + let close_deadline = Instant::now() + Duration::from_secs(1); + while !client.inner.reader_stopped.load(Ordering::Acquire) { + assert!(Instant::now() < close_deadline, "initial CDP reader did not stop"); + thread::yield_now(); + } + + NEXT_RESOLVE_DELAY_MS.store(250, Ordering::Release); + let (reconnect_events, _reconnect_receiver) = sync_channel(1); + let reconnect = client.reconnect_with_timeout(reconnect_events, Duration::from_millis(100)); + let unused_delay = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); + + let reconnected = reconnect.is_ok(); + drop(reconnect); + drop(client); + let connections = server.join().unwrap(); + assert!(reconnected, "cached-peer reconnect did not complete"); + assert_eq!(connections, 2, "cached-peer reconnect did not reach the server"); + assert_eq!(unused_delay, 250, "reconnect invoked the hostname resolver"); + } + #[test] fn resolver_initialization_obeys_the_first_request_deadline() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 1e8f1d28a612..d838359145ac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -253,8 +253,6 @@ pub struct BrowserRuntime { stealth_user_agent: Option, routes: Mutex, closed: AtomicBool, - #[cfg(test)] - shutdown_reconnect_before_resolve: Mutex>>, } #[derive(Default)] @@ -434,8 +432,6 @@ impl BrowserRuntime { stealth_user_agent, routes: Mutex::new(Routes::default()), closed: AtomicBool::new(false), - #[cfg(test)] - shutdown_reconnect_before_resolve: Mutex::new(None), }); start_router(Arc::downgrade(&runtime), event_rx)?; runtime.client.set_discover_targets(true)?; @@ -3341,7 +3337,7 @@ mod tests { } #[test] - fn server_shutdown_reconnect_does_not_resolve_endpoint_after_deadline_starts() { + fn server_shutdown_reconnect_obeys_its_deadline() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let server = thread::spawn(move || { @@ -3364,8 +3360,6 @@ mod tests { thread::yield_now(); } assert!(runtime.is_closed(), "initial CDP transport did not close"); - *runtime.shutdown_reconnect_before_resolve.lock().unwrap() = - Some(Arc::new(|| thread::sleep(Duration::from_millis(250)))); let started = Instant::now(); let terminated = runtime.close_surface_for_shutdown( From bfb1f0dd9d45bd910fc3cdc99b88eeec3d5d816f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:24:02 -0700 Subject: [PATCH 147/521] test(tui): expose unbounded host child observation --- .../cmux-tui-core/src/process_session.rs | 84 +++++++++++++++++++ .../src/terminal_host_runtime.rs | 73 ++++++++++++++++ 2 files changed, 157 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 522195ceaa3f..9d64e94efad9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1474,6 +1474,90 @@ mod tests { assert!(reap_called.load(Ordering::Acquire)); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn natural_reaper_degrades_persistent_observation_errors() { + let release = Arc::new(AtomicBool::new(false)); + let observe_release = release.clone(); + let (attempt_sender, attempt_receiver) = mpsc::channel(); + let (finished_sender, finished_receiver) = mpsc::channel(); + let session = libc::pid_t::try_from(std::process::id()).unwrap(); + enqueue_reserved_session_leader( + reserve_child_reaper().unwrap(), + session, + Duration::ZERO, + move || { + let _ = attempt_sender.send(()); + if observe_release.load(Ordering::Acquire) { + Ok(true) + } else { + Err(io::Error::other("injected persistent observation error")) + } + }, + || false, + || true, + move |_| { + let _ = finished_sender.send(()); + true + }, + ); + + for _ in 0..NATURAL_REAP_MAX_ATTEMPTS { + attempt_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + } + let retried_while_degraded = + attempt_receiver.recv_timeout(Duration::from_millis(250)).is_ok(); + release.store(true, Ordering::Release); + wake_child_reaper(); + finished_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + + assert!( + !retried_while_degraded, + "persistent observation errors kept the shared child reaper on its hot retry cadence" + ); + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn natural_reaper_degrades_unresolved_finish_without_session_cleanup() { + let release = Arc::new(AtomicBool::new(false)); + let finish_release = release.clone(); + let (attempt_sender, attempt_receiver) = mpsc::channel(); + let (finished_sender, finished_receiver) = mpsc::channel(); + let session = libc::pid_t::try_from(std::process::id()).unwrap(); + enqueue_reserved_session_leader( + reserve_child_reaper().unwrap(), + session, + Duration::ZERO, + || Ok(true), + || false, + || true, + move |_| { + let _ = attempt_sender.send(()); + if finish_release.load(Ordering::Acquire) { + let _ = finished_sender.send(()); + true + } else { + false + } + }, + ); + + for _ in 0..NATURAL_REAP_MAX_ATTEMPTS { + attempt_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + } + let retried_while_degraded = + attempt_receiver.recv_timeout(Duration::from_millis(250)).is_ok(); + release.store(true, Ordering::Release); + wake_child_reaper(); + finished_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); + + assert!( + !retried_while_degraded, + "an unresolved ownership-loss finish kept the shared child reaper on its hot retry cadence" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn ignored_sigchld_fails_before_reserving_a_child_reaper() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 5796a24af89d..3f710b0527e7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -337,6 +337,8 @@ mod unix { static NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES: AtomicUsize = AtomicUsize::new(0); #[cfg(test)] static HOST_PROCESS_INLINE_WAITS: AtomicUsize = AtomicUsize::new(0); + #[cfg(test)] + static HOST_CHILD_OBSERVER_SPAWNS: AtomicUsize = AtomicUsize::new(0); struct HostProcessReaper { sender: Sender, @@ -3418,6 +3420,8 @@ mod unix { })?; let child_host = shared.clone(); thread::Builder::new().name("terminal-host-child".into()).spawn(move || { + #[cfg(test)] + HOST_CHILD_OBSERVER_SPAWNS.fetch_add(1, Ordering::AcqRel); let observation = loop { match crate::process_session::observe_child_without_reaping(session, false) { Ok(crate::process_session::ChildWaitState::Running) => continue, @@ -4630,6 +4634,75 @@ mod unix { assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); } + #[test] + fn hosted_child_observation_uses_the_reserved_shared_reaper() { + const CHILDREN: usize = 4; + + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + let worker_baseline = crate::process_session::dedicated_natural_reap_workers_for_test(); + let observer_baseline = HOST_CHILD_OBSERVER_SPAWNS.load(Ordering::Acquire); + let mut hosts = Vec::new(); + for index in 0..CHILDREN { + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: format!("/tmp/cmux-host-reap-bound-{index}.sock"), + record_path: format!("/tmp/cmux-host-reap-bound-{index}.json"), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec!["/bin/sh".into(), "-c".into(), "exit 0".into()], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + hosts.push(spawn_host_runtime(&launch, &bootstrapped).unwrap()); + } + + for host in &hosts { + let deadline = Instant::now() + Duration::from_secs(3); + while !host.child_reaped.load(Ordering::Acquire) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + assert!( + host.child_reaped.load(Ordering::Acquire), + "hosted PTY child was not reaped by the bounded shared worker" + ); + } + let dedicated_workers = + crate::process_session::dedicated_natural_reap_workers_for_test() + .saturating_sub(worker_baseline); + let dedicated_observers = HOST_CHILD_OBSERVER_SPAWNS + .load(Ordering::Acquire) + .saturating_sub(observer_baseline); + + for host in hosts { + host.request_termination(); + let _ = host.wait_for_child_exit(Duration::from_secs(1)); + } + + assert!( + dedicated_workers <= 1, + "hosted PTY observation retained {dedicated_workers} shared retry workers" + ); + assert_eq!( + dedicated_observers, 0, + "hosted PTY child observation retained one dedicated thread per child" + ); + } + #[test] fn hosted_child_exit_cleans_descendants_that_keep_the_slave_open() { let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); From 5bf3f4ac4ffcbe9a38eca913732a1f94c1c13aea Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:30:59 -0700 Subject: [PATCH 148/521] fix(tui): bound terminal host child reaping --- .../cmux-tui-core/src/process_session.rs | 153 +++++++++++++----- cmux-tui/crates/cmux-tui-core/src/surface.rs | 20 ++- .../src/terminal_host_runtime.rs | 152 +++++++++-------- 3 files changed, 201 insertions(+), 124 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 9d64e94efad9..07191a62e96a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -18,7 +18,10 @@ const SESSION_KILL_POLL: Duration = Duration::from_millis(5); const PROCESS_SESSION_PREFLIGHT_TIMEOUT: Duration = Duration::from_secs(1); const NATURAL_REAP_RETRY_INITIAL: Duration = Duration::from_millis(25); const NATURAL_REAP_RETRY_MAX: Duration = Duration::from_secs(1); +#[cfg(not(test))] const NATURAL_REAP_DEGRADED_RETRY: Duration = Duration::from_secs(30); +#[cfg(test)] +const NATURAL_REAP_DEGRADED_RETRY: Duration = Duration::from_millis(500); const NATURAL_REAP_BATCH_WINDOW: Duration = Duration::from_millis(5); const NATURAL_REAP_CAPACITY: usize = 4_096; #[cfg(not(test))] @@ -83,6 +86,19 @@ pub(crate) enum ChildWaitState { OwnershipLost, } +pub(crate) enum NaturalReapFinish { + Complete, + Pending, + Failed, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +enum NaturalChildObservation { + Ready, + Pending, + Failed, +} + enum NaturalReaperCommand { Add(NaturalReapRequest), Wake, @@ -95,7 +111,7 @@ struct NaturalReapRequest { child_observed: bool, needs_cleanup: Box bool + Send>, prepare_cleanup: Box bool + Send>, - finish: Box bool + Send>, + finish: Box NaturalReapFinish + Send>, _lease: ReservedChildReaperLease, next_attempt: Instant, retry_delay: Duration, @@ -103,6 +119,43 @@ struct NaturalReapRequest { degraded: bool, } +impl NaturalReapRequest { + fn clear_degraded(&mut self) { + if self.degraded { + self.degraded = false; + self._lease.degraded.fetch_sub(1, Ordering::AcqRel); + } + } + + fn recover_from_failure(&mut self) { + if self.attempts != 0 || self.degraded { + self.attempts = 0; + self.retry_delay = NATURAL_REAP_RETRY_INITIAL; + self.clear_degraded(); + } + } + + fn schedule_pending(&mut self) { + self.recover_from_failure(); + self.next_attempt = Instant::now() + self.retry_delay; + self.retry_delay = (self.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + } + + fn schedule_failure(&mut self) { + self.attempts = self.attempts.saturating_add(1); + if self.attempts >= NATURAL_REAP_MAX_ATTEMPTS { + if !self.degraded { + self.degraded = true; + self._lease.degraded.fetch_add(1, Ordering::AcqRel); + } + self.next_attempt = Instant::now() + NATURAL_REAP_DEGRADED_RETRY; + } else { + self.next_attempt = Instant::now() + self.retry_delay; + self.retry_delay = (self.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); + } + } +} + struct NaturalReaper { sender: mpsc::Sender, active: Arc, @@ -236,7 +289,7 @@ pub(crate) fn enqueue_reserved_session_leader( observe_child: impl FnMut() -> io::Result + Send + 'static, needs_cleanup: impl Fn() -> bool + Send + 'static, prepare_cleanup: impl FnMut() -> bool + Send + 'static, - finish: impl FnMut(bool) -> bool + Send + 'static, + finish: impl FnMut(bool) -> NaturalReapFinish + Send + 'static, ) { let sender = lease.sender.clone(); sender @@ -339,25 +392,41 @@ fn run_natural_reaper( continue; } - let child_observed = due + let observations = due .iter_mut() .map(|request| { - if !request.child_observed { - request.child_observed = (request.observe_child)().unwrap_or(false); + if request.child_observed { + return NaturalChildObservation::Ready; + } + match (request.observe_child)() { + Ok(true) => { + request.child_observed = true; + request.recover_from_failure(); + request.retry_delay = NATURAL_REAP_RETRY_INITIAL; + NaturalChildObservation::Ready + } + Ok(false) => { + request.recover_from_failure(); + NaturalChildObservation::Pending + } + Err(_) => NaturalChildObservation::Failed, } - request.child_observed }) .collect::>(); let needs_cleanup = due .iter() - .zip(&child_observed) - .map(|(request, observed)| *observed && (request.needs_cleanup)()) + .zip(&observations) + .map(|(request, observation)| { + *observation == NaturalChildObservation::Ready && (request.needs_cleanup)() + }) .collect::>(); let prepared = due .iter_mut() - .zip(child_observed.iter().zip(&needs_cleanup)) - .map(|(request, (observed, needed))| { - !observed || !needed || (request.prepare_cleanup)() + .zip(observations.iter().zip(&needs_cleanup)) + .map(|(request, (observation, needed))| { + *observation != NaturalChildObservation::Ready + || !needed + || (request.prepare_cleanup)() }) .collect::>(); let sessions = due @@ -388,37 +457,41 @@ fn run_natural_reaper( .unwrap_or_default() }; - for (((mut request, observed), needed), prepared) in - due.into_iter().zip(child_observed).zip(needs_cleanup).zip(prepared) + for (((mut request, observation), needed), prepared) in + due.into_iter().zip(observations).zip(needs_cleanup).zip(prepared) { - if !observed { - request.next_attempt = Instant::now() + request.retry_delay; - request.retry_delay = (request.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); - pending.push(request); - continue; + match observation { + NaturalChildObservation::Pending => { + request.schedule_pending(); + pending.push(request); + continue; + } + NaturalChildObservation::Failed => { + request.schedule_failure(); + pending.push(request); + continue; + } + NaturalChildObservation::Ready => {} } let cleanup_succeeded = needed && prepared && cleaned.contains(&request.session); - let done = (request.finish)(cleanup_succeeded); - if done { - if request.degraded { - request._lease.degraded.fetch_sub(1, Ordering::AcqRel); + let cleanup_failed = needed && !cleanup_succeeded; + match (request.finish)(cleanup_succeeded) { + NaturalReapFinish::Complete => { + request.clear_degraded(); } - continue; - } - if needed { - request.attempts = request.attempts.saturating_add(1); - } - if request.attempts >= NATURAL_REAP_MAX_ATTEMPTS { - if !request.degraded { - request.degraded = true; - request._lease.degraded.fetch_add(1, Ordering::AcqRel); + NaturalReapFinish::Failed => { + request.schedule_failure(); + pending.push(request); + } + NaturalReapFinish::Pending if cleanup_failed => { + request.schedule_failure(); + pending.push(request); + } + NaturalReapFinish::Pending => { + request.schedule_pending(); + pending.push(request); } - request.next_attempt = Instant::now() + NATURAL_REAP_DEGRADED_RETRY; - } else { - request.next_attempt = Instant::now() + request.retry_delay; - request.retry_delay = (request.retry_delay * 2).min(NATURAL_REAP_RETRY_MAX); } - pending.push(request); } } } @@ -509,7 +582,7 @@ fn accept_natural_reaper_command( NaturalReaperCommand::Wake => { wake_pending.store(false, Ordering::Release); let now = Instant::now(); - for request in pending { + for request in pending.iter_mut().filter(|request| !request.degraded) { request.next_attempt = now; } } @@ -1498,7 +1571,7 @@ mod tests { || true, move |_| { let _ = finished_sender.send(()); - true + NaturalReapFinish::Complete }, ); @@ -1536,9 +1609,9 @@ mod tests { let _ = attempt_sender.send(()); if finish_release.load(Ordering::Acquire) { let _ = finished_sender.send(()); - true + NaturalReapFinish::Complete } else { - false + NaturalReapFinish::Failed } }, ); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 3e2bb915cd69..92786f1b8dd7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1004,17 +1004,21 @@ impl LocalPtyProcess { move |cleanup_succeeded| { if reap_process.child_wait_ownership_lost.load(Ordering::Acquire) { let _signal = reap_process.child_signal_lock.lock().unwrap(); - if !reap_process.pty_drained.load(Ordering::Acquire) - || !matches!(child_identity.matches_current(), Ok(false)) - { - return false; + if !reap_process.pty_drained.load(Ordering::Acquire) { + return crate::process_session::NaturalReapFinish::Pending; + } + match child_identity.matches_current() { + Ok(false) => {} + Ok(true) | Err(_) => { + return crate::process_session::NaturalReapFinish::Failed; + } } drop(child.take()); reap_process.child_reaped.store(true, Ordering::Release); drop(_signal); *reap_process.exited.0.lock().unwrap() = true; reap_process.exited.1.notify_all(); - return true; + return crate::process_session::NaturalReapFinish::Complete; } let done = crate::process_session::poll_reserved_session_leader( crate::process_session::ReservedChildReap { @@ -1035,7 +1039,11 @@ impl LocalPtyProcess { *reap_process.exited.0.lock().unwrap() = true; reap_process.exited.1.notify_all(); } - done + if done { + crate::process_session::NaturalReapFinish::Complete + } else { + crate::process_session::NaturalReapFinish::Pending + } }, ); Ok(()) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 3f710b0527e7..769bad5b0456 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2757,9 +2757,9 @@ mod unix { signal: libc::c_int, deadline: Instant, ) -> std::io::Result<()> { - // The wait thread observes exit with WNOWAIT, then takes this lock - // before reaping. Holding it reserves the session leader and its - // numeric session id across enumeration and signaling. + // The shared reaper observes exit with WNOWAIT, then takes this + // lock before reaping. Holding it reserves the session leader and + // its numeric session id across enumeration and signaling. let _signal = self.child_signal_lock.lock().unwrap(); let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { return Err(std::io::Error::other("PTY child has no process id")); @@ -3418,86 +3418,82 @@ mod unix { crate::process_session::wake_child_reaper(); reader_host.publish_exit_if_drained(); })?; - let child_host = shared.clone(); - thread::Builder::new().name("terminal-host-child".into()).spawn(move || { - #[cfg(test)] - HOST_CHILD_OBSERVER_SPAWNS.fetch_add(1, Ordering::AcqRel); - let observation = loop { - match crate::process_session::observe_child_without_reaping(session, false) { - Ok(crate::process_session::ChildWaitState::Running) => continue, - observation => break observation, - } - }; - if matches!(observation, Ok(crate::process_session::ChildWaitState::Waitable)) { - child_host.child_waitable.store(true, Ordering::Release); - child_host.child_exit.1.notify_all(); - let cleanup_host = child_host.clone(); - let prepare_host = child_host.clone(); - let reap_host = child_host.clone(); - let mut child = Some(child); - crate::process_session::enqueue_reserved_session_leader( - reaper, - session, - HOST_KILL_WAIT, - || Ok(true), - move || { - crate::process_session::reserved_child_needs_cleanup( - crate::process_session::ReservedChildReap { - signal_lock: &cleanup_host.child_signal_lock, - pty_drained: &cleanup_host.pty_drained, - termination_started: &cleanup_host.termination_started, - cleanup_complete: &cleanup_host.group_escalation_complete, - child_reaped: &cleanup_host.child_reaped, - wait_ownership_lost: &cleanup_host.child_wait_ownership_lost, - }, - ) + let observe_host = shared.clone(); + let cleanup_host = shared.clone(); + let prepare_host = shared.clone(); + let reap_host = shared.clone(); + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, + HOST_KILL_WAIT, + move || { + let _signal = observe_host.child_signal_lock.lock().unwrap(); + let state = observe_host.observe_child_wait_state_locked(session, true)?; + Ok(state != crate::process_session::ChildWaitState::Running) + }, + move || { + crate::process_session::reserved_child_needs_cleanup( + crate::process_session::ReservedChildReap { + signal_lock: &cleanup_host.child_signal_lock, + pty_drained: &cleanup_host.pty_drained, + termination_started: &cleanup_host.termination_started, + cleanup_complete: &cleanup_host.group_escalation_complete, + child_reaped: &cleanup_host.child_reaped, + wait_ownership_lost: &cleanup_host.child_wait_ownership_lost, }, - move || prepare_host.prepare_natural_cleanup(), - move |cleanup_succeeded| { - let done = crate::process_session::poll_reserved_session_leader( - crate::process_session::ReservedChildReap { - signal_lock: &reap_host.child_signal_lock, - pty_drained: &reap_host.pty_drained, - termination_started: &reap_host.termination_started, - cleanup_complete: &reap_host.group_escalation_complete, - child_reaped: &reap_host.child_reaped, - wait_ownership_lost: &reap_host.child_wait_ownership_lost, - }, - cleanup_succeeded, - || { - let mut child = - child.take().expect("reserved child is reaped once"); - let _ = child.wait(); - }, - ); - if done { - let mut exited = reap_host.child_exit.0.lock().unwrap(); - *exited = true; - drop(exited); - reap_host.child_exit.1.notify_all(); - reap_host.publish_exit_if_drained(); + ) + }, + move || prepare_host.prepare_natural_cleanup(), + move |cleanup_succeeded| { + if reap_host.child_wait_ownership_lost.load(Ordering::Acquire) { + let _signal = reap_host.child_signal_lock.lock().unwrap(); + if !reap_host.pty_drained.load(Ordering::Acquire) { + return crate::process_session::NaturalReapFinish::Pending; + } + match child_identity.matches_current() { + Ok(false) => {} + Ok(true) | Err(_) => { + return crate::process_session::NaturalReapFinish::Failed; } - done + } + drop(child.take()); + reap_host.child_reaped.store(true, Ordering::Release); + drop(_signal); + let mut exited = reap_host.child_exit.0.lock().unwrap(); + *exited = true; + drop(exited); + reap_host.child_exit.1.notify_all(); + reap_host.publish_exit_if_drained(); + return crate::process_session::NaturalReapFinish::Complete; + } + let done = crate::process_session::poll_reserved_session_leader( + crate::process_session::ReservedChildReap { + signal_lock: &reap_host.child_signal_lock, + pty_drained: &reap_host.pty_drained, + termination_started: &reap_host.termination_started, + cleanup_complete: &reap_host.group_escalation_complete, + child_reaped: &reap_host.child_reaped, + wait_ownership_lost: &reap_host.child_wait_ownership_lost, + }, + cleanup_succeeded, + || { + let mut child = child.take().expect("reserved child is reaped once"); + let _ = child.wait(); }, ); - } else { - { - let _signal = child_host.child_signal_lock.lock().unwrap(); - child_host.child_wait_ownership_lost.store(true, Ordering::Release); - } - child_host.child_exit.1.notify_all(); - while !matches!(child_identity.matches_current(), Ok(false)) { - thread::sleep(Duration::from_millis(25)); + if done { + let mut exited = reap_host.child_exit.0.lock().unwrap(); + *exited = true; + drop(exited); + reap_host.child_exit.1.notify_all(); + reap_host.publish_exit_if_drained(); + crate::process_session::NaturalReapFinish::Complete + } else { + crate::process_session::NaturalReapFinish::Pending } - drop(child); - child_host.child_reaped.store(true, Ordering::Release); - let mut exited = child_host.child_exit.0.lock().unwrap(); - *exited = true; - child_host.child_exit.1.notify_all(); - drop(exited); - child_host.publish_exit_if_drained(); - } - })?; + }, + ); Ok(shared) } From f12dc6fb23a38f90710b955b6d4e489c10b96d4f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 03:01:03 -0700 Subject: [PATCH 149/521] test(tui): gate POSIX process fixtures --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 3 ++- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index c9f3cc0036ed..5d45296efe87 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -22,7 +22,7 @@ static FORCE_REAPER_PENDING: AtomicBool = AtomicBool::new(false); static FORCE_REAPER_WAIT_ERROR: AtomicBool = AtomicBool::new(false); #[cfg(test)] static REAPER_POLL_ATTEMPTS: AtomicUsize = AtomicUsize::new(0); -#[cfg(test)] +#[cfg(all(test, unix))] static REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); #[cfg(test)] @@ -499,6 +499,7 @@ mod tests { assert_eq!(args.last().map(String::as_str), Some("about:blank")); } + #[cfg(unix)] #[test] fn kill_until_confirms_the_owned_process_within_its_deadline() { let child = Command::new("sleep") diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 94ccbe12c300..6f017152f57a 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -2255,6 +2255,7 @@ mod tests { ); } + #[cfg(unix)] #[test] fn resolver_reaper_backs_off_retryable_wait_errors() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); From 45cace97631801918a8e57fd4983a3e69641fc04 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 03:11:34 -0700 Subject: [PATCH 150/521] test(tui): expose layout undo owner leak --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a6040a812934..64eb9ed6dfc9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12590,6 +12590,36 @@ mod tests { }); } + #[test] + fn layout_undo_retires_removed_surface_owner_before_reusing_capacity() { + let mux = test_mux(); + mux.set_shutdown_owner_capacity_for_test(2); + let first = mux.new_workspace(None, Some((80, 22))).unwrap(); + let first_pane = mux.with_state(|state| state.pane_of(first.id).unwrap()); + let added = mux.new_pane(first_pane, Some((80, 10))).unwrap(); + let added_pane = mux.with_state(|state| state.pane_of(added.id).unwrap()); + + let LayoutUndoResult::ConfirmationRequired { revision, .. } = + mux.undo_layout(added_pane, None, false).unwrap() + else { + panic!("new pane undo must require confirmation"); + }; + assert!(matches!( + mux.undo_layout(added_pane, Some(revision), true).unwrap(), + LayoutUndoResult::Undone { .. } + )); + + assert!(mux.surface(added.id).is_none()); + assert!( + mux.shutdown_owners.is_empty(), + "layout undo retained the removed surface owner after confirmed shutdown" + ); + let replacement = mux + .new_pane(first_pane, Some((80, 10))) + .expect("confirmed undo cleanup must release capacity for a replacement pane"); + assert!(mux.surface(replacement.id).is_some()); + } + #[test] fn layout_undo_confirmation_fences_exact_created_pane_tab_membership() { let mux = test_mux(); From 2fae084a02bec9749dc2b1a8fc29ea6b3a0840db Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 03:13:58 -0700 Subject: [PATCH 151/521] fix(tui): retire layout undo surfaces --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 64eb9ed6dfc9..d2aad36f32ac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9062,10 +9062,10 @@ impl Mux { }; drop(registry); - for surface in removed { + for surface in &removed { self.purge_surface_side_tables(surface.id); - surface.kill(); } + self.retire_surface_runtimes(removed); if terminal_count != 0 { let registry = self.workspace_registry.lock().unwrap(); self.emit_terminal_registry_changed(®istry, terminal_revision); From 5957581ce5305e0f537c4fbdc6cf556bf43bfc04 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 05:14:43 -0700 Subject: [PATCH 152/521] test(tui): expose incomplete build input tracking --- cmux-tui/crates/cmux-tui-core/build.rs | 28 +++++ .../tests/build_invalidation.rs | 104 ++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 cmux-tui/crates/ghostty-vt-sys/tests/build_invalidation.rs diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index 1009ed4c3a8b..8012c75b9f26 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -362,4 +362,32 @@ mod tests { assert_ne!(changed, clean, "Ghostty package changes retained the clean build identity"); } + + #[test] + fn cmux_identity_tracks_vendored_crossterm_sources() { + let repository = TestRepository::new("vendored-crossterm"); + fs::create_dir(repository.root.join("bindings")).unwrap(); + fs::create_dir(repository.root.join("bindings/rust")).unwrap(); + fs::create_dir(repository.root.join("crates")).unwrap(); + fs::create_dir(repository.root.join("vendor")).unwrap(); + fs::create_dir(repository.root.join("vendor/crossterm")).unwrap(); + fs::write(repository.root.join("Cargo.toml"), b"[workspace]").unwrap(); + fs::write(repository.root.join("Cargo.lock"), b"version = 4").unwrap(); + fs::write(repository.root.join("bindings/rust/input"), b"bindings").unwrap(); + fs::write(repository.root.join("crates/input"), b"crate").unwrap(); + fs::write(repository.root.join("vendor/crossterm/input"), b"one").unwrap(); + run_git(&repository.root, &["add", "."]); + run_git(&repository.root, &["commit", "-qm", "add cmux inputs"]); + let clean = + source_identity(&repository.root, &repository.root, CMUX_TUI_SOURCE_PATHS).unwrap(); + + fs::write(repository.root.join("vendor/crossterm/input"), b"two").unwrap(); + let changed = + source_identity(&repository.root, &repository.root, CMUX_TUI_SOURCE_PATHS).unwrap(); + + assert_ne!( + changed, clean, + "vendored Crossterm changes retained the clean cmux-tui build identity" + ); + } } diff --git a/cmux-tui/crates/ghostty-vt-sys/tests/build_invalidation.rs b/cmux-tui/crates/ghostty-vt-sys/tests/build_invalidation.rs new file mode 100644 index 000000000000..9539dc0d99f3 --- /dev/null +++ b/cmux-tui/crates/ghostty-vt-sys/tests/build_invalidation.rs @@ -0,0 +1,104 @@ +#![cfg(unix)] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct TestTree { + root: PathBuf, +} + +impl TestTree { + fn new() -> Self { + let nonce = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let root = std::env::temp_dir() + .join(format!("cmux-ghostty-vt-invalidation-{}-{nonce}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + Self { root } + } +} + +impl Drop for TestTree { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } +} + +fn run_build(workspace: &Path, ghostty: &Path, target: &Path, zig: &Path, log: &Path) { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let output = Command::new(cargo) + .current_dir(workspace) + .args(["build", "--quiet", "-p", "ghostty-vt-sys"]) + .env("CMUX_GHOSTTY_SRC", ghostty) + .env("CARGO_TARGET_DIR", target) + .env("ZIG", zig) + .env("FAKE_ZIG_LOG", log) + .output() + .unwrap(); + assert!( + output.status.success(), + "nested ghostty-vt-sys build failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn native_archive_rebuilds_when_a_ghostty_package_source_changes() { + let tree = TestTree::new(); + let ghostty = tree.root.join("ghostty"); + fs::create_dir_all(ghostty.join("include/ghostty")).unwrap(); + fs::create_dir(ghostty.join("src")).unwrap(); + fs::create_dir_all(ghostty.join("pkg/simdutf")).unwrap(); + fs::write(ghostty.join("build.zig"), b"// fake build").unwrap(); + fs::write(ghostty.join("build.zig.zon"), b".{}").unwrap(); + fs::write(ghostty.join("include/ghostty/vt.h"), b"typedef unsigned short GhosttyMode;\n") + .unwrap(); + fs::write(ghostty.join("src/main.zig"), b"// fake source").unwrap(); + let package_input = ghostty.join("pkg/simdutf/input"); + fs::write(&package_input, b"one").unwrap(); + + let zig = tree.root.join("fake-zig"); + fs::write( + &zig, + br#"#!/bin/sh +set -eu +printf 'run\n' >> "$FAKE_ZIG_LOG" +prefix= +while [ "$#" -gt 0 ]; do + if [ "$1" = "--prefix" ]; then + shift + prefix=$1 + fi + shift +done +test -n "$prefix" +mkdir -p "$prefix/lib" +printf 'void cmux_ghostty_vt_test_archive(void) {}\n' > "$prefix/empty.c" +/usr/bin/cc -c "$prefix/empty.c" -o "$prefix/empty.o" +/usr/bin/ar crs "$prefix/lib/libghostty-vt.a" "$prefix/empty.o" +"#, + ) + .unwrap(); + let mut permissions = fs::metadata(&zig).unwrap().permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&zig, permissions).unwrap(); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let workspace = manifest_dir.ancestors().nth(2).unwrap(); + let target = tree.root.join("target"); + let log = tree.root.join("zig.log"); + run_build(workspace, &ghostty, &target, &zig, &log); + assert_eq!(fs::read_to_string(&log).unwrap().lines().count(), 1); + + fs::write(package_input, b"two").unwrap(); + run_build(workspace, &ghostty, &target, &zig, &log); + + assert_eq!( + fs::read_to_string(&log).unwrap().lines().count(), + 2, + "Ghostty package changes did not rerun the native build script" + ); +} From b35460d003d124550efc8eb1a6b710de37102383 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 05:26:25 -0700 Subject: [PATCH 153/521] fix(tui): track complete build input scopes --- cmux-tui/crates/cmux-tui-core/build.rs | 17 +++++++++++++++-- cmux-tui/crates/ghostty-vt-sys/build.rs | 10 ++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/build.rs b/cmux-tui/crates/cmux-tui-core/build.rs index 8012c75b9f26..3120bc34365b 100644 --- a/cmux-tui/crates/cmux-tui-core/build.rs +++ b/cmux-tui/crates/cmux-tui-core/build.rs @@ -8,8 +8,10 @@ use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; const FNV_OFFSET: u64 = 0xcbf29ce484222325; const FNV_PRIME: u64 = 0x100000001b3; -const CMUX_TUI_SOURCE_PATHS: &[&str] = &["Cargo.toml", "Cargo.lock", "bindings/rust", "crates"]; +const CMUX_TUI_SOURCE_PATHS: &[&str] = + &["Cargo.toml", "Cargo.lock", "bindings/rust", "crates", "vendor/crossterm"]; const GHOSTTY_SOURCE_PATHS: &[&str] = &["build.zig", "build.zig.zon", "include", "pkg", "src"]; +const GHOSTTY_OPTIONAL_SOURCE_PATHS: &[&str] = &["VERSION"]; fn main() { for name in [ @@ -34,6 +36,7 @@ fn main() { track_source_paths(workspace_root, CMUX_TUI_SOURCE_PATHS); track_source_paths(&ghostty_root, GHOSTTY_SOURCE_PATHS); + track_source_paths(&ghostty_root, GHOSTTY_OPTIONAL_SOURCE_PATHS); track_git_identity(repository_root); track_git_identity(&ghostty_root); @@ -43,7 +46,17 @@ fn main() { println!("cargo:rustc-env=CMUX_TUI_SOURCE_COMMIT={identity}"); } if !nonempty_env("CMUX_TUI_GHOSTTY_COMMIT") { - let identity = source_identity(&ghostty_root, &ghostty_root, GHOSTTY_SOURCE_PATHS) + let ghostty_identity_paths = GHOSTTY_SOURCE_PATHS + .iter() + .copied() + .chain( + GHOSTTY_OPTIONAL_SOURCE_PATHS + .iter() + .copied() + .filter(|path| ghostty_root.join(path).exists()), + ) + .collect::>(); + let identity = source_identity(&ghostty_root, &ghostty_root, &ghostty_identity_paths) .expect("derive an exact Ghostty source identity"); println!("cargo:rustc-env=CMUX_TUI_SOURCE_GHOSTTY_COMMIT={identity}"); } diff --git a/cmux-tui/crates/ghostty-vt-sys/build.rs b/cmux-tui/crates/ghostty-vt-sys/build.rs index 0a9b76795f57..ea7717209007 100644 --- a/cmux-tui/crates/ghostty-vt-sys/build.rs +++ b/cmux-tui/crates/ghostty-vt-sys/build.rs @@ -2,6 +2,9 @@ use std::env; use std::path::PathBuf; use std::process::Command; +const GHOSTTY_VT_SOURCE_PATHS: &[&str] = + &["VERSION", "build.zig", "build.zig.zon", "include", "pkg", "src"]; + fn main() { let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); @@ -25,10 +28,9 @@ fn main() { println!("cargo:rerun-if-env-changed=CMUX_GHOSTTY_SRC"); println!("cargo:rerun-if-env-changed=ZIG"); println!("cargo:rerun-if-env-changed=CMUX_GHOSTTY_VT_ZIG_CPU"); - println!("cargo:rerun-if-changed={}", ghostty_dir.join("include").display()); - println!("cargo:rerun-if-changed={}", ghostty_dir.join("build.zig").display()); - println!("cargo:rerun-if-changed={}", ghostty_dir.join("build.zig.zon").display()); - println!("cargo:rerun-if-changed={}", ghostty_dir.join("src").display()); + for path in GHOSTTY_VT_SOURCE_PATHS { + println!("cargo:rerun-if-changed={}", ghostty_dir.join(path).display()); + } // Build libghostty-vt.a with zig. ReleaseFast regardless of the cargo // profile: the VT parser is on the PTY hot path and a debug zig build From f842c1268664e1c0363572456dc146b4d32a3275 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:28:05 -0700 Subject: [PATCH 154/521] test(tui): expose lost startup deadline --- cmux-tui/crates/cmux-tui/tests/cli.rs | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index ca0b815305d8..d221479f3181 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -2,6 +2,8 @@ use std::fs; #[cfg(unix)] use std::fs::File; use std::io::{BufRead, BufReader, Read, Write}; +#[cfg(target_os = "linux")] +use std::os::fd::AsRawFd; #[cfg(unix)] use std::os::fd::FromRawFd; #[cfg(unix)] @@ -1070,6 +1072,58 @@ fn plain_launch_attaches_to_existing_local_session() { panic!("plain launch never attached as a TUI client"); } +#[cfg(target_os = "linux")] +#[test] +fn plain_launch_preserves_the_existing_session_connect_deadline() { + let dir = unique_temp_dir("saturated-existing-session"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + // SAFETY: listener owns a valid listening Unix socket. Reapplying listen + // only narrows its pending connection queue for this isolated fixture. + assert_eq!(unsafe { libc::listen(listener.as_raw_fd(), 0) }, 0); + + let mut queued = Vec::new(); + for _ in 0..32 { + match transport::connect_until(&socket, Instant::now() + Duration::from_millis(75)) { + Ok(stream) => queued.push(stream), + Err(error) => { + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + break; + } + } + } + assert!(queued.len() < 32, "could not saturate the existing session listener"); + + let mut launch = Command::new(bin()) + .args(["--ephemeral", "--socket"]) + .arg(&socket) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(3); + let status = loop { + if let Some(status) = launch.try_wait().unwrap() { + break status; + } + if Instant::now() >= deadline { + let _ = launch.kill(); + let _ = launch.wait(); + panic!("plain launch exceeded the existing-session connection deadline"); + } + std::thread::sleep(Duration::from_millis(10)); + }; + + assert!(!status.success(), "plain launch replaced a live saturated session"); + assert!(socket.exists(), "plain launch unlinked the live saturated session socket"); + drop(queued); + drop(listener); + fs::remove_file(socket).unwrap(); + fs::remove_dir_all(dir).unwrap(); +} + #[test] fn plain_launch_reports_an_incompatible_server_without_reconnecting() { let dir = unique_temp_dir("incompatible-server"); From eec23d2b18e2568181ffe4f0301fcea38c63bf0e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:30:38 -0700 Subject: [PATCH 155/521] fix(tui): preserve startup connect deadlines --- cmux-tui/crates/cmux-tui-core/src/server.rs | 33 +++++++++++++++++++-- cmux-tui/crates/cmux-tui/src/main.rs | 8 ++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 8e7ea54e963a..0a09e8e266ca 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -68,6 +68,8 @@ pub const SERVER_SHUTDOWN_INCOMPLETE_ERROR: &str = "shutdown_cleanup_incomplete" /// Maximum wall-clock time the server spends draining a shutdown request. /// Clients add their own transport margin to this server-owned bound. pub const SERVER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +/// Maximum time startup spends determining whether a local session socket is live. +pub const LOCAL_SOCKET_CONNECT_TIMEOUT: Duration = Duration::from_secs(1); pub const VIEWPORT_SPLITS_CAPABILITY: &str = "viewport-splits-v1"; pub const VIEWPORT_COLUMN_RESIZE_CAPABILITY: &str = "viewport-column-resize-v1"; pub const LAYOUT_UNDO_CAPABILITY: &str = "layout-undo-v1"; @@ -2623,6 +2625,26 @@ fn clamp_client_label(value: String) -> String { sanitize_window_title(&value).chars().take(64).collect() } +/// Connect to a live local session, or return `None` only when its socket is +/// absent or definitively refusing connections. +pub fn connect_existing_until( + path: &Path, + deadline: Instant, +) -> std::io::Result>> { + match transport::connect_until(path, deadline) { + Ok(stream) => Ok(Some(stream)), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound + ) => + { + Ok(None) + } + Err(error) => Err(error), + } +} + /// Bind the socket and serve connections on background threads. pub fn serve(mux: Arc, path: Option) -> anyhow::Result { let path = path.unwrap_or_else(|| default_socket_path(&mux.session)); @@ -2632,12 +2654,17 @@ pub fn serve(mux: Arc, path: Option) -> anyhow::Result { } // Refuse to clobber a live socket; remove a stale one. if path.exists() { - match transport::connect(&path) { - Ok(_) => anyhow::bail!( + let deadline = Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT; + match connect_existing_until(&path, deadline)? { + Some(_) => anyhow::bail!( "session socket {} is already in use (another instance running?)", path.display() ), - Err(_) => std::fs::remove_file(&path)?, + None => match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + }, } } let listener = transport::listen(&path)?; diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index cf4728a9ed4a..bdc0d1e926b5 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -67,7 +67,6 @@ const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORIT const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1); const SERVER_SHUTDOWN_RETRY_INITIAL: std::time::Duration = std::time::Duration::from_millis(100); const SERVER_SHUTDOWN_RETRY_MAX: std::time::Duration = std::time::Duration::from_secs(5); -const EXISTING_SESSION_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -1281,10 +1280,9 @@ fn run_server( fn connect_existing_local_session( socket_path: &std::path::Path, ) -> anyhow::Result>> { - let deadline = std::time::Instant::now() + EXISTING_SESSION_CONNECT_TIMEOUT; - let stream = match cmux_tui_core::platform::transport::connect_until(socket_path, deadline) { - Ok(stream) => stream, - Err(_) => return Ok(None), + let deadline = std::time::Instant::now() + cmux_tui_core::server::LOCAL_SOCKET_CONNECT_TIMEOUT; + let Some(stream) = cmux_tui_core::server::connect_existing_until(socket_path, deadline)? else { + return Ok(None); }; RemoteSession::connect_local_stream(stream, socket_path).map(Some) } From 9875830133bbaafcf38891325eaf3e9f3954965e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:12:37 -0700 Subject: [PATCH 156/521] test(tui): expose shutdown owner lock cycle --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 41 +++++++++++++ .../src/terminal_host_runtime.rs | 58 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 92786f1b8dd7..e2994676ac41 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4147,6 +4147,47 @@ mod tests { assert_eq!(signals.load(Ordering::Relaxed), 0); } + #[cfg(unix)] + #[test] + fn local_session_kill_observes_the_waitable_leader_under_its_owner_lock() { + let mux = Mux::new_for_test("local-owner-observation", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + "trap '' HUP TERM; while :; do sleep 60; done".into(), + ]), + ..SurfaceOptions::default() + }; + let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); + let process = match surface.as_pty().unwrap().local_process.as_deref().unwrap() { + LocalProcess::Owned(process) => process.clone(), + LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), + }; + process.termination_started.store(true, Ordering::Release); + + let started = Instant::now(); + let killed = process + .kill_terminal_process_session_until(Instant::now() + Duration::from_secs(1), None) + .unwrap(); + let elapsed = started.elapsed(); + + process.group_escalation_complete.store(true, Ordering::Release); + process.exited.1.notify_all(); + crate::process_session::wake_child_reaper(); + let reap_deadline = Instant::now() + Duration::from_secs(2); + while !process.child_reaped.load(Ordering::Acquire) && Instant::now() < reap_deadline { + std::thread::sleep(Duration::from_millis(10)); + } + + assert!(killed, "session cleanup could not observe its SIGKILLed leader"); + assert!( + elapsed < Duration::from_millis(500), + "session cleanup waited on the reaper while retaining its owner lock: {elapsed:?}" + ); + assert!(process.child_reaped.load(Ordering::Acquire)); + } + #[cfg(unix)] #[test] fn completed_natural_cleanup_does_not_revalidate_reusable_session_id() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 769bad5b0456..effc2fa60970 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4630,6 +4630,64 @@ mod unix { assert!(reaped, "the hosted PTY leader remained unreaped after cleanup could succeed"); } + #[test] + fn hosted_session_kill_observes_the_waitable_leader_under_its_owner_lock() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + let root = std::env::temp_dir() + .join(format!("cmux-host-owner-{}", crate::workspace_registry::new_uuid_v4())); + fs::create_dir_all(&root).unwrap(); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: root.join("host.sock").to_string_lossy().into_owned(), + record_path: root.join("host.json").to_string_lossy().into_owned(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec![ + "/bin/sh".into(), + "-c".into(), + "trap '' HUP TERM; while :; do sleep 60; done".into(), + ], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + let host = spawn_host_runtime(&launch, &bootstrapped).unwrap(); + host.termination_started.store(true, Ordering::Release); + + let started = Instant::now(); + let killed = host + .kill_terminal_process_session_until(Instant::now() + Duration::from_secs(1)) + .unwrap(); + let elapsed = started.elapsed(); + + host.group_escalation_complete.store(true, Ordering::Release); + host.child_exit.1.notify_all(); + crate::process_session::wake_child_reaper(); + let _ = host.wait_for_child_exit(Duration::from_secs(2)); + let _ = fs::remove_dir_all(root); + + assert!(killed, "host cleanup could not observe its SIGKILLed leader"); + assert!( + elapsed < Duration::from_millis(500), + "host cleanup waited on the reaper while retaining its owner lock: {elapsed:?}" + ); + assert!(host.child_reaped.load(Ordering::Acquire)); + } + #[test] fn hosted_child_observation_uses_the_reserved_shared_reaper() { const CHILDREN: usize = 4; From 9bbf4fd29b5453374f757ae2698745786428666c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:14:42 -0700 Subject: [PATCH 157/521] fix(tui): reconcile killed children under ownership --- .../crates/cmux-tui-core/src/process_session.rs | 10 +++++----- cmux-tui/crates/cmux-tui-core/src/surface.rs | 16 ++++++++++++++-- .../cmux-tui-core/src/terminal_host_runtime.rs | 11 +++++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 07191a62e96a..903366b8fe91 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -769,7 +769,7 @@ pub(crate) fn kill_until_only_leader( session: libc::pid_t, leader: libc::pid_t, deadline: Instant, - leader_exited: impl Fn() -> bool, + leader_exited: impl FnMut() -> io::Result, ) -> io::Result { validate_session_target(session)?; let snapshot = SessionProcessSnapshot::capture([session], deadline)?; @@ -781,7 +781,7 @@ pub(crate) fn kill_until_only_leader_from_snapshot( session: libc::pid_t, leader: libc::pid_t, deadline: Instant, - leader_exited: impl Fn() -> bool, + leader_exited: impl FnMut() -> io::Result, ) -> io::Result { kill_until_only_reserved_from_snapshot(snapshot, session, leader, deadline, leader_exited, None) } @@ -804,7 +804,7 @@ pub fn kill_until_only_reserved( session, reserved, deadline, - || true, + || Ok(true), Some(reserved), ) } @@ -814,7 +814,7 @@ fn kill_until_only_reserved_from_snapshot( session: libc::pid_t, reserved: libc::pid_t, deadline: Instant, - can_reconcile: impl Fn() -> bool, + mut can_reconcile: impl FnMut() -> io::Result, signal_exclusion: Option, ) -> io::Result { validate_session_target(session)?; @@ -829,7 +829,7 @@ fn kill_until_only_reserved_from_snapshot( .collect::>>()? .into_iter() .all(|alive| !alive); - if can_reconcile() && known_drained { + if known_drained && can_reconcile()? { let (current_generation, current) = snapshot.refresh_members(session, generation, deadline)?; if current.iter().all(|pid| *pid == reserved) { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index e2994676ac41..5a92b861b5df 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1232,11 +1232,23 @@ impl LocalPtyProcess { leader, leader, deadline, - || self.child_waitable.load(Ordering::Acquire), + || match self.observe_child_wait_state_locked(leader)? { + crate::process_session::ChildWaitState::Running => Ok(false), + crate::process_session::ChildWaitState::Waitable => Ok(true), + crate::process_session::ChildWaitState::OwnershipLost => { + Err(std::io::Error::other("PTY child wait ownership was lost")) + } + }, ), None => { crate::process_session::kill_until_only_leader(leader, leader, deadline, || { - self.child_waitable.load(Ordering::Acquire) + match self.observe_child_wait_state_locked(leader)? { + crate::process_session::ChildWaitState::Running => Ok(false), + crate::process_session::ChildWaitState::Waitable => Ok(true), + crate::process_session::ChildWaitState::OwnershipLost => { + Err(std::io::Error::other("PTY child wait ownership was lost")) + } + } }) } } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index effc2fa60970..7b806c0231d9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2800,8 +2800,15 @@ mod unix { { return Err(std::io::Error::other("PTY child wait ownership was lost")); } - crate::process_session::kill_until_only_leader(leader, leader, deadline, || { - self.child_waitable.load(Ordering::Acquire) + crate::process_session::kill_until_only_leader(leader, leader, deadline, || match self + .observe_child_wait_state_locked( + leader, true, + )? { + crate::process_session::ChildWaitState::Running => Ok(false), + crate::process_session::ChildWaitState::Waitable => Ok(true), + crate::process_session::ChildWaitState::OwnershipLost => { + Err(std::io::Error::other("PTY child wait ownership was lost")) + } }) } From e162ee62a912257e505cffcdf8f41d4ee2096080 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:16:36 -0700 Subject: [PATCH 158/521] test(tui): expose resolver output backpressure --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 46 +++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 6f017152f57a..3dc207ad19a4 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -1747,7 +1747,13 @@ mod tests { ]) .env( "CMUX_TUI_TEST_HOST_RESOLVER_MODE", - if host.starts_with("cmux-blocked-") { "blocked" } else { "success" }, + if host.starts_with("cmux-blocked-") { + "blocked" + } else if host.starts_with("cmux-oversized-") { + "oversized" + } else { + "success" + }, ) .env("CMUX_TUI_TEST_HOST_RESOLVER_PORT", port.to_string()); command @@ -1760,6 +1766,11 @@ mod tests { thread::sleep(Duration::from_secs(30)); return; } + if std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_MODE").as_deref() == Ok("oversized") { + let output = vec![b'x'; usize::try_from(CDP_RESOLVER_OUTPUT_LIMIT).unwrap() + 1]; + std::io::stdout().write_all(&output).unwrap(); + return; + } let port = std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_PORT").unwrap(); println!("127.0.0.1:{port}"); } @@ -2180,6 +2191,39 @@ mod tests { assert!(called.load(Ordering::Acquire), "host resolver was bypassed"); } + #[test] + fn resolver_drains_output_before_waiting_for_helper_exit() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + resolver_child_reaper().unwrap(); + let active = { + let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); + slot.as_ref().unwrap().active.clone() + }; + let baseline = active.load(Ordering::Acquire); + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(resolver_fixture_command)); + + let started = Instant::now(); + let result = resolve_host_addresses_until( + "cmux-oversized-output.invalid", + 9222, + started + Duration::from_secs(1), + ); + let elapsed = started.elapsed(); + + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; + let cleanup_deadline = Instant::now() + Duration::from_secs(1); + while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { + thread::sleep(Duration::from_millis(5)); + } + + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + assert!( + elapsed < Duration::from_millis(500), + "resolver helper blocked on its undrained output pipe: {elapsed:?}" + ); + assert_eq!(active.load(Ordering::Acquire), baseline); + } + #[cfg(target_os = "macos")] #[test] fn resolver_spawn_obeys_the_absolute_deadline() { From 95c3f5960da1f5e02f81fc5fdce13ad0a1eea2ac Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:17:47 -0700 Subject: [PATCH 159/521] fix(tui): drain resolver output concurrently --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 54 ++++++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 3dc207ad19a4..8c51dfcbf402 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -219,6 +219,37 @@ fn resolve_host_addresses_until( let mut command = host_resolver_command(host, port)?; command.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()); let mut child = cmux_tui_process::spawn_until(&mut command, deadline)?; + let mut stdout = child + .stdout + .take() + .ok_or_else(|| std::io::Error::other("host resolver helper stdout was unavailable"))?; + let (output_sender, output_receiver) = sync_channel(1); + // Resolver workers bound the number of concurrent helpers and readers. + // Timeout cleanup kills the owned helper, which closes this pipe even + // when the helper was blocked while producing output. + let output_reader = + std::thread::Builder::new().name("cmux-tui-cdp-resolver-output".into()).spawn(move || { + let mut output = Vec::new(); + let mut buffer = [0_u8; 8 * 1024]; + let limit = usize::try_from(CDP_RESOLVER_OUTPUT_LIMIT).unwrap_or(usize::MAX); + let result = loop { + match stdout.read(&mut buffer) { + Ok(0) => break Ok(output), + Ok(count) => { + let retained = + limit.saturating_add(1).saturating_sub(output.len()).min(count); + output.extend_from_slice(&buffer[..retained]); + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + Err(error) => break Err(error), + } + }; + let _ = output_sender.send(result); + }); + if let Err(error) = output_reader { + handoff_resolver_child(child, reaper); + return Err(error); + } loop { match child.try_wait() { Ok(Some(status)) => { @@ -228,11 +259,24 @@ fn resolve_host_addresses_until( "host resolver helper exited with {status}" ))); } - let stdout = child.stdout.take().ok_or_else(|| { - std::io::Error::other("host resolver helper stdout was unavailable") - })?; - let mut output = Vec::new(); - stdout.take(CDP_RESOLVER_OUTPUT_LIMIT + 1).read_to_end(&mut output)?; + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )); + }; + let output = match output_receiver.recv_timeout(remaining) { + Ok(output) => output?, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )); + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + return Err(std::io::Error::other("host resolver output reader stopped")); + } + }; if output.len() as u64 > CDP_RESOLVER_OUTPUT_LIMIT { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, From f4f857bb08f1e57298bf137a9d9a9ee32e6035d9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:20:12 -0700 Subject: [PATCH 160/521] test(tui): expose sticky preflight failures --- .../cmux-tui-core/src/process_session.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 903366b8fe91..a196d61496d5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1878,6 +1878,31 @@ mod tests { assert_eq!(error.kind(), io::ErrorKind::Unsupported); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn successful_preflight_survives_a_transient_initializer_failure() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_TRANSIENT_PREFLIGHT_CACHE"; + const TEST_NAME: &str = + "process_session::tests::successful_preflight_survives_a_transient_initializer_failure"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "preflight cache subprocess failed: {status}"); + return; + } + + require_stable_process_signaling().unwrap(); + set_process_session_preflight_failure_for_test(true); + initialize_stable_process_signaling(); + set_process_session_preflight_failure_for_test(false); + + require_cached_stable_process_signaling() + .expect("a later transient scan replaced the successful startup preflight"); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn shutdown_batch_reuses_one_process_table_snapshot() { From b273c5da75290301f752f427f5970a52e45b2493 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 16:21:52 -0700 Subject: [PATCH 161/521] fix(tui): cache successful process preflights --- .../cmux-tui-core/src/process_session.rs | 41 ++++++------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index a196d61496d5..3ddd1570bb52 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -33,23 +33,7 @@ const NATURAL_REAP_MAX_ATTEMPTS: usize = 3; static NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); static NATURAL_REAPER: OnceLock>> = OnceLock::new(); -static STABLE_PROCESS_SIGNALING: OnceLock> = - OnceLock::new(); - -struct StableProcessSignalingFailure { - kind: io::ErrorKind, - message: String, -} - -impl StableProcessSignalingFailure { - fn from_error(error: io::Error) -> Self { - Self { kind: error.kind(), message: error.to_string() } - } - - fn to_error(&self) -> io::Error { - io::Error::new(self.kind, self.message.clone()) - } -} +static STABLE_PROCESS_SIGNALING: OnceLock<()> = OnceLock::new(); #[cfg(test)] thread_local! { @@ -1246,24 +1230,23 @@ impl StableProcessHandle { /// Verify that this runtime can enumerate PTY session members and signal exact /// process instances before any PTY is spawned or shutdown topology mutates. pub fn require_stable_process_signaling() -> io::Result<()> { - require_stable_process_signaling_until(Instant::now() + PROCESS_SESSION_PREFLIGHT_TIMEOUT) + if STABLE_PROCESS_SIGNALING.get().is_some() { + return Ok(()); + } + require_stable_process_signaling_until(Instant::now() + PROCESS_SESSION_PREFLIGHT_TIMEOUT)?; + let _ = STABLE_PROCESS_SIGNALING.set(()); + Ok(()) } -/// Cache the process-level capability once per server generation. Startup, -/// identity responses, and PTY creation share this result; shutdown still -/// performs a fresh bounded preflight immediately before mutating topology. +/// Cache a successful process-level capability check once per server +/// generation. Transient failures remain retryable; shutdown still performs a +/// fresh bounded preflight immediately before mutating topology. pub(crate) fn initialize_stable_process_signaling() { - let _ = stable_process_signaling(); + let _ = require_stable_process_signaling(); } pub(crate) fn require_cached_stable_process_signaling() -> io::Result<()> { - stable_process_signaling().as_ref().map_err(StableProcessSignalingFailure::to_error).copied() -} - -fn stable_process_signaling() -> &'static Result<(), StableProcessSignalingFailure> { - STABLE_PROCESS_SIGNALING.get_or_init(|| { - require_stable_process_signaling().map_err(StableProcessSignalingFailure::from_error) - }) + require_stable_process_signaling() } pub(crate) fn require_stable_process_signaling_until(deadline: Instant) -> io::Result<()> { From e388916add409eff95029387043007ef147171e4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:57:55 -0700 Subject: [PATCH 162/521] test(tui): expose sidebar owner loss --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d2aad36f32ac..e96ef4ff2ece 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -13920,6 +13920,32 @@ mod tests { ); } + #[test] + fn exited_sidebar_surface_transfers_its_shutdown_owner_before_removal() { + let mux = test_mux(); + mux.configure_sidebar_plugin(Some(SidebarPluginOptions { + command: vec!["sidebar-test".into()], + cwd: None, + })); + let status = mux.ensure_sidebar_plugin(80, 24, true); + let surface_id = status.surface.expect("sidebar surface was not created"); + let surface = mux.surface(surface_id).unwrap(); + surface.set_server_shutdown_failure_for_test(true); + + mux.surface_exited(surface_id); + + assert!(mux.surface(surface_id).is_none()); + assert_eq!( + mux.shutdown_owners.len(), + 1, + "sidebar exit discarded its still-retryable process owner" + ); + + surface.set_server_shutdown_failure_for_test(false); + let _ = mux.terminate_staged_shutdown_owners_until(Instant::now() + Duration::from_secs(1)); + assert!(mux.shutdown_owners.is_empty()); + } + #[cfg(unix)] #[test] fn rejected_terminal_admission_does_not_launch_a_process() { From 4477c67279864fbaaaeae28ed00629f661c52b78 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:59:29 -0700 Subject: [PATCH 163/521] fix(tui): retire exited sidebar surfaces --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e96ef4ff2ece..f3fea88ee0e6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8266,7 +8266,14 @@ impl Mux { runtime.last_error = Some("sidebar plugin exited".to_string()); runtime.retry_at = Some(Instant::now() + delay); drop(runtime); - self.state.lock().unwrap().surfaces.remove(&id); + let retired = { + let mut state = self.state.lock().unwrap(); + take_surface_for_retirement(self, &mut state, id) + }; + if let Some(surface) = retired { + self.purge_surface_side_tables(id); + self.retire_surface_runtime(surface); + } true } From 6fdf349f06cd0d4177a231e7417715b4906f3b0d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:08:27 -0700 Subject: [PATCH 164/521] test(tui): expose identity registry scan --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 ++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f3fea88ee0e6..1c61b8c527c9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3136,6 +3136,16 @@ impl Mux { self.workspace_registry.lock().unwrap().terminal_snapshot() } + #[cfg(test)] + pub(crate) fn reset_terminal_snapshot_count_for_test(&self) { + self.workspace_registry.lock().unwrap().reset_terminal_snapshot_count_for_test(); + } + + #[cfg(test)] + pub(crate) fn terminal_snapshot_count_for_test(&self) -> usize { + self.workspace_registry.lock().unwrap().terminal_snapshot_count_for_test() + } + pub fn terminal_registry_events_page( &self, revision: u64, diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 0a09e8e266ca..e5675faf6cb2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -7000,6 +7000,21 @@ mod tests { ); } + #[test] + fn identify_reads_terminal_revision_without_materializing_the_registry() { + let mux = test_mux(); + mux.reset_terminal_snapshot_count_for_test(); + + let identity = handle_command(&mux, 0, Command::Identify, &test_writer()).unwrap(); + + assert_eq!(identity["terminal_revision"].as_u64(), Some(0)); + assert_eq!( + mux.terminal_snapshot_count_for_test(), + 0, + "identity preflight materialized every terminal record" + ); + } + #[test] fn identify_exposes_shutdown_cleanup_health() { let mux = test_mux(); From 39aa2e0c730a12cdc2a533fd74809dbff309bd4a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:09:31 -0700 Subject: [PATCH 165/521] fix(tui): read identity revision directly --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 4 ++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 1c61b8c527c9..2a14a8eed5e5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3136,6 +3136,10 @@ impl Mux { self.workspace_registry.lock().unwrap().terminal_snapshot() } + pub fn terminal_registry_revision(&self) -> anyhow::Result { + self.workspace_registry.lock().unwrap().terminal_revision() + } + #[cfg(test)] pub(crate) fn reset_terminal_snapshot_count_for_test(&self) { self.workspace_registry.lock().unwrap().reset_terminal_snapshot_count_for_test(); diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index e5675faf6cb2..6eea6c650a44 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4266,7 +4266,7 @@ fn handle_command_with_cancellation( "registry_id": registry_id, "generation": generation, "workspace_revision": mux.with_state(|state| state.workspace_revision), - "terminal_revision": mux.terminal_registry_snapshot()?.revision, + "terminal_revision": mux.terminal_registry_revision()?, "daemon_handoff": 1, "shutdown_cleanup": { "pending": shutdown_cleanup.pending, diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 70e5461f9992..99a3c60d532a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -361,6 +361,10 @@ impl WorkspaceRegistry { &self.generation } + pub fn terminal_revision(&self) -> anyhow::Result { + current_terminal_revision(&self.connection) + } + /// Returns the canonical, non-tombstoned terminal placement projection. /// Runtime surface ids and renderer process ids are intentionally absent. pub fn terminal_snapshot(&self) -> anyhow::Result { From 5583ba0df2d357e2fba673dc054583a8d6e80aa2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:11:01 -0700 Subject: [PATCH 166/521] test(tui): expose legacy client drain race --- cmux-tui/crates/cmux-tui/tests/cli.rs | 32 ++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index d221479f3181..250dbaa2c5a5 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -337,6 +337,7 @@ fn legacy_server_process_helper() { | "exit-after-close" | "persistent-close-error" | "concurrent-client" + | "draining-client" | "browser-surface" ) { let mut command = Command::new("yes"); @@ -510,6 +511,7 @@ fn legacy_server_process_helper() { let mut reader = BufReader::new(stream.try_clone_box().unwrap()); let mut request = String::new(); let mut snapshot_index = 0; + let mut list_client_scans = 0; 'snapshots: loop { request.clear(); if reader.read_line(&mut request).unwrap() == 0 { @@ -519,12 +521,15 @@ fn legacy_server_process_helper() { } let list_request: serde_json::Value = serde_json::from_str(&request).unwrap(); if list_request["cmd"].as_str() == Some("list-clients") { + list_client_scans += 1; let mut clients = vec![serde_json::json!({ "client": 1, "transport": "unix", "self": true, })]; - if scenario == "concurrent-client" { + if scenario == "concurrent-client" + || (scenario == "draining-client" && list_client_scans == 1) + { clients.push(serde_json::json!({ "client": 2, "transport": "unix", @@ -568,6 +573,7 @@ fn legacy_server_process_helper() { | "exit-after-close" | "persistent-close-error" | "concurrent-client" + | "draining-client" | "zombie-child" | "reparent-on-close" | "unowned-surface" @@ -1766,6 +1772,30 @@ fn server_stop_fails_closed_when_another_legacy_client_can_create_surfaces() { assert!(transport::connect(&server.socket).is_ok(), "failed shutdown did not restore socket"); } +#[cfg(unix)] +#[test] +fn server_stop_waits_for_the_initiating_legacy_client_to_drain() { + let mut server = + LegacyServerProcess::start("legacy-server-draining-client", "draining-client", None); + let descendant_pid = server.descendant_pid().unwrap(); + + let output = Command::new(bin()) + .args(["server", "stop", "--socket"]) + .arg(&server.socket) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + + assert_success(&output); + let status = server.child.wait().unwrap(); + assert_eq!(status.signal(), Some(libc::SIGKILL)); + let deadline = Instant::now() + Duration::from_secs(5); + while process_exists(descendant_pid) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(!process_exists(descendant_pid)); +} + #[cfg(unix)] #[test] fn server_stop_closes_exited_legacy_placeholders_without_a_pid() { From 021b48a82cdcc198265ac2207ea0b77ebb465b8c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:12:26 -0700 Subject: [PATCH 167/521] fix(tui): await legacy client drain --- .../crates/cmux-tui/src/server_lifecycle.rs | 57 ++++++++++++------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 4088368c6d91..3a0fef6cb764 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -50,6 +50,8 @@ const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); #[cfg(unix)] const LEGACY_HELPER_POLL_INTERVAL: Duration = Duration::from_millis(10); #[cfg(unix)] +const LEGACY_CLIENT_DRAIN_POLL_INTERVAL: Duration = Duration::from_millis(10); +#[cfg(unix)] const LEGACY_HELPER_CANCEL_MARGIN: Duration = Duration::from_millis(100); #[cfg(unix)] const LEGACY_HELPER_REAPER_CAPACITY: usize = 64; @@ -871,31 +873,44 @@ impl ServerLifecycle { next_request_id: &mut u64, deadline: Instant, ) -> anyhow::Result<()> { - let request_id = take_legacy_request_id(next_request_id)?; - set_transport_deadline(self.reader.get_mut().as_ref(), deadline).map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - write_json_line(self.reader.get_mut(), &json!({"id": request_id, "cmd": "list-clients"})) + loop { + ensure_legacy_helper_active()?; + if Instant::now() >= deadline { + anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + } + let request_id = take_legacy_request_id(next_request_id)?; + set_transport_deadline(self.reader.get_mut().as_ref(), deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + write_json_line( + self.reader.get_mut(), + &json!({"id": request_id, "cmd": "list-clients"}), + ) .map_err(|_| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - let response = - read_response_until(&mut self.reader, request_id, deadline).map_err(|_| { anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; - let clients = response_data(&response)?.as_array().ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) - })?; - let exclusive = clients.len() == 1 - && clients[0].get("self").and_then(Value::as_bool) == Some(true) - && clients[0].get("transport").and_then(Value::as_str) == Some("unix"); - // The public listener name is already quarantined. Refusing every - // previously accepted peer leaves this helper as the only process - // capable of mutating legacy topology before the final freeze. - if !exclusive { - anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); + let response = + read_response_until(&mut self.reader, request_id, deadline).map_err(|_| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let clients = response_data(&response)?.as_array().ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) + })?; + let exclusive = clients.len() == 1 + && clients[0].get("self").and_then(Value::as_bool) == Some(true) + && clients[0].get("transport").and_then(Value::as_str) == Some("unix"); + // The public listener name is already quarantined, so the set can + // only drain. The initiating client's EOF cleanup may lag behind + // helper startup; persistent peers still consume the same bounded + // deadline and fail closed. + if exclusive { + return Ok(()); + } + std::thread::sleep( + LEGACY_CLIENT_DRAIN_POLL_INTERVAL + .min(deadline.saturating_duration_since(Instant::now())), + ); } - Ok(()) } #[cfg(unix)] From 76c9ddc20192d6092f9f5b46a63a372d01a686af Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:14:08 -0700 Subject: [PATCH 168/521] test(tui): expose listener ownership gap --- cmux-tui/crates/cmux-tui/src/main.rs | 6 ++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 48 +++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index bdc0d1e926b5..b3852ffab379 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1048,6 +1048,12 @@ struct ServerProcessShutdownGuard { impl ServerProcessShutdownGuard { fn start(mux: &Arc, socket_path: PathBuf) -> io::Result { + #[cfg(debug_assertions)] + if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_WATCHDOG_START_FAILURE") { + std::fs::write(marker, b"starting")?; + std::thread::sleep(std::time::Duration::from_millis(500)); + return Err(io::Error::other("forced watchdog start failure")); + } let shutdown_watch = mux.watch_shutdown_request(); let worker_watch = shutdown_watch.clone(); let worker_socket_path = socket_path.clone(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 250dbaa2c5a5..4da95cd23cf7 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -166,6 +166,54 @@ fn try_json_socket_request( (response["ok"] == true).then(|| response["data"].clone()) } +#[cfg(unix)] +#[test] +fn server_establishes_shutdown_ownership_before_publishing_its_listener() { + let dir = unique_temp_dir("watchdog-start-failure"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let marker = dir.join("watchdog-starting"); + let mut child = Command::new(bin()) + .args(["--headless", "--ephemeral", "--socket"]) + .arg(&socket) + .env("CMUX_TUI_TEST_WATCHDOG_START_FAILURE", &marker) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let marker_deadline = Instant::now() + Duration::from_secs(5); + while !marker.exists() && Instant::now() < marker_deadline { + assert!( + child.try_wait().unwrap().is_none(), + "server exited before entering the watchdog failure hook" + ); + std::thread::sleep(Duration::from_millis(10)); + } + assert!(marker.exists(), "watchdog failure hook was not reached"); + let listener_was_published = transport::connect(&socket).is_ok(); + + let exit_deadline = Instant::now() + Duration::from_secs(5); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if Instant::now() >= exit_deadline { + child.kill().unwrap(); + break child.wait().unwrap(); + } + std::thread::sleep(Duration::from_millis(10)); + }; + let _ = fs::remove_file(&socket); + let _ = fs::remove_dir_all(&dir); + + assert!(!status.success(), "forced watchdog failure unexpectedly started the server"); + assert!( + !listener_was_published, + "server accepted clients before its shutdown watchdog owned cleanup" + ); +} + fn terminal_host_pids(root: &std::path::Path) -> Vec { fs::read_dir(root) .ok() From fb88e5f80bb4ddc578cc8dabbbad039b6dc303ed Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:18:34 -0700 Subject: [PATCH 169/521] fix(tui): own shutdown before listeners --- cmux-tui/crates/cmux-tui/src/main.rs | 57 ++++++++++++++++++++------- cmux-tui/crates/cmux-tui/tests/cli.rs | 2 +- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index b3852ffab379..e86df44738f7 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1231,35 +1231,60 @@ fn run_server( // Ghostty's config before any protocol client can create a surface. mux.set_default_colors(config.terminal_defaults); mux.configure_sidebar_plugin(config.sidebar.plugin.clone()); + let server_process = ServerProcessShutdownGuard::start(&mux, socket_path.clone())?; #[cfg(target_os = "linux")] - let _provider_management = provider_management_listener + let _provider_management = match provider_management_listener .map(|listener| cmux_tui_core::provider_management::serve(listener, mux.clone())) - .transpose()?; + .transpose() + { + Ok(server) => server, + Err(error) => { + server_process.shutdown(); + server_process.complete(); + return Err(error.into()); + } + }; let websocket_server = match ws_addr { Some(addr) => { - let addr = addr - .parse() - .map_err(|error| anyhow::anyhow!("invalid WebSocket address {addr:?}: {error}"))?; - Some(cmux_tui_core::server::serve_websocket( + let addr = match addr.parse() { + Ok(addr) => addr, + Err(error) => { + #[cfg(target_os = "linux")] + drop(_provider_management); + server_process.shutdown(); + server_process.complete(); + return Err(anyhow::anyhow!("invalid WebSocket address {addr:?}: {error}")); + } + }; + match cmux_tui_core::server::serve_websocket( mux.clone(), addr, ws_token, args.ws_insecure_bind, - )?) + ) { + Ok(server) => Some(server), + Err(error) => { + #[cfg(target_os = "linux")] + drop(_provider_management); + server_process.shutdown(); + server_process.complete(); + return Err(error); + } + } } None => None, }; if let Some(server) = &websocket_server { eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } - cmux_tui_core::server::serve(mux.clone(), Some(socket_path.clone()))?; - let server_process = match ServerProcessShutdownGuard::start(&mux, socket_path.clone()) { - Ok(server_process) => server_process, - Err(error) => { - cmux_tui_core::server::cleanup(&socket_path); - return Err(error.into()); - } - }; + if let Err(error) = cmux_tui_core::server::serve(mux.clone(), Some(socket_path.clone())) { + drop(websocket_server); + #[cfg(target_os = "linux")] + drop(_provider_management); + server_process.shutdown(); + server_process.complete(); + return Err(error); + } #[cfg(debug_assertions)] if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { @@ -1278,6 +1303,8 @@ fn run_server( run_tui(Session::Local(mux), args.session, None) }; drop(websocket_server); + #[cfg(target_os = "linux")] + drop(_provider_management); server_process.shutdown(); server_process.complete(); result diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 4da95cd23cf7..36d2bd8e44be 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -191,7 +191,7 @@ fn server_establishes_shutdown_ownership_before_publishing_its_listener() { std::thread::sleep(Duration::from_millis(10)); } assert!(marker.exists(), "watchdog failure hook was not reached"); - let listener_was_published = transport::connect(&socket).is_ok(); + let listener_was_published = socket.exists(); let exit_deadline = Instant::now() + Duration::from_secs(5); let status = loop { From cb5df5d9932ce996925269ab974e470b22f05458 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 22:36:36 -0700 Subject: [PATCH 170/521] test(tui): expose macOS package mismatch --- cmux-tui/dist/scripts/test_package_pypi.py | 32 +++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 4fe16f801731..68280a0fb7c5 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -5,9 +5,10 @@ import os import unittest +from pathlib import Path from unittest.mock import patch -from package_pypi import wheel_bytes +from package_pypi import TARGETS, wheel_bytes def launcher_command(version: str, argv0: str) -> str: @@ -63,5 +64,34 @@ def test_stable_install_path_is_reused_directly(self) -> None: ) +class DarwinCompatibilityTests(unittest.TestCase): + def test_wheels_require_the_exact_signaling_macos_release(self) -> None: + tags = { + target.rust_target: target.platform_tags + for target in TARGETS + if target.rust_target.endswith("-apple-darwin") + } + + self.assertEqual( + tags, + { + "aarch64-apple-darwin": ("macosx_14_2_arm64",), + "x86_64-apple-darwin": ("macosx_14_2_x86_64",), + }, + ) + + def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: + compatibility_file = Path(__file__).resolve().parents[1] / "macos-deployment-target.txt" + self.assertEqual(compatibility_file.read_text().strip(), "14.2") + + workflow = ( + Path(__file__).resolve().parents[3] + / ".github" + / "workflows" + / "cmux-tui-build-package.yml" + ).read_text() + self.assertGreaterEqual(workflow.count("dist/macos-deployment-target.txt"), 2) + + if __name__ == "__main__": unittest.main() From c716651b3aa4f4237fc323f24984511a1bad7d5e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:00:36 -0700 Subject: [PATCH 171/521] test(tui): reject unrepresentable macOS wheel floor --- cmux-tui/dist/scripts/test_package_pypi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 68280a0fb7c5..50f06221ed73 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -65,7 +65,7 @@ def test_stable_install_path_is_reused_directly(self) -> None: class DarwinCompatibilityTests(unittest.TestCase): - def test_wheels_require_the_exact_signaling_macos_release(self) -> None: + def test_wheels_require_the_first_safe_macos_wheel_floor(self) -> None: tags = { target.rust_target: target.platform_tags for target in TARGETS @@ -75,14 +75,14 @@ def test_wheels_require_the_exact_signaling_macos_release(self) -> None: self.assertEqual( tags, { - "aarch64-apple-darwin": ("macosx_14_2_arm64",), - "x86_64-apple-darwin": ("macosx_14_2_x86_64",), + "aarch64-apple-darwin": ("macosx_15_0_arm64",), + "x86_64-apple-darwin": ("macosx_15_0_x86_64",), }, ) def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: compatibility_file = Path(__file__).resolve().parents[1] / "macos-deployment-target.txt" - self.assertEqual(compatibility_file.read_text().strip(), "14.2") + self.assertEqual(compatibility_file.read_text().strip(), "15.0") workflow = ( Path(__file__).resolve().parents[3] From c93ff005a9db7c091c39b91d8d7060183d53e5d1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:34:24 -0700 Subject: [PATCH 172/521] fix(tui): align macOS package target --- .github/workflows/cmux-tui-build-package.yml | 16 ++++++++++++++++ cmux-tui/dist/macos-deployment-target.txt | 1 + cmux-tui/dist/scripts/package_pypi.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 cmux-tui/dist/macos-deployment-target.txt diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index d4605d085dd1..376a3f9b28d5 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -108,6 +108,13 @@ jobs: shell: bash run: rustup target add ${{ matrix.target }} + - name: Configure macOS deployment target + if: runner.os == 'macOS' + shell: bash + run: | + target="$(tr -d '[:space:]' < cmux-tui/dist/macos-deployment-target.txt)" + printf 'MACOSX_DEPLOYMENT_TARGET=%s\n' "$target" >> "$GITHUB_ENV" + - name: Install cargo-zigbuild if: matrix.cross == true shell: bash @@ -154,6 +161,15 @@ jobs: fi ls -la dist + - name: Verify macOS deployment target + if: runner.os == 'macOS' + shell: bash + run: | + binary="dist/cmux-tui-${{ matrix.target }}${{ matrix.ext }}" + expected="$(tr -d '[:space:]' < cmux-tui/dist/macos-deployment-target.txt)" + actual="$(xcrun vtool -show-build "$binary" | awk '$1 == "minos" { print $2; exit }')" + test "$actual" = "$expected" + - name: Verify native binary identity if: matrix.cross == false shell: bash diff --git a/cmux-tui/dist/macos-deployment-target.txt b/cmux-tui/dist/macos-deployment-target.txt new file mode 100644 index 000000000000..fe6b09a7d14c --- /dev/null +++ b/cmux-tui/dist/macos-deployment-target.txt @@ -0,0 +1 @@ +15.0 diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index c4e90c6fabde..a3884d5ee80a 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -19,6 +19,17 @@ DIST_NAME = "cmux" PACKAGE_NAME = "cmux_tui" ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) +MACOS_DEPLOYMENT_TARGET = ( + Path(__file__).resolve().parents[1] / "macos-deployment-target.txt" +).read_text().strip() +if not re.fullmatch(r"[0-9]+\.[0-9]+", MACOS_DEPLOYMENT_TARGET): + raise RuntimeError("invalid macOS deployment target") +macos_major, macos_minor = (int(part) for part in MACOS_DEPLOYMENT_TARGET.split(".")) +if macos_major >= 11 and macos_minor != 0: + raise RuntimeError( + "macOS 11+ wheel compatibility floors must use a major .0 release" + ) +MACOS_WHEEL_VERSION = MACOS_DEPLOYMENT_TARGET.replace(".", "_") @dataclass(frozen=True) @@ -28,8 +39,8 @@ class Target: TARGETS = [ - Target("aarch64-apple-darwin", ("macosx_11_0_arm64",)), - Target("x86_64-apple-darwin", ("macosx_10_12_x86_64",)), + Target("aarch64-apple-darwin", (f"macosx_{MACOS_WHEEL_VERSION}_arm64",)), + Target("x86_64-apple-darwin", (f"macosx_{MACOS_WHEEL_VERSION}_x86_64",)), Target( "x86_64-unknown-linux-musl", ( From 9aa5d8dd72e5b8a4576aea33e9d9c2987008903a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:40:51 -0700 Subject: [PATCH 173/521] test(tui): expose Intel deployment target drift --- cmux-tui/dist/scripts/test_package_pypi.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 50f06221ed73..8f44d4f0c6d9 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -92,6 +92,23 @@ def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: ).read_text() self.assertGreaterEqual(workflow.count("dist/macos-deployment-target.txt"), 2) + def test_intel_release_uses_the_apple_linker_deployment_target(self) -> None: + workflow = ( + Path(__file__).resolve().parents[3] + / ".github" + / "workflows" + / "cmux-tui-build-package.yml" + ).read_text() + cross_build = workflow.split("- name: Build cmux-tui (cross)", 1)[1].split( + "- name: Stage binary", 1 + )[0] + + self.assertIn('if [[ "$RUNNER_OS" == macOS ]]; then', cross_build) + self.assertIn( + "cargo build -p cmux-tui --bin cmux-tui --release --locked --target", + cross_build, + ) + if __name__ == "__main__": unittest.main() From caf1d82224190ddc6e7f0d0b2a7917a9c132e961 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:41:44 -0700 Subject: [PATCH 174/521] fix(tui): honor Intel macOS package target --- .github/workflows/cmux-tui-build-package.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 376a3f9b28d5..3635d8cceead 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -116,7 +116,7 @@ jobs: printf 'MACOSX_DEPLOYMENT_TARGET=%s\n' "$target" >> "$GITHUB_ENV" - name: Install cargo-zigbuild - if: matrix.cross == true + if: matrix.cross == true && runner.os != 'macOS' shell: bash run: cargo install --locked cargo-zigbuild@0.20.0 @@ -148,7 +148,11 @@ jobs: CMUX_TUI_BUILD_COMMIT="$(git -C .. rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ../ghostty rev-parse HEAD)" export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT - cargo zigbuild -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.target }} + if [[ "$RUNNER_OS" == macOS ]]; then + cargo build -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.target }} + else + cargo zigbuild -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.target }} + fi - name: Stage binary shell: bash From 99e43eaf5517755170cb9af339086b98bf2e1396 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:08:01 -0700 Subject: [PATCH 175/521] test(tui): expose failed listener ownership --- cmux-tui/crates/cmux-tui/src/main.rs | 40 +++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index e86df44738f7..a6d4428f5ea2 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1577,11 +1577,38 @@ mod tests { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } + #[cfg(unix)] + #[test] + fn failed_server_publication_preserves_live_socket() { + use std::os::unix::net::{UnixListener, UnixStream}; + use std::time::{SystemTime, UNIX_EPOCH}; + + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let socket_dir = + std::env::temp_dir().join(format!("cg-publish-{}-{suffix:x}", std::process::id())); + std::fs::create_dir(&socket_dir).unwrap(); + let socket_path = socket_dir.join("server.sock"); + let existing = UnixListener::bind(&socket_path).unwrap(); + let mux = Mux::new("server-publication-test", SurfaceOptions::default()); + let guard = ServerProcessShutdownGuard::start(&mux, socket_path.clone()).unwrap(); + + assert!(cmux_tui_core::server::serve(mux, Some(socket_path.clone())).is_err()); + guard.shutdown(); + guard.complete(); + + let client = UnixStream::connect(&socket_path) + .expect("failed publication cleanup unlinked the live server socket"); + drop(client); + drop(existing); + cmux_tui_core::server::cleanup(&socket_path); + std::fs::remove_dir(socket_dir).unwrap(); + } + #[cfg(unix)] #[test] fn completed_server_guard_never_unlinks_a_replacement_socket() { use std::os::unix::net::{UnixListener, UnixStream}; - use std::time::{Duration, SystemTime, UNIX_EPOCH}; + use std::time::{SystemTime, UNIX_EPOCH}; let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); let socket_path = @@ -1592,13 +1619,7 @@ mod tests { let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); cleanup.state.lock().unwrap().complete = true; let (completion, completed) = std::sync::mpsc::channel(); - let (replacement_tx, replacement_rx) = std::sync::mpsc::sync_channel(1); - let replacement_path = socket_path.clone(); - let worker = std::thread::spawn(move || { - completed.recv().unwrap(); - let replacement = UnixListener::bind(&replacement_path).unwrap(); - replacement_tx.send(replacement).unwrap(); - }); + let worker = std::thread::spawn(move || completed.recv().unwrap()); let guard = ServerProcessShutdownGuard { socket_path: socket_path.clone(), shutdown_watch, @@ -1606,10 +1627,11 @@ mod tests { completion: Some(completion), worker: Some(worker), }; + std::fs::remove_file(&socket_path).unwrap(); + let replacement = UnixListener::bind(&socket_path).unwrap(); guard.complete(); - let replacement = replacement_rx.recv_timeout(Duration::from_secs(1)).unwrap(); let client = UnixStream::connect(&socket_path) .expect("guard completion unlinked the replacement server socket"); drop(client); From 92f0045473bb40ec734ef581f6af6a7edac224c8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 00:20:32 -0700 Subject: [PATCH 176/521] fix(tui): bind cleanup to published sockets --- cmux-tui/crates/cmux-tui-core/src/server.rs | 148 +++++++++++++++++++- cmux-tui/crates/cmux-tui/src/main.rs | 63 +++++++-- 2 files changed, 195 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 6eea6c650a44..9a0117fba53a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -84,6 +84,8 @@ pub const STACK_LAYOUT_PROTOCOL_VERSION: u32 = 9; pub const PER_SURFACE_CLIENT_SIZING_PROTOCOL_VERSION: u32 = 10; pub const PROTOCOL_VERSION: u32 = PER_SURFACE_CLIENT_SIZING_PROTOCOL_VERSION; const PROTOCOL_KEY_TEXT_MAX_BYTES: usize = CLEAR_HISTORY_KEY_TEXT_MAX_BYTES; +#[cfg(unix)] +static SOCKET_CLEANUP_SEQUENCE: AtomicU64 = AtomicU64::new(1); fn advertised_capabilities(bounded_clear_history_fallback_writes: bool) -> Vec<&'static str> { let mut capabilities = vec![ @@ -2645,8 +2647,132 @@ pub fn connect_existing_until( } } +/// Identity-coupled ownership of one published local server socket. +/// +/// Cleanup removes only the socket inode created by the matching successful +/// publication. A failed bind or a later replacement at the same path remains +/// outside this owner's authority. +pub struct PublishedSocket { + path: PathBuf, + #[cfg(unix)] + device: u64, + #[cfg(unix)] + inode: u64, +} + +impl PublishedSocket { + /// Claim a socket path immediately after a successful bind. + pub fn claim(path: PathBuf) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + + let metadata = std::fs::symlink_metadata(&path)?; + if !metadata.file_type().is_socket() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "published server path is not a socket", + )); + } + Ok(Self { path, device: metadata.dev(), inode: metadata.ino() }) + } + #[cfg(not(unix))] + { + Ok(Self { path }) + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Remove this publication without unlinking a path reused by another server. + pub fn cleanup(self) { + #[cfg(unix)] + { + let _ = self.cleanup_unix(); + } + #[cfg(not(unix))] + { + let _ = std::fs::remove_file(&self.path); + } + } + + #[cfg(unix)] + fn matches(&self, path: &Path) -> std::io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt}; + + let metadata = std::fs::symlink_metadata(path)?; + Ok(metadata.file_type().is_socket() + && metadata.dev() == self.device + && metadata.ino() == self.inode) + } + + #[cfg(unix)] + fn cleanup_unix(&self) -> std::io::Result<()> { + match self.matches(&self.path) { + Ok(true) => {} + Ok(false) => return Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + + let parent = self.path.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "published server socket has no parent directory", + ) + })?; + let file_name = + self.path.file_name().map(|name| name.to_string_lossy()).unwrap_or_default(); + let quarantine_dir = loop { + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let candidate = + parent.join(format!(".{file_name}.cleanup-{}-{sequence:x}", std::process::id())); + match std::fs::create_dir(&candidate) { + Ok(()) => { + if let Err(error) = platform::restrict_directory(&candidate) { + let _ = std::fs::remove_dir(&candidate); + return Err(error); + } + break candidate; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + }; + let quarantined = quarantine_dir.join("socket"); + if let Err(error) = std::fs::rename(&self.path, &quarantined) { + let _ = std::fs::remove_dir(&quarantine_dir); + return if error.kind() == std::io::ErrorKind::NotFound { Ok(()) } else { Err(error) }; + } + + match self.matches(&quarantined) { + Ok(true) => { + let removed = std::fs::remove_file(&quarantined); + let _ = std::fs::remove_dir(&quarantine_dir); + removed + } + Ok(false) | Err(_) => { + // A replacement won the race between the public identity + // check and rename. Restore it without overwriting any newer + // publication that may already occupy the public path. + std::fs::hard_link(&quarantined, &self.path)?; + std::fs::remove_file(&quarantined)?; + std::fs::remove_dir(&quarantine_dir) + } + } + } +} + /// Bind the socket and serve connections on background threads. pub fn serve(mux: Arc, path: Option) -> anyhow::Result { + let published = serve_owned(mux, path)?; + Ok(published.path().to_path_buf()) +} + +/// Bind the socket and return its exact publication ownership. +pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result { let path = path.unwrap_or_else(|| default_socket_path(&mux.session)); if let Some(dir) = path.parent() { std::fs::create_dir_all(dir)?; @@ -2668,10 +2794,21 @@ pub fn serve(mux: Arc, path: Option) -> anyhow::Result { } } let listener = transport::listen(&path)?; - platform::restrict_file(&path)?; + let published = match PublishedSocket::claim(path.clone()) { + Ok(published) => published, + Err(error) => { + // The listener drops here. Leave the path for the next liveness + // probe rather than unlinking without a captured socket identity. + return Err(error.into()); + } + }; + if let Err(error) = platform::restrict_file(&path) { + published.cleanup(); + return Err(error.into()); + } let active_connections = Arc::new(AtomicU64::new(0)); - std::thread::Builder::new().name("mux-server".into()).spawn(move || { + if let Err(error) = std::thread::Builder::new().name("mux-server".into()).spawn(move || { loop { let Ok(stream) = listener.accept() else { continue }; let Some(permit) = claim_connection(&active_connections) else { continue }; @@ -2680,8 +2817,11 @@ pub fn serve(mux: Arc, path: Option) -> anyhow::Result { handle_connection_with_permit(mux, stream, Some(permit)); }); } - })?; - Ok(path) + }) { + published.cleanup(); + return Err(error.into()); + } + Ok(published) } /// A running opt-in WebSocket listener. Dropping it stops accepts and closes clients. diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index a6d4428f5ea2..44a59584d1ef 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1039,7 +1039,7 @@ impl ServerShutdownCleanup { } struct ServerProcessShutdownGuard { - socket_path: PathBuf, + published_socket: Arc>>, shutdown_watch: cmux_tui_core::ShutdownRequestWatch, cleanup: Arc, completion: Option>, @@ -1047,7 +1047,7 @@ struct ServerProcessShutdownGuard { } impl ServerProcessShutdownGuard { - fn start(mux: &Arc, socket_path: PathBuf) -> io::Result { + fn start(mux: &Arc) -> io::Result { #[cfg(debug_assertions)] if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_WATCHDOG_START_FAILURE") { std::fs::write(marker, b"starting")?; @@ -1056,7 +1056,8 @@ impl ServerProcessShutdownGuard { } let shutdown_watch = mux.watch_shutdown_request(); let worker_watch = shutdown_watch.clone(); - let worker_socket_path = socket_path.clone(); + let published_socket = Arc::new(std::sync::Mutex::new(None)); + let worker_published_socket = published_socket.clone(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux.clone())); let worker_cleanup = cleanup.clone(); let (completion, completed) = std::sync::mpsc::channel(); @@ -1078,13 +1079,13 @@ impl ServerProcessShutdownGuard { // Cleanup already proved that no external ownership // remains. The frontend alone is stuck, so the old // process can now release its control socket. - cmux_tui_core::server::cleanup(&worker_socket_path); + cleanup_published_socket(&worker_published_socket); std::process::exit(0); } }, )?; Ok(Self { - socket_path, + published_socket, shutdown_watch, cleanup, completion: Some(completion), @@ -1092,6 +1093,20 @@ impl ServerProcessShutdownGuard { }) } + fn publish_socket(&self, published: cmux_tui_core::server::PublishedSocket) -> io::Result<()> { + let mut slot = + self.published_socket.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if slot.is_some() { + published.cleanup(); + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "server socket ownership is already published", + )); + } + *slot = Some(published); + Ok(()) + } + fn shutdown(&self) { self.cleanup.run_until_complete(); } @@ -1105,7 +1120,7 @@ impl ServerProcessShutdownGuard { return; } let Some(completion) = self.completion.take() else { return }; - cmux_tui_core::server::cleanup(&self.socket_path); + cleanup_published_socket(&self.published_socket); let _ = completion.send(()); self.shutdown_watch.cancel(); if let Some(worker) = self.worker.take() { @@ -1120,6 +1135,16 @@ impl Drop for ServerProcessShutdownGuard { } } +fn cleanup_published_socket( + published_socket: &std::sync::Mutex>, +) { + let published = + published_socket.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + if let Some(published) = published { + published.cleanup(); + } +} + fn server_shutdown_exit_grace() -> std::time::Duration { #[cfg(debug_assertions)] if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS") @@ -1231,7 +1256,7 @@ fn run_server( // Ghostty's config before any protocol client can create a surface. mux.set_default_colors(config.terminal_defaults); mux.configure_sidebar_plugin(config.sidebar.plugin.clone()); - let server_process = ServerProcessShutdownGuard::start(&mux, socket_path.clone())?; + let server_process = ServerProcessShutdownGuard::start(&mux)?; #[cfg(target_os = "linux")] let _provider_management = match provider_management_listener .map(|listener| cmux_tui_core::provider_management::serve(listener, mux.clone())) @@ -1277,13 +1302,25 @@ fn run_server( if let Some(server) = &websocket_server { eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } - if let Err(error) = cmux_tui_core::server::serve(mux.clone(), Some(socket_path.clone())) { + let published_socket = + match cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())) { + Ok(published) => published, + Err(error) => { + drop(websocket_server); + #[cfg(target_os = "linux")] + drop(_provider_management); + server_process.shutdown(); + server_process.complete(); + return Err(error); + } + }; + if let Err(error) = server_process.publish_socket(published_socket) { drop(websocket_server); #[cfg(target_os = "linux")] drop(_provider_management); server_process.shutdown(); server_process.complete(); - return Err(error); + return Err(error.into()); } #[cfg(debug_assertions)] @@ -1590,7 +1627,7 @@ mod tests { let socket_path = socket_dir.join("server.sock"); let existing = UnixListener::bind(&socket_path).unwrap(); let mux = Mux::new("server-publication-test", SurfaceOptions::default()); - let guard = ServerProcessShutdownGuard::start(&mux, socket_path.clone()).unwrap(); + let guard = ServerProcessShutdownGuard::start(&mux).unwrap(); assert!(cmux_tui_core::server::serve(mux, Some(socket_path.clone())).is_err()); guard.shutdown(); @@ -1614,6 +1651,7 @@ mod tests { let socket_path = std::env::temp_dir().join(format!("cg-{}-{suffix:x}.sock", std::process::id())); let original = UnixListener::bind(&socket_path).unwrap(); + let published = cmux_tui_core::server::PublishedSocket::claim(socket_path.clone()).unwrap(); let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); @@ -1621,7 +1659,7 @@ mod tests { let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || completed.recv().unwrap()); let guard = ServerProcessShutdownGuard { - socket_path: socket_path.clone(), + published_socket: Arc::new(std::sync::Mutex::new(Some(published))), shutdown_watch, cleanup, completion: Some(completion), @@ -1650,6 +1688,7 @@ mod tests { let socket_path = std::env::temp_dir() .join(format!("cg-incomplete-{}-{suffix:x}.sock", std::process::id())); let listener = UnixListener::bind(&socket_path).unwrap(); + let published = cmux_tui_core::server::PublishedSocket::claim(socket_path.clone()).unwrap(); let mux = Mux::new("incomplete-server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); @@ -1658,7 +1697,7 @@ mod tests { let _ = completed.recv(); }); let guard = ServerProcessShutdownGuard { - socket_path: socket_path.clone(), + published_socket: Arc::new(std::sync::Mutex::new(Some(published))), shutdown_watch, cleanup, completion: Some(completion), From 481e0d5d561b54821dd39e0d9367e3374e8ece21 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:09:18 -0700 Subject: [PATCH 177/521] test(tui): expose inaccessible proc entries --- .../cmux-tui-core/src/process_session.rs | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 3ddd1570bb52..b3e4154feb80 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1343,6 +1343,19 @@ fn scan_sessions( sessions: &HashSet, deadline: Option, ) -> io::Result>> { + scan_sessions_with(sessions, deadline, || all_process_ids(deadline), active_process_session) +} + +fn scan_sessions_with( + sessions: &HashSet, + deadline: Option, + process_ids: P, + mut process_session: S, +) -> io::Result>> +where + P: FnOnce() -> io::Result>, + S: FnMut(libc::pid_t) -> io::Result>, +{ let mut members = sessions.iter().copied().map(|session| (session, Vec::new())).collect::>(); if sessions.is_empty() { @@ -1350,12 +1363,12 @@ fn scan_sessions( } #[cfg(test)] PROCESS_TABLE_SCAN_COUNT.set(PROCESS_TABLE_SCAN_COUNT.get() + 1); - for pid in all_process_ids(deadline)? { + for pid in process_ids()? { ensure_before_deadline(deadline)?; if pid <= 1 { continue; } - let Some(current_session) = active_process_session(pid)? else { + let Some(current_session) = process_session(pid)? else { continue; }; ensure_before_deadline(deadline)?; @@ -1914,6 +1927,35 @@ mod tests { ); } + #[cfg(target_os = "linux")] + #[test] + fn session_scan_skips_inaccessible_foreign_processes_but_not_leaders() { + let sessions = HashSet::from([41]); + let snapshot = scan_sessions_with( + &sessions, + None, + || Ok(vec![73, 41]), + |pid| { + if pid == 73 { + Err(io::Error::from_raw_os_error(libc::EACCES)) + } else { + Ok(Some(41)) + } + }, + ) + .expect("an inaccessible foreign process aborted the session scan"); + assert_eq!(snapshot.get(&41), Some(&vec![41])); + + let error = scan_sessions_with( + &sessions, + None, + || Ok(vec![41]), + |_| Err(io::Error::from_raw_os_error(libc::EACCES)), + ) + .expect_err("an inaccessible session leader was silently skipped"); + assert_eq!(error.raw_os_error(), Some(libc::EACCES)); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn concurrent_shutdown_refreshes_share_one_new_snapshot_generation() { From 3c626f1d2aaadbd0e7f0cc09b9f12e9032bca1fb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:10:21 -0700 Subject: [PATCH 178/521] fix(tui): skip inaccessible foreign processes --- cmux-tui/crates/cmux-tui-core/src/process_session.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index b3e4154feb80..a329303e2330 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1368,8 +1368,15 @@ where if pid <= 1 { continue; } - let Some(current_session) = process_session(pid)? else { - continue; + let current_session = match process_session(pid) { + Ok(Some(current_session)) => current_session, + Ok(None) => continue, + Err(error) + if error.kind() == io::ErrorKind::PermissionDenied && !sessions.contains(&pid) => + { + continue; + } + Err(error) => return Err(error), }; ensure_before_deadline(deadline)?; if let Some(session_members) = members.get_mut(¤t_session) { From ac9a33f8c5e8c41a1cf6732fa9e8c698a87f2079 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:13:00 -0700 Subject: [PATCH 179/521] test(tui): expose unverified socket cleanup --- cmux-tui/crates/cmux-tui-core/src/server.rs | 29 ++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 9a0117fba53a..9bdace6601c0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2647,6 +2647,11 @@ pub fn connect_existing_until( } } +#[cfg(any(not(unix), test))] +fn cleanup_without_stable_identity(path: &Path) { + let _ = std::fs::remove_file(path); +} + /// Identity-coupled ownership of one published local server socket. /// /// Cleanup removes only the socket inode created by the matching successful @@ -2694,7 +2699,7 @@ impl PublishedSocket { } #[cfg(not(unix))] { - let _ = std::fs::remove_file(&self.path); + cleanup_without_stable_identity(&self.path); } } @@ -6190,6 +6195,28 @@ mod tests { ); } + #[test] + fn cleanup_without_stable_identity_preserves_a_replacement_path() { + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let directory = std::env::temp_dir() + .join(format!("cmux-unverified-publication-{}-{sequence:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("server.sock"); + std::fs::write(&path, b"original").unwrap(); + std::fs::remove_file(&path).unwrap(); + std::fs::write(&path, b"replacement").unwrap(); + + cleanup_without_stable_identity(&path); + + let replacement = std::fs::read(&path); + let _ = std::fs::remove_file(&path); + std::fs::remove_dir(directory).unwrap(); + assert_eq!( + replacement.expect("unverified cleanup removed a replacement publication"), + b"replacement" + ); + } + #[cfg(unix)] #[test] fn default_socket_path_falls_back_for_long_tmpdir() { From d244937627bf1255f5c9dc49e179e78e8cf07997 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:14:26 -0700 Subject: [PATCH 180/521] fix(tui): fail closed without socket identity --- cmux-tui/crates/cmux-tui-core/src/server.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 9bdace6601c0..3c3af47f935a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2648,15 +2648,18 @@ pub fn connect_existing_until( } #[cfg(any(not(unix), test))] -fn cleanup_without_stable_identity(path: &Path) { - let _ = std::fs::remove_file(path); +fn cleanup_without_stable_identity(_path: &Path) { + // Windows does not expose a race-free socket publication identity through + // this transport. Leave the path in place; the next server performs a + // bounded liveness probe before reclaiming a stale socket. } /// Identity-coupled ownership of one published local server socket. /// -/// Cleanup removes only the socket inode created by the matching successful -/// publication. A failed bind or a later replacement at the same path remains -/// outside this owner's authority. +/// Unix cleanup removes only the socket inode created by the matching +/// successful publication. Platforms without a stable socket identity leave +/// cleanup to the next server's liveness probe. A failed bind or a later +/// replacement at the same path remains outside this owner's authority. pub struct PublishedSocket { path: PathBuf, #[cfg(unix)] From dbf1ee700d4502d1eab3c8f5c85a2fa1b86cfddb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:15:10 -0700 Subject: [PATCH 181/521] test(tui): expose launcher wheel tag drift --- tests/test_tui_package_launchers.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index 2aa7c28de28f..97b16a995896 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -44,6 +44,21 @@ def test_pypi_fixture_targets_match_packager() -> None: assert RUST_TARGETS == declared +def test_pypi_fixture_wheel_tags_match_packager() -> None: + packager = runpy.run_path(str(PYPI_PACKAGER)) + fixture_tags = tuple( + WHEEL_TAG_BY_PLATFORM[key] + for key in ( + ("darwin", "arm64"), + ("darwin", "x86_64"), + ("linux", "x86_64"), + ("linux", "aarch64"), + ) + ) + declared = tuple(target.platform_tags[0] for target in packager["TARGETS"]) + assert fixture_tags == declared + + def normalized_platform() -> tuple[str, str]: machine = platform.machine().lower() if machine in {"amd64", "x64"}: From 21b5fbbc9666d0fe21fc1e41a169576381958cbb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:16:06 -0700 Subject: [PATCH 182/521] fix(tui): derive launcher wheel tags --- tests/test_tui_package_launchers.py | 39 +++++++++++++---------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index 97b16a995896..c59aa9ded4f0 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -23,39 +23,34 @@ ("linux", "aarch64"): "cmux-tui-linux-arm64", } +RUST_TARGET_BY_PLATFORM = { + ("darwin", "arm64"): "aarch64-apple-darwin", + ("darwin", "x86_64"): "x86_64-apple-darwin", + ("linux", "x86_64"): "x86_64-unknown-linux-musl", + ("linux", "aarch64"): "aarch64-unknown-linux-musl", +} +PYPI_TARGETS = tuple(runpy.run_path(str(PYPI_PACKAGER))["TARGETS"]) +RUST_TARGETS = tuple(target.rust_target for target in PYPI_TARGETS) WHEEL_TAG_BY_PLATFORM = { - ("darwin", "arm64"): "macosx_11_0_arm64", - ("darwin", "x86_64"): "macosx_10_12_x86_64", - ("linux", "x86_64"): "manylinux_2_17_x86_64.manylinux2014_x86_64", - ("linux", "aarch64"): "manylinux_2_17_aarch64.manylinux2014_aarch64", + platform_key: next( + target.platform_tags[0] + for target in PYPI_TARGETS + if target.rust_target == rust_target + ) + for platform_key, rust_target in RUST_TARGET_BY_PLATFORM.items() } -RUST_TARGETS = ( - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", -) - def test_pypi_fixture_targets_match_packager() -> None: - packager = runpy.run_path(str(PYPI_PACKAGER)) - declared = tuple(target.rust_target for target in packager["TARGETS"]) - assert RUST_TARGETS == declared + assert tuple(RUST_TARGET_BY_PLATFORM.values()) == RUST_TARGETS def test_pypi_fixture_wheel_tags_match_packager() -> None: - packager = runpy.run_path(str(PYPI_PACKAGER)) fixture_tags = tuple( WHEEL_TAG_BY_PLATFORM[key] - for key in ( - ("darwin", "arm64"), - ("darwin", "x86_64"), - ("linux", "x86_64"), - ("linux", "aarch64"), - ) + for key in RUST_TARGET_BY_PLATFORM ) - declared = tuple(target.platform_tags[0] for target in packager["TARGETS"]) + declared = tuple(target.platform_tags[0] for target in PYPI_TARGETS) assert fixture_tags == declared From 852cb5875db0902f355967f6cdf80b4af5b8bc1c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:08:32 -0700 Subject: [PATCH 183/521] test(tui): expose post-spawn PTY leaks --- .../cmux-tui-core/src/process_session.rs | 41 ++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 41 ++++++++++++ .../src/terminal_host_runtime.rs | 64 +++++++++++++++++++ 3 files changed, 146 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index a329303e2330..ef60dd43cb44 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -39,6 +39,8 @@ static STABLE_PROCESS_SIGNALING: OnceLock<()> = OnceLock::new(); thread_local! { static FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; + static POST_SPAWN_FAILURE_MARKER: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; } #[cfg(test)] @@ -51,6 +53,45 @@ pub(crate) fn set_process_session_preflight_failure_for_test(enabled: bool) { FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.set(enabled); } +#[cfg(test)] +pub(crate) struct ForcedPostSpawnFailure { + previous: Option, +} + +#[cfg(test)] +impl Drop for ForcedPostSpawnFailure { + fn drop(&mut self) { + POST_SPAWN_FAILURE_MARKER.with_borrow_mut(|marker| { + *marker = self.previous.take(); + }); + } +} + +#[cfg(test)] +pub(crate) fn force_post_spawn_failure_for_test( + marker: std::path::PathBuf, +) -> ForcedPostSpawnFailure { + let previous = POST_SPAWN_FAILURE_MARKER.replace(Some(marker)); + ForcedPostSpawnFailure { previous } +} + +#[cfg(test)] +pub(crate) fn fail_after_pty_spawn_for_test() -> io::Result<()> { + let marker = POST_SPAWN_FAILURE_MARKER.with_borrow(Clone::clone); + let Some(marker) = marker else { return Ok(()) }; + let deadline = Instant::now() + Duration::from_secs(2); + while !marker.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + if !marker.exists() { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "forced post-spawn failure marker was not published", + )); + } + Err(io::Error::other("forced post-spawn PTY initialization failure")) +} + /// Synchronization shared by a WNOWAIT child observer and explicit PTY /// shutdown. The session leader remains reserved until this state machine /// confirms that cleanup completed and performs the sole final reap. diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 5a92b861b5df..4ae991573792 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1795,6 +1795,8 @@ impl Surface { drop(pty.slave); let killer = child.clone_killer(); let process = LocalPtyProcess::new(pid, killer); + #[cfg(test)] + crate::process_session::fail_after_pty_spawn_for_test()?; let reader_process = process.clone(); let mut reader = pty.master.try_clone_reader()?; let writer = pty.master.take_writer()?; @@ -4084,6 +4086,45 @@ mod tests { assert_eq!(observed, baseline, "PTY descriptors were created outside the process barrier"); } + #[cfg(unix)] + #[test] + fn failed_local_pty_initialization_reaps_spawned_child() { + let root = std::env::temp_dir() + .join(format!("cmux-local-spawn-failure-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let ready = root.join("ready"); + let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); + let mux = Mux::new_for_test("local-spawn-failure", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + format!("trap '' HUP TERM; echo $$ > {}; exec /bin/sleep 60", ready.display()), + ]), + ..SurfaceOptions::default() + }; + + let error = Surface::spawn(1, options, Arc::downgrade(&mux)) + .expect_err("forced local PTY initialization failure unexpectedly succeeded"); + assert!(error.to_string().contains("forced post-spawn PTY initialization failure")); + let pid = std::fs::read_to_string(&ready).unwrap().trim().parse::().unwrap(); + let mut status = 0; + // SAFETY: the marker contains the exact child PID spawned by this test. + let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; + let wait_error = std::io::Error::last_os_error(); + if result == 0 { + // SAFETY: the child is still owned by this test process. + unsafe { + libc::kill(pid, libc::SIGKILL); + libc::waitpid(pid, &raw mut status, 0); + } + } + let _ = std::fs::remove_dir_all(root); + + assert_eq!(result, -1, "failed local PTY initialization left child {pid} unreaped"); + assert_eq!(wait_error.raw_os_error(), Some(libc::ECHILD)); + } + #[derive(Clone, Default)] struct CapturingWriter(Arc>>); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 7b806c0231d9..fd96779b0bf5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3253,6 +3253,8 @@ mod unix { command.cwd(cwd); } let mut child = pty.slave.spawn_command(command)?; + #[cfg(test)] + crate::process_session::fail_after_pty_spawn_for_test()?; drop(process_creation); let pid = child.process_id(); let Some(session) = pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { @@ -4129,6 +4131,68 @@ mod unix { ); } + #[test] + fn failed_host_pty_initialization_reaps_spawned_child() { + let root = std::env::temp_dir().join(format!( + "cmux-host-spawn-failure-{}", + crate::workspace_registry::new_uuid_v4() + )); + fs::create_dir_all(&root).unwrap(); + let ready = root.join("ready"); + let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: root.join("host.sock").to_string_lossy().into_owned(), + record_path: root.join("host.json").to_string_lossy().into_owned(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec![ + "/bin/sh".into(), + "-c".into(), + format!("trap '' HUP TERM; echo $$ > {}; exec /bin/sleep 60", ready.display()), + ], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + + let error = match spawn_host_runtime(&launch, &bootstrapped) { + Ok(_) => panic!("forced hosted PTY initialization failure unexpectedly succeeded"), + Err(error) => error, + }; + assert!(error.to_string().contains("forced post-spawn PTY initialization failure")); + let pid = fs::read_to_string(&ready).unwrap().trim().parse::().unwrap(); + let mut status = 0; + // SAFETY: the marker contains the exact child PID spawned by this test. + let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; + let wait_error = std::io::Error::last_os_error(); + if result == 0 { + // SAFETY: the child is still owned by this test process. + unsafe { + libc::kill(pid, libc::SIGKILL); + libc::waitpid(pid, &raw mut status, 0); + } + } + let _ = fs::remove_dir_all(root); + + assert_eq!(result, -1, "failed hosted PTY initialization left child {pid} unreaped"); + assert_eq!(wait_error.raw_os_error(), Some(libc::ECHILD)); + } + #[cfg(target_os = "linux")] #[test] fn saturated_listener_never_returns_an_unconnected_host_stream() { From fcb9783aa7bade807db5c31112806cb0e7f6723e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:11:42 -0700 Subject: [PATCH 184/521] fix(tui): own PTY children before initialization --- cmux-tui/crates/cmux-tui-core/src/lib.rs | 1 + .../cmux-tui-core/src/spawned_pty_child.rs | 42 +++++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 38 ++++------------- .../src/terminal_host_runtime.rs | 18 ++++---- 4 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index d4d4f08e556a..b78f1c2c8205 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -17,6 +17,7 @@ mod pairing; pub mod process_session; pub mod provider_management; mod short_id; +mod spawned_pty_child; mod surface; mod workspace_registry; diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs new file mode 100644 index 000000000000..317a9b344304 --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -0,0 +1,42 @@ +//! Immediate ownership of a spawned PTY child until its reaper completes. + +pub(crate) struct SpawnedPtyChild { + child: Option>, +} + +impl SpawnedPtyChild { + pub(crate) fn new(child: Box) -> Self { + Self { child: Some(child) } + } + + pub(crate) fn process_id(&self) -> Option { + self.child.as_ref().expect("spawned PTY child is present").process_id() + } + + pub(crate) fn clone_killer(&self) -> Box { + self.child.as_ref().expect("spawned PTY child is present").clone_killer() + } + + pub(crate) fn wait(&mut self) -> std::io::Result { + let result = self.child.as_mut().expect("spawned PTY child is present").wait(); + if result.is_ok() { + self.child.take(); + } + result + } + + /// Release a child only after stable identity proves that this process no + /// longer owns its wait handle and the numeric PID belongs to no live + /// process from the original publication. + pub(crate) fn abandon_wait_ownership(mut self) { + self.child.take(); + } +} + +impl Drop for SpawnedPtyChild { + fn drop(&mut self) { + let Some(mut child) = self.child.take() else { return }; + let _ = child.kill(); + let _ = child.wait(); + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 4ae991573792..700cd96688aa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -933,19 +933,13 @@ impl LocalPtyProcess { fn spawn_reaper( self: &Arc, - mut child: Box, + child: crate::spawned_pty_child::SpawnedPtyChild, reaper: LocalChildReaperLease, ) -> std::io::Result<()> { #[cfg(unix)] { let process = self.clone(); let Some(session) = process.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { - let _ = child.kill(); - let _ = child.wait(); - process.child_reaped.store(true, Ordering::Release); - process.child_waitable.store(true, Ordering::Release); - *process.exited.0.lock().unwrap() = true; - process.exited.1.notify_all(); return Err(std::io::Error::new( std::io::ErrorKind::Unsupported, "native PTY child did not expose a process ID", @@ -955,24 +949,12 @@ impl LocalPtyProcess { { Ok(Some(identity)) => identity, Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - process.child_reaped.store(true, Ordering::Release); - *process.exited.0.lock().unwrap() = true; - process.exited.1.notify_all(); return Err(std::io::Error::new( std::io::ErrorKind::NotFound, "native PTY child exited before its process identity was captured", )); } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - process.child_reaped.store(true, Ordering::Release); - *process.exited.0.lock().unwrap() = true; - process.exited.1.notify_all(); - return Err(error); - } + Err(error) => return Err(error), }; let observe_process = process.clone(); let cleanup_process = process.clone(); @@ -1013,7 +995,10 @@ impl LocalPtyProcess { return crate::process_session::NaturalReapFinish::Failed; } } - drop(child.take()); + child + .take() + .expect("reserved child releases lost wait ownership once") + .abandon_wait_ownership(); reap_process.child_reaped.store(true, Ordering::Release); drop(_signal); *reap_process.exited.0.lock().unwrap() = true; @@ -1053,7 +1038,7 @@ impl LocalPtyProcess { { let process = self.clone(); let (child_sender, child_receiver) = - sync_channel::>(1); + sync_channel::(1); let observer = std::thread::Builder::new().name("surface-child".into()).spawn(move || { let Ok(mut child) = child_receiver.recv() else { return }; @@ -1062,14 +1047,9 @@ impl LocalPtyProcess { process.exited.1.notify_all(); }); if let Err(error) = observer { - let _ = child.kill(); - let _ = child.wait(); return Err(error); } - if let Err(error) = child_sender.send(child) { - let mut child = error.0; - let _ = child.kill(); - let _ = child.wait(); + if child_sender.send(child).is_err() { return Err(std::io::Error::new( std::io::ErrorKind::BrokenPipe, "surface child observer exited before accepting ownership", @@ -1789,7 +1769,7 @@ impl Surface { cmd.cwd(cwd); } - let child = pty.slave.spawn_command(cmd)?; + let child = crate::spawned_pty_child::SpawnedPtyChild::new(pty.slave.spawn_command(cmd)?); drop(process_creation); let pid = child.process_id(); drop(pty.slave); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index fd96779b0bf5..2aed22b3dc5f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3252,30 +3252,23 @@ mod unix { if let Some(cwd) = launch.cwd.as_deref() { command.cwd(cwd); } - let mut child = pty.slave.spawn_command(command)?; + let child = pty.slave.spawn_command(command)?; + let child = crate::spawned_pty_child::SpawnedPtyChild::new(child); #[cfg(test)] crate::process_session::fail_after_pty_spawn_for_test()?; drop(process_creation); let pid = child.process_id(); let Some(session) = pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { - let _ = child.kill(); - let _ = child.wait(); anyhow::bail!("native terminal-host PTY child did not expose a process ID"); }; let child_identity = match crate::process_session::StableProcessHandle::capture(session) { Ok(Some(identity)) => identity, Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); anyhow::bail!( "terminal-host PTY child exited before its process identity was captured" ); } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(error).context("capture terminal-host PTY child identity"); - } + Err(error) => return Err(error).context("capture terminal-host PTY child identity"), }; drop(pty.slave); let killer = child.clone_killer(); @@ -3466,7 +3459,10 @@ mod unix { return crate::process_session::NaturalReapFinish::Failed; } } - drop(child.take()); + child + .take() + .expect("reserved child releases lost wait ownership once") + .abandon_wait_ownership(); reap_host.child_reaped.store(true, Ordering::Release); drop(_signal); let mut exited = reap_host.child_exit.0.lock().unwrap(); From 3100ea836b06ca5980c603c2fcd97142c59a03c2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:13:58 -0700 Subject: [PATCH 185/521] test(web): update Subrouter env fixture --- web/tests/client-config-env.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/tests/client-config-env.test.ts b/web/tests/client-config-env.test.ts index d5644f936b87..f23d7f3f8e2a 100644 --- a/web/tests/client-config-env.test.ts +++ b/web/tests/client-config-env.test.ts @@ -10,6 +10,8 @@ const requiredEnv = { STACK_SECRET_SERVER_KEY: "stack-secret", NEXT_PUBLIC_STACK_PROJECT_ID: "stack-project", NEXT_PUBLIC_STACK_PUBLISHABLE_CLIENT_KEY: "stack-public", + SUBROUTER_ENFORCE_STACK_PERMISSIONS: "1", + SUBROUTER_ALLOWED_TEAM_IDS: "team-test", }; const requiredIrohProductionEnv = { From 49882924e71db30786123b5c208ee11e66f435c0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 03:15:25 -0700 Subject: [PATCH 186/521] fix(tui): gate PTY failure hook to Unix --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 700cd96688aa..d53a6a11b00d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1775,7 +1775,7 @@ impl Surface { drop(pty.slave); let killer = child.clone_killer(); let process = LocalPtyProcess::new(pid, killer); - #[cfg(test)] + #[cfg(all(test, unix))] crate::process_session::fail_after_pty_spawn_for_test()?; let reader_process = process.clone(); let mut reader = pty.master.try_clone_reader()?; From fffe9cd0b3359566e1d0b86d513ff335968bd15e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:07:13 -0700 Subject: [PATCH 187/521] test(tui): expose launcher metadata leakage --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 45 ++++++++++++ .../src/terminal_host_runtime.rs | 71 +++++++++++++++++++ 2 files changed, 116 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index d53a6a11b00d..b0592a52213a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4066,6 +4066,51 @@ mod tests { assert_eq!(observed, baseline, "PTY descriptors were created outside the process barrier"); } + #[cfg(unix)] + #[test] + fn local_pty_does_not_inherit_launcher_command() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_LOCAL_LAUNCHER_ENV"; + const LAUNCHER_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; + const TEST_NAME: &str = "surface::tests::local_pty_does_not_inherit_launcher_command"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .env(LAUNCHER_ENV, "npx cmux@1.2.3") + .status() + .unwrap(); + assert!(status.success(), "local PTY environment subprocess failed: {status}"); + return; + } + + let root = std::env::temp_dir() + .join(format!("cmux-local-launcher-env-{}", crate::workspace_registry::new_uuid_v4())); + std::fs::create_dir_all(&root).unwrap(); + let result = root.join("launcher.txt"); + let mux = Mux::new_for_test("local-launcher-env", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec![ + "/bin/sh".into(), + "-c".into(), + "printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \ + \"$CMUX_TUI_TEST_RESULT\"; exec /bin/sleep 60" + .into(), + ]), + extra_env: vec![("CMUX_TUI_TEST_RESULT".into(), result.to_string_lossy().into_owned())], + ..SurfaceOptions::default() + }; + let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while !result.exists() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + let inherited = std::fs::read_to_string(&result).unwrap(); + let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); + let _ = std::fs::remove_dir_all(root); + + assert_eq!(inherited, "unset", "local PTY inherited launcher replay metadata"); + } + #[cfg(unix)] #[test] fn failed_local_pty_initialization_reaps_spawned_child() { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 2aed22b3dc5f..317414bcff34 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4127,6 +4127,77 @@ mod unix { ); } + #[test] + fn host_pty_does_not_inherit_launcher_command() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_HOST_LAUNCHER_ENV"; + const LAUNCHER_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; + const TEST_NAME: &str = + "terminal_host_runtime::unix::tests::host_pty_does_not_inherit_launcher_command"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .env(LAUNCHER_ENV, "npx cmux@1.2.3") + .status() + .unwrap(); + assert!(status.success(), "host PTY environment subprocess failed: {status}"); + return; + } + + let root = std::env::temp_dir().join(format!( + "cmux-host-launcher-env-{}", + crate::workspace_registry::new_uuid_v4() + )); + fs::create_dir_all(&root).unwrap(); + let result = root.join("launcher.txt"); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: root.join("host.sock").to_string_lossy().into_owned(), + record_path: root.join("host.json").to_string_lossy().into_owned(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec![ + "/bin/sh".into(), + "-c".into(), + "printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \ + \"$CMUX_TUI_TEST_RESULT\"; exec /bin/sleep 60" + .into(), + ], + extra_env: vec![( + "CMUX_TUI_TEST_RESULT".into(), + result.to_string_lossy().into_owned(), + )], + default_colors: DefaultColors::default(), + }; + + let host = spawn_host_runtime(&launch, &bootstrapped).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + while !result.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let inherited = fs::read_to_string(&result).unwrap(); + host.request_termination(); + let _ = host.wait_for_child_exit(Duration::from_secs(1)); + let _ = fs::remove_dir_all(root); + + assert_eq!(inherited, "unset", "host PTY inherited launcher replay metadata"); + } + #[test] fn failed_host_pty_initialization_reaps_spawned_child() { let root = std::env::temp_dir().join(format!( From 68e56aaacb9f408ca49a1422070bf2c3c29b9319 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:08:08 -0700 Subject: [PATCH 188/521] fix(tui): scope launcher metadata to daemon --- cmux-tui/crates/cmux-tui-core/src/release.rs | 3 +++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 1 + cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs | 1 + cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 3 +-- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/release.rs b/cmux-tui/crates/cmux-tui-core/src/release.rs index 3ffbd5881c53..d60d7b0888c8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/release.rs +++ b/cmux-tui/crates/cmux-tui-core/src/release.rs @@ -1,5 +1,8 @@ use serde_json::Value; +/// One-hop package launcher metadata used to replay the current invocation. +pub const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; + /// The build identity that must match between a local client and server. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ReleaseIdentity { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index b0592a52213a..3808803e55b5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1761,6 +1761,7 @@ impl Surface { for (k, v) in &opts.extra_env { cmd.env(k, v); } + cmd.env_remove(crate::release::LAUNCHER_COMMAND_ENV); let cwd = opts .cwd .clone() diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 317414bcff34..fc355afc561b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3249,6 +3249,7 @@ mod unix { for (key, value) in &launch.extra_env { command.env(key, value); } + command.env_remove(crate::release::LAUNCHER_COMMAND_ENV); if let Some(cwd) = launch.cwd.as_deref() { command.cwd(cwd); } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 3a0fef6cb764..aed4678c6845 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; -use cmux_tui_core::release::ReleaseIdentity; +use cmux_tui_core::release::{LAUNCHER_COMMAND_ENV, ReleaseIdentity}; use cmux_tui_core::server::{ PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, @@ -57,7 +57,6 @@ const LEGACY_HELPER_CANCEL_MARGIN: Duration = Duration::from_millis(100); const LEGACY_HELPER_REAPER_CAPACITY: usize = 64; #[cfg(unix)] const LEGACY_HELPER_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); -const LAUNCHER_COMMAND_ENV: &str = "CMUX_TUI_LAUNCHER_COMMAND"; const MAX_LAUNCHER_COMMAND_BYTES: usize = 4096; type TransportReader = BufReader>; From f4f0b4a8e00909662b000fe485cf22b9f54e0dba Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:10:35 -0700 Subject: [PATCH 189/521] test(tui): publish launcher env result atomically --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 5 +++-- cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 3808803e55b5..210d1893034e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4093,8 +4093,9 @@ mod tests { command: Some(vec![ "/bin/sh".into(), "-c".into(), - "printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \ - \"$CMUX_TUI_TEST_RESULT\"; exec /bin/sleep 60" + "result=\"$CMUX_TUI_TEST_RESULT\"; \ + printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \"$result.tmp\"; \ + mv \"$result.tmp\" \"$result\"; exec /bin/sleep 60" .into(), ]), extra_env: vec![("CMUX_TUI_TEST_RESULT".into(), result.to_string_lossy().into_owned())], diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index fc355afc561b..b8491db21cf2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4175,8 +4175,9 @@ mod unix { command: vec![ "/bin/sh".into(), "-c".into(), - "printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \ - \"$CMUX_TUI_TEST_RESULT\"; exec /bin/sleep 60" + "result=\"$CMUX_TUI_TEST_RESULT\"; \ + printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \"$result.tmp\"; \ + mv \"$result.tmp\" \"$result\"; exec /bin/sleep 60" .into(), ], extra_env: vec![( From 3a68d0872d7e8243ad3499da249299fcb9b4246e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:04:47 -0700 Subject: [PATCH 190/521] test(tui): expose post-spawn descendant leaks --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 35 ++++++++++++++++++- .../src/terminal_host_runtime.rs | 35 ++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 210d1893034e..b2307bf0ec87 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4120,13 +4120,21 @@ mod tests { .join(format!("cmux-local-spawn-failure-{}", crate::workspace_registry::new_uuid_v4())); std::fs::create_dir_all(&root).unwrap(); let ready = root.join("ready"); + let descendant = root.join("descendant"); let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); let mux = Mux::new_for_test("local-spawn-failure", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ "/bin/sh".into(), "-c".into(), - format!("trap '' HUP TERM; echo $$ > {}; exec /bin/sleep 60", ready.display()), + format!( + "trap '' HUP TERM; \ + (trap '' HUP TERM; exec /bin/sleep 60) & \ + echo $! > {}; \ + echo $$ > {}; exec /bin/sleep 60", + descendant.display(), + ready.display() + ), ]), ..SurfaceOptions::default() }; @@ -4135,6 +4143,8 @@ mod tests { .expect_err("forced local PTY initialization failure unexpectedly succeeded"); assert!(error.to_string().contains("forced post-spawn PTY initialization failure")); let pid = std::fs::read_to_string(&ready).unwrap().trim().parse::().unwrap(); + let descendant_pid = + std::fs::read_to_string(&descendant).unwrap().trim().parse::().unwrap(); let mut status = 0; // SAFETY: the marker contains the exact child PID spawned by this test. let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; @@ -4146,10 +4156,33 @@ mod tests { libc::waitpid(pid, &raw mut status, 0); } } + let descendant_deadline = Instant::now() + Duration::from_secs(2); + let descendant_gone = loop { + // SAFETY: signal zero performs a non-mutating liveness probe. + if unsafe { libc::kill(descendant_pid, 0) } < 0 + && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + break true; + } + if Instant::now() >= descendant_deadline { + break false; + } + std::thread::sleep(Duration::from_millis(10)); + }; + if !descendant_gone { + // SAFETY: the marker names the test's retained-session descendant. + unsafe { + libc::kill(descendant_pid, libc::SIGKILL); + } + } let _ = std::fs::remove_dir_all(root); assert_eq!(result, -1, "failed local PTY initialization left child {pid} unreaped"); assert_eq!(wait_error.raw_os_error(), Some(libc::ECHILD)); + assert!( + descendant_gone, + "failed local PTY initialization left descendant {descendant_pid} running" + ); } #[derive(Clone, Default)] diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index b8491db21cf2..30b703f41767 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4208,6 +4208,7 @@ mod unix { )); fs::create_dir_all(&root).unwrap(); let ready = root.join("ready"); + let descendant = root.join("descendant"); let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); let bootstrap = HostBootstrap { min_version: PROTOCOL_VERSION, @@ -4233,7 +4234,14 @@ mod unix { command: vec![ "/bin/sh".into(), "-c".into(), - format!("trap '' HUP TERM; echo $$ > {}; exec /bin/sleep 60", ready.display()), + format!( + "trap '' HUP TERM; \ + (trap '' HUP TERM; exec /bin/sleep 60) & \ + echo $! > {}; \ + echo $$ > {}; exec /bin/sleep 60", + descendant.display(), + ready.display() + ), ], extra_env: Vec::new(), default_colors: DefaultColors::default(), @@ -4245,6 +4253,8 @@ mod unix { }; assert!(error.to_string().contains("forced post-spawn PTY initialization failure")); let pid = fs::read_to_string(&ready).unwrap().trim().parse::().unwrap(); + let descendant_pid = + fs::read_to_string(&descendant).unwrap().trim().parse::().unwrap(); let mut status = 0; // SAFETY: the marker contains the exact child PID spawned by this test. let result = unsafe { libc::waitpid(pid, &raw mut status, libc::WNOHANG) }; @@ -4256,10 +4266,33 @@ mod unix { libc::waitpid(pid, &raw mut status, 0); } } + let descendant_deadline = Instant::now() + Duration::from_secs(2); + let descendant_gone = loop { + // SAFETY: signal zero performs a non-mutating liveness probe. + if unsafe { libc::kill(descendant_pid, 0) } < 0 + && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + { + break true; + } + if Instant::now() >= descendant_deadline { + break false; + } + thread::sleep(Duration::from_millis(10)); + }; + if !descendant_gone { + // SAFETY: the marker names the test's retained-session descendant. + unsafe { + libc::kill(descendant_pid, libc::SIGKILL); + } + } let _ = fs::remove_dir_all(root); assert_eq!(result, -1, "failed hosted PTY initialization left child {pid} unreaped"); assert_eq!(wait_error.raw_os_error(), Some(libc::ECHILD)); + assert!( + descendant_gone, + "failed hosted PTY initialization left descendant {descendant_pid} running" + ); } #[cfg(target_os = "linux")] From 99f8a5dd5f9eadd09f6af1d4a23a02976e30d966 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:06:16 -0700 Subject: [PATCH 191/521] test(tui): expose concurrent socket reclamation --- cmux-tui/crates/cmux-tui-core/src/server.rs | 156 +++++++++++++++++++- 1 file changed, 151 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 3c3af47f935a..d4959edc6be8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2647,6 +2647,33 @@ pub fn connect_existing_until( } } +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum SocketPublicationStage { + StaleProbeComplete, + Bound, +} + +#[cfg(test)] +type SocketPublicationHook = Arc; + +#[cfg(test)] +static SOCKET_PUBLICATION_HOOK: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +fn set_socket_publication_hook(hook: Option) { + *SOCKET_PUBLICATION_HOOK.get_or_init(Default::default).lock().unwrap() = hook; +} + +#[cfg(test)] +fn socket_publication_hook(path: &Path, stage: SocketPublicationStage) { + let hook = SOCKET_PUBLICATION_HOOK.get_or_init(Default::default).lock().unwrap().clone(); + if let Some(hook) = hook { + hook(path, stage); + } +} + #[cfg(any(not(unix), test))] fn cleanup_without_stable_identity(_path: &Path) { // Windows does not expose a race-free socket publication identity through @@ -2794,14 +2821,20 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result match std::fs::remove_file(&path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - }, + None => { + #[cfg(test)] + socket_publication_hook(&path, SocketPublicationStage::StaleProbeComplete); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } } } let listener = transport::listen(&path)?; + #[cfg(test)] + socket_publication_hook(&path, SocketPublicationStage::Bound); let published = match PublishedSocket::claim(path.clone()) { Ok(published) => published, Err(error) => { @@ -6220,6 +6253,119 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn concurrent_stale_socket_reclamation_has_one_owner() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_CONCURRENT_SOCKET_RECLAMATION"; + const TEST_NAME: &str = "server::tests::concurrent_stale_socket_reclamation_has_one_owner"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "socket reclamation subprocess failed: {status}"); + return; + } + + #[derive(Default)] + struct RaceState { + second_probed: bool, + first_bound: bool, + second_bound: bool, + } + + struct ClearPublicationHook; + impl Drop for ClearPublicationHook { + fn drop(&mut self) { + set_socket_publication_hook(None); + } + } + + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let directory = std::env::temp_dir() + .join(format!("cmux-concurrent-publication-{}-{sequence:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("server.sock"); + std::os::unix::net::UnixListener::bind(&path).unwrap(); + + let state = Arc::new((Mutex::new(RaceState::default()), Condvar::new())); + let hook_path = path.clone(); + let hook_state = state.clone(); + set_socket_publication_hook(Some(Arc::new(move |path, stage| { + if path != hook_path { + return; + } + let thread_name = std::thread::current().name().unwrap_or_default().to_string(); + let (state, changed) = &*hook_state; + let mut state = state.lock().unwrap(); + match (thread_name.as_str(), stage) { + ("socket-startup-second", SocketPublicationStage::StaleProbeComplete) => { + state.second_probed = true; + changed.notify_all(); + let (next, _) = changed + .wait_timeout_while(state, Duration::from_secs(1), |state| { + !state.first_bound + }) + .unwrap(); + drop(next); + } + ("socket-startup-first", SocketPublicationStage::Bound) => { + state.first_bound = true; + changed.notify_all(); + let (next, _) = changed + .wait_timeout_while(state, Duration::from_secs(1), |state| { + !state.second_bound + }) + .unwrap(); + drop(next); + } + ("socket-startup-second", SocketPublicationStage::Bound) => { + state.second_bound = true; + changed.notify_all(); + } + _ => {} + } + }))); + let _clear_hook = ClearPublicationHook; + + let second_path = path.clone(); + let second = std::thread::Builder::new() + .name("socket-startup-second".into()) + .spawn(move || serve_owned(test_mux(), Some(second_path))) + .unwrap(); + { + let (state, changed) = &*state; + let state = state.lock().unwrap(); + let (state, timeout) = changed + .wait_timeout_while(state, Duration::from_secs(2), |state| !state.second_probed) + .unwrap(); + assert!(!timeout.timed_out() && state.second_probed); + } + let first_path = path.clone(); + let first = std::thread::Builder::new() + .name("socket-startup-first".into()) + .spawn(move || serve_owned(test_mux(), Some(first_path))) + .unwrap(); + + let first = first.join().unwrap(); + let second = second.join().unwrap(); + let first_error = first.as_ref().err().map(ToString::to_string); + let second_error = second.as_ref().err().map(ToString::to_string); + let publications: Vec<_> = [first.ok(), second.ok()].into_iter().flatten().collect(); + let owners = publications.len(); + for publication in publications { + publication.cleanup(); + } + let _ = std::fs::remove_file(&path); + std::fs::remove_dir(&directory).unwrap(); + + assert_eq!( + owners, 1, + "concurrent startup created {owners} owners; first={first_error:?}, second={second_error:?}" + ); + } + #[cfg(unix)] #[test] fn default_socket_path_falls_back_for_long_tmpdir() { From 776fbc54b91ca135b67eace9c14abf1fbea7e6f5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:07:45 -0700 Subject: [PATCH 192/521] fix(tui): serialize socket publication --- cmux-tui/crates/cmux-tui-core/src/server.rs | 60 +++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index d4959edc6be8..aed68bbe28ed 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -32,6 +32,7 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant}; use base64::Engine; +use fs4::FileExt; use ghostty_vt::{ Dirty, KeyAction, KeyEncoder, KeyInput, Mods, StyledRun, UnderlineStyle, key_input_from_chord, rows_to_runs, sys, @@ -2674,6 +2675,60 @@ fn socket_publication_hook(path: &Path, stage: SocketPublicationStage) { } } +fn socket_publication_lock_path(path: &Path) -> std::io::Result { + let parent = path.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "server socket has no parent directory", + ) + })?; + let file_name = path.file_name().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "server socket has no file name") + })?; + let mut lock_name = std::ffi::OsString::from("."); + lock_name.push(file_name); + lock_name.push(".publish.lock"); + Ok(parent.join(lock_name)) +} + +struct SocketPublicationLock { + file: std::fs::File, +} + +impl SocketPublicationLock { + fn acquire(path: &Path, deadline: Instant) -> std::io::Result { + let lock_path = socket_publication_lock_path(path)?; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&lock_path)?; + platform::restrict_file(&lock_path)?; + loop { + match FileExt::try_lock(&file) { + Ok(()) => return Ok(Self { file }), + Err(fs4::TryLockError::WouldBlock) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(fs4::TryLockError::WouldBlock) => { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("timed out waiting to publish server socket {}", path.display()), + )); + } + Err(fs4::TryLockError::Error(error)) => return Err(error), + } + } + } +} + +impl Drop for SocketPublicationLock { + fn drop(&mut self) { + let _ = FileExt::unlock(&self.file); + } +} + #[cfg(any(not(unix), test))] fn cleanup_without_stable_identity(_path: &Path) { // Windows does not expose a race-free socket publication identity through @@ -2813,6 +2868,10 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result Date: Wed, 29 Jul 2026 16:11:42 -0700 Subject: [PATCH 193/521] fix(tui): reap failed PTY sessions --- .../cmux-tui-core/src/spawned_pty_child.rs | 114 ++++++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 242 +++++++++--------- .../src/terminal_host_runtime.rs | 6 +- 3 files changed, 241 insertions(+), 121 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 317a9b344304..0fff8997030b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -1,12 +1,40 @@ //! Immediate ownership of a spawned PTY child until its reaper completes. +#[cfg(unix)] +use std::time::Duration; + +#[cfg(unix)] +const FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT: Duration = Duration::from_secs(2); + pub(crate) struct SpawnedPtyChild { child: Option>, + #[cfg(unix)] + reaper: Option, } impl SpawnedPtyChild { pub(crate) fn new(child: Box) -> Self { - Self { child: Some(child) } + Self { + child: Some(child), + #[cfg(unix)] + reaper: None, + } + } + + #[cfg(unix)] + pub(crate) fn with_reaper( + mut self, + reaper: crate::process_session::ReservedChildReaperLease, + ) -> Self { + assert!(self.reaper.replace(reaper).is_none(), "spawned PTY child already has a reaper"); + self + } + + #[cfg(unix)] + pub(crate) fn take_reaper( + &mut self, + ) -> Option { + self.reaper.take() } pub(crate) fn process_id(&self) -> Option { @@ -33,9 +61,91 @@ impl SpawnedPtyChild { } } +#[cfg(unix)] +fn enqueue_failed_spawn_cleanup( + child: Box, + reaper: crate::process_session::ReservedChildReaperLease, + session: libc::pid_t, +) { + // The unreaped Child handle reserves this numeric PID, so this raw signal + // cannot retarget a reused process. Keep the leader waitable while the + // shared reaper enumerates and removes every other session member. + loop { + // SAFETY: `session` is the process ID exposed by the solely owned, + // unreaped child handle above. + if unsafe { libc::kill(session, libc::SIGKILL) } == 0 { + break; + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + if error.raw_os_error() == Some(libc::ESRCH) { + break; + } + break; + } + + let (finished_sender, finished_receiver) = std::sync::mpsc::sync_channel(1); + let mut finished_sender = Some(finished_sender); + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, + FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT, + move || match crate::process_session::observe_child_without_reaping(session, true)? { + crate::process_session::ChildWaitState::Running => Ok(false), + crate::process_session::ChildWaitState::Waitable => Ok(true), + crate::process_session::ChildWaitState::OwnershipLost => { + Err(std::io::Error::from(std::io::ErrorKind::NotFound)) + } + }, + || true, + || true, + move |cleanup_succeeded| { + if !cleanup_succeeded { + return crate::process_session::NaturalReapFinish::Pending; + } + let Some(owned_child) = child.as_mut() else { + return crate::process_session::NaturalReapFinish::Complete; + }; + match owned_child.wait() { + Ok(_) => { + child.take(); + if let Some(sender) = finished_sender.take() { + let _ = sender.send(()); + } + crate::process_session::NaturalReapFinish::Complete + } + Err(_) => crate::process_session::NaturalReapFinish::Failed, + } + }, + ); + let _ = finished_receiver + .recv_timeout(FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT + Duration::from_millis(250)); +} + impl Drop for SpawnedPtyChild { fn drop(&mut self) { - let Some(mut child) = self.child.take() else { return }; + let Some(child) = self.child.take() else { return }; + #[cfg(unix)] + let child = match self.reaper.take() { + Some(reaper) => { + let Some(session) = + child.process_id().and_then(|pid| libc::pid_t::try_from(pid).ok()) + else { + drop(reaper); + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + return; + }; + enqueue_failed_spawn_cleanup(child, reaper, session); + return; + } + None => child, + }; + let mut child = child; let _ = child.kill(); let _ = child.wait(); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index b2307bf0ec87..1dd055ef52ac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -931,133 +931,134 @@ impl LocalPtyProcess { }) } + #[cfg(unix)] fn spawn_reaper( self: &Arc, - child: crate::spawned_pty_child::SpawnedPtyChild, - reaper: LocalChildReaperLease, + mut child: crate::spawned_pty_child::SpawnedPtyChild, ) -> std::io::Result<()> { - #[cfg(unix)] - { - let process = self.clone(); - let Some(session) = process.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + let process = self.clone(); + let Some(session) = process.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "native PTY child did not expose a process ID", + )); + }; + let child_identity = match crate::process_session::StableProcessHandle::capture(session) { + Ok(Some(identity)) => identity, + Ok(None) => { return Err(std::io::Error::new( - std::io::ErrorKind::Unsupported, - "native PTY child did not expose a process ID", + std::io::ErrorKind::NotFound, + "native PTY child exited before its process identity was captured", )); - }; - let child_identity = match crate::process_session::StableProcessHandle::capture(session) - { - Ok(Some(identity)) => identity, - Ok(None) => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "native PTY child exited before its process identity was captured", - )); - } - Err(error) => return Err(error), - }; - let observe_process = process.clone(); - let cleanup_process = process.clone(); - let prepare_process = process.clone(); - let reap_process = process; - let mut child = Some(child); - crate::process_session::enqueue_reserved_session_leader( - reaper, - session, - NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, - move || { - let _signal = observe_process.child_signal_lock.lock().unwrap(); - let state = observe_process.observe_child_wait_state_locked(session)?; - Ok(state != crate::process_session::ChildWaitState::Running) - }, - move || { - crate::process_session::reserved_child_needs_cleanup( - crate::process_session::ReservedChildReap { - signal_lock: &cleanup_process.child_signal_lock, - pty_drained: &cleanup_process.pty_drained, - termination_started: &cleanup_process.termination_started, - cleanup_complete: &cleanup_process.group_escalation_complete, - child_reaped: &cleanup_process.child_reaped, - wait_ownership_lost: &cleanup_process.child_wait_ownership_lost, - }, - ) - }, - move || prepare_process.prepare_natural_cleanup(), - move |cleanup_succeeded| { - if reap_process.child_wait_ownership_lost.load(Ordering::Acquire) { - let _signal = reap_process.child_signal_lock.lock().unwrap(); - if !reap_process.pty_drained.load(Ordering::Acquire) { - return crate::process_session::NaturalReapFinish::Pending; - } - match child_identity.matches_current() { - Ok(false) => {} - Ok(true) | Err(_) => { - return crate::process_session::NaturalReapFinish::Failed; - } - } - child - .take() - .expect("reserved child releases lost wait ownership once") - .abandon_wait_ownership(); - reap_process.child_reaped.store(true, Ordering::Release); - drop(_signal); - *reap_process.exited.0.lock().unwrap() = true; - reap_process.exited.1.notify_all(); - return crate::process_session::NaturalReapFinish::Complete; - } - let done = crate::process_session::poll_reserved_session_leader( - crate::process_session::ReservedChildReap { - signal_lock: &reap_process.child_signal_lock, - pty_drained: &reap_process.pty_drained, - termination_started: &reap_process.termination_started, - cleanup_complete: &reap_process.group_escalation_complete, - child_reaped: &reap_process.child_reaped, - wait_ownership_lost: &reap_process.child_wait_ownership_lost, - }, - cleanup_succeeded, - || { - let mut child = child.take().expect("reserved child is reaped once"); - let _ = child.wait(); - }, - ); - if done { - *reap_process.exited.0.lock().unwrap() = true; - reap_process.exited.1.notify_all(); + } + Err(error) => return Err(error), + }; + let reaper = child.take_reaper().ok_or_else(|| { + std::io::Error::other("spawned PTY child has no reserved session reaper") + })?; + let observe_process = process.clone(); + let cleanup_process = process.clone(); + let prepare_process = process.clone(); + let reap_process = process; + let mut child = Some(child); + crate::process_session::enqueue_reserved_session_leader( + reaper, + session, + NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, + move || { + let _signal = observe_process.child_signal_lock.lock().unwrap(); + let state = observe_process.observe_child_wait_state_locked(session)?; + Ok(state != crate::process_session::ChildWaitState::Running) + }, + move || { + crate::process_session::reserved_child_needs_cleanup( + crate::process_session::ReservedChildReap { + signal_lock: &cleanup_process.child_signal_lock, + pty_drained: &cleanup_process.pty_drained, + termination_started: &cleanup_process.termination_started, + cleanup_complete: &cleanup_process.group_escalation_complete, + child_reaped: &cleanup_process.child_reaped, + wait_ownership_lost: &cleanup_process.child_wait_ownership_lost, + }, + ) + }, + move || prepare_process.prepare_natural_cleanup(), + move |cleanup_succeeded| { + if reap_process.child_wait_ownership_lost.load(Ordering::Acquire) { + let _signal = reap_process.child_signal_lock.lock().unwrap(); + if !reap_process.pty_drained.load(Ordering::Acquire) { + return crate::process_session::NaturalReapFinish::Pending; } - if done { - crate::process_session::NaturalReapFinish::Complete - } else { - crate::process_session::NaturalReapFinish::Pending + match child_identity.matches_current() { + Ok(false) => {} + Ok(true) | Err(_) => { + return crate::process_session::NaturalReapFinish::Failed; + } } - }, - ); - Ok(()) - } + child + .take() + .expect("reserved child releases lost wait ownership once") + .abandon_wait_ownership(); + reap_process.child_reaped.store(true, Ordering::Release); + drop(_signal); + *reap_process.exited.0.lock().unwrap() = true; + reap_process.exited.1.notify_all(); + return crate::process_session::NaturalReapFinish::Complete; + } + let done = crate::process_session::poll_reserved_session_leader( + crate::process_session::ReservedChildReap { + signal_lock: &reap_process.child_signal_lock, + pty_drained: &reap_process.pty_drained, + termination_started: &reap_process.termination_started, + cleanup_complete: &reap_process.group_escalation_complete, + child_reaped: &reap_process.child_reaped, + wait_ownership_lost: &reap_process.child_wait_ownership_lost, + }, + cleanup_succeeded, + || { + let mut child = child.take().expect("reserved child is reaped once"); + let _ = child.wait(); + }, + ); + if done { + *reap_process.exited.0.lock().unwrap() = true; + reap_process.exited.1.notify_all(); + } + if done { + crate::process_session::NaturalReapFinish::Complete + } else { + crate::process_session::NaturalReapFinish::Pending + } + }, + ); + Ok(()) + } - #[cfg(not(unix))] - { - let process = self.clone(); - let (child_sender, child_receiver) = - sync_channel::(1); - let observer = - std::thread::Builder::new().name("surface-child".into()).spawn(move || { - let Ok(mut child) = child_receiver.recv() else { return }; - let _ = child.wait(); - *process.exited.0.lock().unwrap() = true; - process.exited.1.notify_all(); - }); - if let Err(error) = observer { - return Err(error); - } - if child_sender.send(child).is_err() { - return Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "surface child observer exited before accepting ownership", - )); - } - let _ = reaper; - Ok(()) + #[cfg(not(unix))] + fn spawn_reaper( + self: &Arc, + child: crate::spawned_pty_child::SpawnedPtyChild, + _reaper: LocalChildReaperLease, + ) -> std::io::Result<()> { + let process = self.clone(); + let (child_sender, child_receiver) = + sync_channel::(1); + let observer = std::thread::Builder::new().name("surface-child".into()).spawn(move || { + let Ok(mut child) = child_receiver.recv() else { return }; + let _ = child.wait(); + *process.exited.0.lock().unwrap() = true; + process.exited.1.notify_all(); + }); + if let Err(error) = observer { + return Err(error); + } + if child_sender.send(child).is_err() { + return Err(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "surface child observer exited before accepting ownership", + )); } + Ok(()) } fn request_hangup(&self) -> bool { @@ -1771,6 +1772,8 @@ impl Surface { } let child = crate::spawned_pty_child::SpawnedPtyChild::new(pty.slave.spawn_command(cmd)?); + #[cfg(unix)] + let child = child.with_reaper(reaper); drop(process_creation); let pid = child.process_id(); drop(pty.slave); @@ -1861,6 +1864,9 @@ impl Surface { frame_requests, })); + #[cfg(unix)] + process.spawn_reaper(child)?; + #[cfg(not(unix))] process.spawn_reaper(child, reaper)?; if let Err(error) = spawn_frame_producer(&surface, frame_rx) { let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 30b703f41767..8fc44beeef4b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3254,7 +3254,7 @@ mod unix { command.cwd(cwd); } let child = pty.slave.spawn_command(command)?; - let child = crate::spawned_pty_child::SpawnedPtyChild::new(child); + let child = crate::spawned_pty_child::SpawnedPtyChild::new(child).with_reaper(reaper); #[cfg(test)] crate::process_session::fail_after_pty_spawn_for_test()?; drop(process_creation); @@ -3425,6 +3425,10 @@ mod unix { let cleanup_host = shared.clone(); let prepare_host = shared.clone(); let reap_host = shared.clone(); + let mut child = child; + let reaper = child + .take_reaper() + .expect("terminal-host PTY child retains its reserved session reaper"); let mut child = Some(child); crate::process_session::enqueue_reserved_session_leader( reaper, From f73281f1bd7f91cc8c8e44e0e900a2942511b151 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:13:54 -0700 Subject: [PATCH 194/521] test(tui): clean socket publication fixtures --- cmux-tui/crates/cmux-tui/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 44a59584d1ef..7bdec39aa63b 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1638,7 +1638,7 @@ mod tests { drop(client); drop(existing); cmux_tui_core::server::cleanup(&socket_path); - std::fs::remove_dir(socket_dir).unwrap(); + std::fs::remove_dir_all(socket_dir).unwrap(); } #[cfg(unix)] From cb21eac17d256b3d75116bc214ec9a6c55b8feab Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:16:05 -0700 Subject: [PATCH 195/521] fix(tui): retry failed-spawn leader signals --- .../cmux-tui-core/src/spawned_pty_child.rs | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 0fff8997030b..273b3a970cae 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -6,6 +6,25 @@ use std::time::Duration; #[cfg(unix)] const FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(unix)] +fn signal_owned_child_for_cleanup(session: libc::pid_t) -> std::io::Result<()> { + loop { + // SAFETY: callers retain the sole unreaped Child handle for `session`, + // so the numeric PID cannot be reused before the final wait. + if unsafe { libc::kill(session, libc::SIGKILL) } == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + if error.raw_os_error() == Some(libc::ESRCH) { + return Ok(()); + } + return Err(error); + } +} + pub(crate) struct SpawnedPtyChild { child: Option>, #[cfg(unix)] @@ -70,21 +89,7 @@ fn enqueue_failed_spawn_cleanup( // The unreaped Child handle reserves this numeric PID, so this raw signal // cannot retarget a reused process. Keep the leader waitable while the // shared reaper enumerates and removes every other session member. - loop { - // SAFETY: `session` is the process ID exposed by the solely owned, - // unreaped child handle above. - if unsafe { libc::kill(session, libc::SIGKILL) } == 0 { - break; - } - let error = std::io::Error::last_os_error(); - if error.kind() == std::io::ErrorKind::Interrupted { - continue; - } - if error.raw_os_error() == Some(libc::ESRCH) { - break; - } - break; - } + let _ = signal_owned_child_for_cleanup(session); let (finished_sender, finished_receiver) = std::sync::mpsc::sync_channel(1); let mut finished_sender = Some(finished_sender); @@ -94,7 +99,10 @@ fn enqueue_failed_spawn_cleanup( session, FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT, move || match crate::process_session::observe_child_without_reaping(session, true)? { - crate::process_session::ChildWaitState::Running => Ok(false), + crate::process_session::ChildWaitState::Running => { + signal_owned_child_for_cleanup(session)?; + Ok(false) + } crate::process_session::ChildWaitState::Waitable => Ok(true), crate::process_session::ChildWaitState::OwnershipLost => { Err(std::io::Error::from(std::io::ErrorKind::NotFound)) From f415ab711c7287fb03ece9081d86f8e819955a8f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:59:07 -0700 Subject: [PATCH 196/521] test(tui): expose replay payload disclosure --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index aed4678c6845..44ebd07fe1c0 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1788,15 +1788,18 @@ mod tests { } #[test] - fn replay_command_preserves_quoted_and_empty_arguments() { + fn replay_command_omits_payload_arguments_from_guidance() { assert_eq!( replay_command_from("cmux-tui".to_string(), std::iter::empty::<&str>()), "cmux-tui" ); - assert_eq!( - replay_command_from("cmux-tui".to_string(), ["--session", "two words", "", "it's"]), - "cmux-tui --session 'two words' '' 'it'\"'\"'s'" + let secret = "cmux-review-secret-do-not-print"; + let replay = replay_command_from( + "cmux-tui".to_string(), + ["--socket", "/tmp/cmux.sock", "send", "--text", secret], ); + assert_eq!(replay, "cmux-tui"); + assert!(!replay.contains(secret)); } #[cfg(unix)] From 37d0063e6f3b21cb3b412a51256c1d48378cb64b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:00:52 -0700 Subject: [PATCH 197/521] fix(tui): omit replay payloads from diagnostics --- cmux-tui/crates/cmux-tui/src/localization.rs | 8 +++---- .../crates/cmux-tui/src/server_lifecycle.rs | 21 ++++++++----------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 0da812fecfa5..d4c57708c403 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -530,8 +530,8 @@ static ENGLISH: Catalog = Catalog { stop_to_use: "Stop the old server to use this cmux-tui version.", stopping_exits_panes: "Stopping exits pane processes.", restart_before_command: "Run `", - restart_between_commands: "`, then run `", - restart_after_command: "` again.", + restart_between_commands: "`, then rerun the previous command using `", + restart_after_command: "`.", stopped: "Stopped the cmux-tui server.", missing_action: "server needs an action: status or stop", unknown_action: "unknown server action", @@ -815,8 +815,8 @@ static JAPANESE: Catalog = Catalog { stop_to_use: "この cmux-tui バージョンを使うには、古いサーバーを停止してください。", stopping_exits_panes: "停止するとペインのプロセスが終了します。", restart_before_command: "`", - restart_between_commands: "` を実行してから、`", - restart_after_command: "` をもう一度実行してください。", + restart_between_commands: "` を実行してから、以前のコマンドを `", + restart_after_command: "` で再実行してください。", stopped: "cmux-tui サーバーを停止しました。", missing_action: "server には status または stop の操作が必要です", unknown_action: "不明な server 操作", diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 44ebd07fe1c0..a7c3033834e3 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1373,15 +1373,15 @@ fn current_replay_command() -> String { replay_command_from(current_launcher_command(), std::env::args_os().skip(1)) } -fn replay_command_from(mut command: String, arguments: I) -> String +fn replay_command_from(command: String, arguments: I) -> String where I: IntoIterator, S: AsRef, { - for argument in arguments { - command.push(' '); - command.push_str(&shell_quote(Path::new(argument.as_ref()))); - } + // Arguments can contain terminal input, child commands, or credentials. + // Shell history already retains them for the user; diagnostics identify only + // the launcher that should rerun the previous command. + drop(arguments); command } @@ -1765,13 +1765,10 @@ mod tests { message.contains(&format!("{launcher} server stop --socket '/tmp/test socket'")), "{message}" ); - let replay_arguments = std::env::args_os().skip(1).collect::>(); - let mut replay = launcher; - for argument in replay_arguments { - replay.push(' '); - replay.push_str(&shell_quote(Path::new(&argument))); - } - assert!(message.contains(&format!("then run `{replay}` again")), "{message}"); + assert!( + message.contains(&format!("then rerun the previous command using `{launcher}`.")), + "{message}" + ); assert!(message.contains("Stopping exits pane processes.")); } From b0afc58e6938b8ee4799f602b633c5e7ca8cb6e8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:20:05 -0700 Subject: [PATCH 198/521] test(tui): expose non-UTF-8 proc names --- .../cmux-tui-core/src/process_session.rs | 24 +++++++++++++++++++ .../src/server_lifecycle/legacy_process.rs | 22 +++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index ef60dd43cb44..f60fc7fd37a8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1824,6 +1824,30 @@ mod tests { assert_eq!(stable_signals, 1, "session cleanup did not use the stable process handle"); } + #[cfg(target_os = "linux")] + #[test] + fn linux_process_metadata_accepts_a_non_utf8_process_name() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_NON_UTF8_PROCESS_NAME"; + const TEST_NAME: &str = + "process_session::tests::linux_process_metadata_accepts_a_non_utf8_process_name"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "non-UTF-8 proc metadata subprocess failed: {status}"); + return; + } + + let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + std::fs::write(format!("/proc/self/task/{pid}/comm"), b"cmux-\xff").unwrap(); + + let metadata = + linux_process_metadata(pid).expect("a valid non-UTF-8 proc name was rejected"); + assert!(metadata.is_some()); + } + #[cfg(target_os = "linux")] #[test] fn shutdown_treats_zombie_members_as_drained_before_reserved_parent() { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 20ccba6c8a3a..22493a8026d1 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -1084,6 +1084,28 @@ mod tests { assert!(result.unwrap()); } + #[cfg(target_os = "linux")] + #[test] + fn linux_process_snapshot_accepts_a_non_utf8_process_name() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_LEGACY_NON_UTF8_PROCESS_NAME"; + const TEST_NAME: &str = "server_lifecycle::legacy_process::tests::linux_process_snapshot_accepts_a_non_utf8_process_name"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "non-UTF-8 legacy proc subprocess failed: {status}"); + return; + } + + let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + std::fs::write(format!("/proc/self/task/{pid}/comm"), b"cmux-\xff").unwrap(); + + let snapshot = process_snapshot(pid).expect("a valid non-UTF-8 proc name was rejected"); + assert!(snapshot.is_some()); + } + #[cfg(target_os = "linux")] #[test] fn linux_child_enumeration_includes_children_spawned_by_worker_threads() { From aa478d252b6cec3aa6168cbf9751e7bd36424fe6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:20:19 -0700 Subject: [PATCH 199/521] test(tui): reject unknown session membership --- .../cmux-tui-core/src/process_session.rs | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index f60fc7fd37a8..5ab2ab949a57 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2001,30 +2001,15 @@ mod tests { #[cfg(target_os = "linux")] #[test] - fn session_scan_skips_inaccessible_foreign_processes_but_not_leaders() { + fn session_scan_does_not_assume_an_inaccessible_pid_is_foreign() { let sessions = HashSet::from([41]); - let snapshot = scan_sessions_with( - &sessions, - None, - || Ok(vec![73, 41]), - |pid| { - if pid == 73 { - Err(io::Error::from_raw_os_error(libc::EACCES)) - } else { - Ok(Some(41)) - } - }, - ) - .expect("an inaccessible foreign process aborted the session scan"); - assert_eq!(snapshot.get(&41), Some(&vec![41])); - let error = scan_sessions_with( &sessions, None, - || Ok(vec![41]), + || Ok(vec![73]), |_| Err(io::Error::from_raw_os_error(libc::EACCES)), ) - .expect_err("an inaccessible session leader was silently skipped"); + .expect_err("unknown session membership was silently treated as foreign"); assert_eq!(error.raw_os_error(), Some(libc::EACCES)); } From 27c25c32368b3d214ef4e0976b0b8ab3e546e320 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:23:19 -0700 Subject: [PATCH 200/521] fix(tui): harden Linux process snapshots --- .../cmux-tui-core/src/process_session.rs | 136 ++++++++++++++---- .../src/server_lifecycle/legacy_process.rs | 41 ++++-- 2 files changed, 133 insertions(+), 44 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 5ab2ab949a57..8c3390415d06 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -940,8 +940,7 @@ fn still_member(pid: libc::pid_t, session: libc::pid_t) -> io::Result { Ok(active_process_session(pid)? == Some(session)) } -#[cfg(not(target_os = "linux"))] -fn active_process_session(pid: libc::pid_t) -> io::Result> { +fn process_session_id(pid: libc::pid_t) -> io::Result> { // SAFETY: getsid only queries process metadata. let current_session = unsafe { libc::getsid(pid) }; if current_session >= 0 { @@ -951,6 +950,11 @@ fn active_process_session(pid: libc::pid_t) -> io::Result> { if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) } } +#[cfg(not(target_os = "linux"))] +fn active_process_session(pid: libc::pid_t) -> io::Result> { + process_session_id(pid) +} + #[cfg(target_os = "linux")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct LinuxProcessMetadata { @@ -960,7 +964,7 @@ struct LinuxProcessMetadata { #[cfg(target_os = "linux")] fn linux_process_metadata(pid: libc::pid_t) -> io::Result> { - let stat = match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + let stat = match std::fs::read(format!("/proc/{pid}/stat")) { Ok(stat) => stat, Err(error) if error.kind() == io::ErrorKind::NotFound @@ -970,22 +974,36 @@ fn linux_process_metadata(pid: libc::pid_t) -> io::Result return Err(error), }; - let (pid_text, fields) = stat - .split_once(" (") - .and_then(|(pid_text, remainder)| { - remainder.rsplit_once(") ").map(|(_, fields)| (pid_text, fields)) - }) - .ok_or_else(|| io::Error::other("invalid process stat record"))?; - if pid_text.parse::().ok() != Some(pid) { - return Err(io::Error::other("process metadata id mismatch")); - } - let fields = fields.split_whitespace().collect::>(); + let fields = linux_process_stat_fields(pid, &stat)?; let state = fields.first().copied().ok_or_else(|| io::Error::other("missing process state"))?; let started_at = fields .get(19) + .and_then(|value| std::str::from_utf8(value).ok()) .and_then(|value| value.parse::().ok()) .ok_or_else(|| io::Error::other("invalid process birth identity"))?; - Ok(Some(LinuxProcessMetadata { started_at, zombie: state == "Z" })) + Ok(Some(LinuxProcessMetadata { started_at, zombie: state == b"Z" })) +} + +#[cfg(target_os = "linux")] +fn linux_process_stat_fields(pid: libc::pid_t, stat: &[u8]) -> io::Result> { + let name_start = stat + .windows(2) + .position(|window| window == b" (") + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + let name_end = stat + .windows(2) + .rposition(|window| window == b") ") + .filter(|name_end| *name_end > name_start) + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + let pid_text = std::str::from_utf8(&stat[..name_start]) + .map_err(|_| io::Error::other("invalid process metadata id"))?; + if pid_text.parse::().ok() != Some(pid) { + return Err(io::Error::other("process metadata id mismatch")); + } + Ok(stat[name_end + 2..] + .split(|byte| byte.is_ascii_whitespace()) + .filter(|field| !field.is_empty()) + .collect()) } #[cfg(target_os = "linux")] @@ -995,12 +1013,7 @@ fn active_process_session(pid: libc::pid_t) -> io::Result> { return Ok(None); } - // SAFETY: getsid only queries process metadata. - let current_session = unsafe { libc::getsid(pid) }; - if current_session < 0 { - let error = io::Error::last_os_error(); - return if error.raw_os_error() == Some(libc::ESRCH) { Ok(None) } else { Err(error) }; - } + let Some(current_session) = process_session_id(pid)? else { return Ok(None) }; // Bracket the reusable PID query with the kernel process birth identity. // Zombies cannot execute or fork, so shutdown may treat them as drained @@ -1012,6 +1025,36 @@ fn active_process_session(pid: libc::pid_t) -> io::Result> { Ok(Some(current_session)) } +#[cfg(target_os = "linux")] +fn tracked_process_session( + pid: libc::pid_t, + sessions: &HashSet, +) -> io::Result> { + // Query the kernel session first so unrelated PIDs never require procfs + // access on hidepid mounts. Candidate members still pass through the + // birth-identity bracket, and any failure there aborts the snapshot. + tracked_process_session_with(pid, sessions, process_session_id, active_process_session) +} + +#[cfg(target_os = "linux")] +fn tracked_process_session_with( + pid: libc::pid_t, + sessions: &HashSet, + mut query_session: Q, + mut verify_session: V, +) -> io::Result> +where + Q: FnMut(libc::pid_t) -> io::Result>, + V: FnMut(libc::pid_t) -> io::Result>, +{ + let Some(candidate_session) = query_session(pid)? else { return Ok(None) }; + if !sessions.contains(&candidate_session) { + return Ok(None); + } + let Some(verified_session) = verify_session(pid)? else { return Ok(None) }; + Ok(sessions.contains(&verified_session).then_some(verified_session)) +} + fn stable_process_in_session( pid: libc::pid_t, session: libc::pid_t, @@ -1384,7 +1427,19 @@ fn scan_sessions( sessions: &HashSet, deadline: Option, ) -> io::Result>> { - scan_sessions_with(sessions, deadline, || all_process_ids(deadline), active_process_session) + #[cfg(target_os = "linux")] + { + scan_sessions_with( + sessions, + deadline, + || all_process_ids(deadline), + |pid| tracked_process_session(pid, sessions), + ) + } + #[cfg(not(target_os = "linux"))] + { + scan_sessions_with(sessions, deadline, || all_process_ids(deadline), active_process_session) + } } fn scan_sessions_with( @@ -1409,16 +1464,7 @@ where if pid <= 1 { continue; } - let current_session = match process_session(pid) { - Ok(Some(current_session)) => current_session, - Ok(None) => continue, - Err(error) - if error.kind() == io::ErrorKind::PermissionDenied && !sessions.contains(&pid) => - { - continue; - } - Err(error) => return Err(error), - }; + let Some(current_session) = process_session(pid)? else { continue }; ensure_before_deadline(deadline)?; if let Some(session_members) = members.get_mut(¤t_session) { session_members.push(pid); @@ -2013,6 +2059,34 @@ mod tests { assert_eq!(error.raw_os_error(), Some(libc::EACCES)); } + #[cfg(target_os = "linux")] + #[test] + fn tracked_session_filter_reads_proc_metadata_only_for_candidates() { + let sessions = HashSet::from([41]); + let metadata_queried = std::cell::Cell::new(false); + let foreign = tracked_process_session_with( + 73, + &sessions, + |_| Ok(Some(99)), + |_| { + metadata_queried.set(true); + Err(io::Error::from_raw_os_error(libc::EACCES)) + }, + ) + .unwrap(); + assert_eq!(foreign, None); + assert!(!metadata_queried.get()); + + let error = tracked_process_session_with( + 73, + &sessions, + |_| Ok(Some(41)), + |_| Err(io::Error::from_raw_os_error(libc::EACCES)), + ) + .expect_err("candidate session metadata failure was ignored"); + assert_eq!(error.raw_os_error(), Some(libc::EACCES)); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn concurrent_shutdown_refreshes_share_one_new_snapshot_generation() { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 22493a8026d1..f699ec3dd090 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -818,7 +818,7 @@ fn direct_child_pids(parent: libc::pid_t) -> io::Result> { #[cfg(target_os = "linux")] fn process_snapshot(pid: libc::pid_t) -> io::Result> { let path = format!("/proc/{pid}/stat"); - let stat = match std::fs::read_to_string(path) { + let stat = match std::fs::read(path) { Ok(stat) => stat, Err(error) if error.kind() == io::ErrorKind::NotFound @@ -828,35 +828,50 @@ fn process_snapshot(pid: libc::pid_t) -> io::Result> { } Err(error) => return Err(error), }; - let (pid_text, remainder) = stat - .split_once(" (") - .and_then(|(pid_text, remainder)| { - remainder.rsplit_once(") ").map(|(_, fields)| (pid_text, fields)) - }) - .ok_or_else(|| io::Error::other("invalid process stat record"))?; - if pid_text.parse::().ok() != Some(pid) { - return Err(io::Error::other("process metadata id mismatch")); - } - let fields = remainder.split_whitespace().collect::>(); + let fields = linux_process_stat_fields(pid, &stat)?; let state = fields.first().copied().ok_or_else(|| io::Error::other("missing process state"))?; - if state == "Z" { + if state == b"Z" { return Ok(None); } let parent = fields .get(1) + .and_then(|value| std::str::from_utf8(value).ok()) .and_then(|value| value.parse::().ok()) .ok_or_else(|| io::Error::other("invalid parent process id"))?; let started_at = fields .get(19) + .and_then(|value| std::str::from_utf8(value).ok()) .and_then(|value| value.parse::().ok()) .ok_or_else(|| io::Error::other("invalid process birth identity"))?; Ok(Some(ProcessSnapshot { identity: ProcessIdentity { pid, started_at }, parent, - stopped: matches!(state, "T" | "t"), + stopped: matches!(state, b"T" | b"t"), })) } +#[cfg(target_os = "linux")] +fn linux_process_stat_fields(pid: libc::pid_t, stat: &[u8]) -> io::Result> { + let name_start = stat + .windows(2) + .position(|window| window == b" (") + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + let name_end = stat + .windows(2) + .rposition(|window| window == b") ") + .filter(|name_end| *name_end > name_start) + .ok_or_else(|| io::Error::other("invalid process stat record"))?; + let pid_text = std::str::from_utf8(&stat[..name_start]) + .map_err(|_| io::Error::other("invalid process metadata id"))?; + if pid_text.parse::().ok() != Some(pid) { + return Err(io::Error::other("process metadata id mismatch")); + } + Ok(stat[name_end + 2..] + .split(|byte| byte.is_ascii_whitespace()) + .filter(|field| !field.is_empty()) + .collect()) +} + #[cfg(target_os = "linux")] fn direct_child_pids(parent: libc::pid_t) -> io::Result> { let task_root = format!("/proc/{parent}/task"); From 73137c3c69cdf5f61fb6589cc15d3a73dba41e89 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:24:14 -0700 Subject: [PATCH 201/521] test(ios): align TestFlight schedule guard --- tests/test_ios_testflight_main_push_filter.py | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tests/test_ios_testflight_main_push_filter.py b/tests/test_ios_testflight_main_push_filter.py index 7bcf5be9ffaf..683d8f1a05a5 100644 --- a/tests/test_ios_testflight_main_push_filter.py +++ b/tests/test_ios_testflight_main_push_filter.py @@ -19,10 +19,7 @@ "scripts/validate-xcframework-archive.py", ".github/workflows/ios-testflight.yml", ) -DEMO_VARIANT_GUARD = ( - "github.event_name == 'workflow_dispatch' " - "&& github.event.inputs.variant == 'demo'" -) +DEMO_VARIANT_GUARD = "needs.decide.outputs.variant == 'demo'" MARKETING_OVERRIDE_GUARD = "github.event.inputs.marketing_version_override != ''" @@ -82,34 +79,43 @@ def mapping_keys(text: str, indent: int) -> tuple[str, ...]: return tuple(keys) -def sequence_values(text: str, key: str, indent: int) -> tuple[str, ...]: - marker = f"{' ' * indent}{key}:\n" - assert marker in text, f"missing {key} sequence" +def javascript_string_array(text: str, declaration: str) -> tuple[str, ...]: + marker = f"const {declaration} = [\n" + assert marker in text, f"missing {declaration} JavaScript array" lines = text[text.index(marker) + len(marker) :].splitlines() - item_prefix = f"{' ' * (indent + 2)}- " values = [] for line in lines: - if not line.strip() or line.lstrip().startswith("#"): - continue - line_indent = len(line) - len(line.lstrip()) - if line_indent <= indent: + stripped = line.strip() + if stripped == "];": break - assert line.startswith(item_prefix), f"invalid {key} item: {line}" - parsed = shlex.split(line.removeprefix(item_prefix), comments=True) - assert len(parsed) == 1, f"invalid {key} value: {line}" + if not stripped or stripped.startswith("//"): + continue + parsed = shlex.split(stripped.removesuffix(","), comments=True) + assert len(parsed) == 1, f"invalid {declaration} value: {line}" values.append(parsed[0]) + else: + raise AssertionError(f"unterminated {declaration} JavaScript array") return tuple(values) -def test_main_push_triggers_only_for_ios_affecting_paths() -> None: +def test_scheduled_runs_filter_ios_changes_with_the_shared_path_contract() -> None: text = workflow_text() triggers = trigger_block(text) - push = mapping_block(triggers, "push", indent=2) + schedule = mapping_block(triggers, "schedule", indent=2) + crons = tuple( + shlex.split(line.strip().removeprefix("- cron:").strip())[0] + for line in schedule.splitlines() + if line.strip().startswith("- cron:") + ) - assert mapping_keys(triggers, indent=2) == ("push", "workflow_dispatch") - assert sequence_values(push, "branches", indent=4) == ("main",) - assert sequence_values(push, "paths", indent=4) == IOS_PATHS - assert "context.eventName === 'push' || context.eventName === 'workflow_dispatch'" in text + assert mapping_keys(triggers, indent=2) == ("schedule", "workflow_dispatch") + assert crons == ("7 * * * *", "37 5,17 * * *") + assert javascript_string_array(text, "iosRelevantPaths") == tuple( + path.removesuffix("**") for path in IOS_PATHS + ) + assert "context.eventName === 'schedule'" in text + assert "github.rest.repos.compareCommits" in text + assert "files.some((file) => touchesIOS(file.filename))" in text def test_mapping_keys_normalizes_quoted_yaml_keys() -> None: @@ -139,16 +145,18 @@ def test_testflight_notes_use_the_same_ios_path_contract() -> None: assert notes_paths == expected_notes_paths -def test_main_push_runs_are_preserved_and_uploaded_in_order() -> None: +def test_scheduled_runs_are_preserved_and_uploaded_in_order() -> None: text = workflow_text() + assert "group: ios-testflight-${{ github.run_id }}" in text assert ( "group: ios-testflight-${{ github.event_name == 'push' && github.sha || github.run_id }}" - in text + not in text ) assert "group: ios-testflight-${{ github.ref_name }}" not in text assert "cancel-in-progress: false" in text assert "Number(run.id) < currentRunId" in text + assert "['push', 'schedule', 'workflow_dispatch'].includes(run.event)" in text assert "uploadJob.status !== 'completed'" in text assert "could not inspect earlier TestFlight runs; retrying" in text assert "ios-testflight-assignment-state-complete" not in text @@ -219,9 +227,9 @@ def test_automatic_lane_stays_on_cmux_internal_identity() -> None: if __name__ == "__main__": - test_main_push_triggers_only_for_ios_affecting_paths() + test_scheduled_runs_filter_ios_changes_with_the_shared_path_contract() test_mapping_keys_normalizes_quoted_yaml_keys() test_testflight_notes_use_the_same_ios_path_contract() - test_main_push_runs_are_preserved_and_uploaded_in_order() + test_scheduled_runs_are_preserved_and_uploaded_in_order() test_automatic_lane_stays_on_cmux_internal_identity() - print("all iOS TestFlight main-push filter tests passed") + print("all iOS TestFlight schedule filter tests passed") From a7cc284e323b36d7068c9a8dc9c5baaa16712235 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:40:21 -0700 Subject: [PATCH 202/521] Revert "test(ios): align TestFlight schedule guard" This reverts commit 73137c3c69cdf5f61fb6589cc15d3a73dba41e89. --- tests/test_ios_testflight_main_push_filter.py | 60 ++++++++----------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/tests/test_ios_testflight_main_push_filter.py b/tests/test_ios_testflight_main_push_filter.py index 683d8f1a05a5..7bcf5be9ffaf 100644 --- a/tests/test_ios_testflight_main_push_filter.py +++ b/tests/test_ios_testflight_main_push_filter.py @@ -19,7 +19,10 @@ "scripts/validate-xcframework-archive.py", ".github/workflows/ios-testflight.yml", ) -DEMO_VARIANT_GUARD = "needs.decide.outputs.variant == 'demo'" +DEMO_VARIANT_GUARD = ( + "github.event_name == 'workflow_dispatch' " + "&& github.event.inputs.variant == 'demo'" +) MARKETING_OVERRIDE_GUARD = "github.event.inputs.marketing_version_override != ''" @@ -79,43 +82,34 @@ def mapping_keys(text: str, indent: int) -> tuple[str, ...]: return tuple(keys) -def javascript_string_array(text: str, declaration: str) -> tuple[str, ...]: - marker = f"const {declaration} = [\n" - assert marker in text, f"missing {declaration} JavaScript array" +def sequence_values(text: str, key: str, indent: int) -> tuple[str, ...]: + marker = f"{' ' * indent}{key}:\n" + assert marker in text, f"missing {key} sequence" lines = text[text.index(marker) + len(marker) :].splitlines() + item_prefix = f"{' ' * (indent + 2)}- " values = [] for line in lines: - stripped = line.strip() - if stripped == "];": - break - if not stripped or stripped.startswith("//"): + if not line.strip() or line.lstrip().startswith("#"): continue - parsed = shlex.split(stripped.removesuffix(","), comments=True) - assert len(parsed) == 1, f"invalid {declaration} value: {line}" + line_indent = len(line) - len(line.lstrip()) + if line_indent <= indent: + break + assert line.startswith(item_prefix), f"invalid {key} item: {line}" + parsed = shlex.split(line.removeprefix(item_prefix), comments=True) + assert len(parsed) == 1, f"invalid {key} value: {line}" values.append(parsed[0]) - else: - raise AssertionError(f"unterminated {declaration} JavaScript array") return tuple(values) -def test_scheduled_runs_filter_ios_changes_with_the_shared_path_contract() -> None: +def test_main_push_triggers_only_for_ios_affecting_paths() -> None: text = workflow_text() triggers = trigger_block(text) - schedule = mapping_block(triggers, "schedule", indent=2) - crons = tuple( - shlex.split(line.strip().removeprefix("- cron:").strip())[0] - for line in schedule.splitlines() - if line.strip().startswith("- cron:") - ) + push = mapping_block(triggers, "push", indent=2) - assert mapping_keys(triggers, indent=2) == ("schedule", "workflow_dispatch") - assert crons == ("7 * * * *", "37 5,17 * * *") - assert javascript_string_array(text, "iosRelevantPaths") == tuple( - path.removesuffix("**") for path in IOS_PATHS - ) - assert "context.eventName === 'schedule'" in text - assert "github.rest.repos.compareCommits" in text - assert "files.some((file) => touchesIOS(file.filename))" in text + assert mapping_keys(triggers, indent=2) == ("push", "workflow_dispatch") + assert sequence_values(push, "branches", indent=4) == ("main",) + assert sequence_values(push, "paths", indent=4) == IOS_PATHS + assert "context.eventName === 'push' || context.eventName === 'workflow_dispatch'" in text def test_mapping_keys_normalizes_quoted_yaml_keys() -> None: @@ -145,18 +139,16 @@ def test_testflight_notes_use_the_same_ios_path_contract() -> None: assert notes_paths == expected_notes_paths -def test_scheduled_runs_are_preserved_and_uploaded_in_order() -> None: +def test_main_push_runs_are_preserved_and_uploaded_in_order() -> None: text = workflow_text() - assert "group: ios-testflight-${{ github.run_id }}" in text assert ( "group: ios-testflight-${{ github.event_name == 'push' && github.sha || github.run_id }}" - not in text + in text ) assert "group: ios-testflight-${{ github.ref_name }}" not in text assert "cancel-in-progress: false" in text assert "Number(run.id) < currentRunId" in text - assert "['push', 'schedule', 'workflow_dispatch'].includes(run.event)" in text assert "uploadJob.status !== 'completed'" in text assert "could not inspect earlier TestFlight runs; retrying" in text assert "ios-testflight-assignment-state-complete" not in text @@ -227,9 +219,9 @@ def test_automatic_lane_stays_on_cmux_internal_identity() -> None: if __name__ == "__main__": - test_scheduled_runs_filter_ios_changes_with_the_shared_path_contract() + test_main_push_triggers_only_for_ios_affecting_paths() test_mapping_keys_normalizes_quoted_yaml_keys() test_testflight_notes_use_the_same_ios_path_contract() - test_scheduled_runs_are_preserved_and_uploaded_in_order() + test_main_push_runs_are_preserved_and_uploaded_in_order() test_automatic_lane_stays_on_cmux_internal_identity() - print("all iOS TestFlight schedule filter tests passed") + print("all iOS TestFlight main-push filter tests passed") From 118096c45b21fbf95d7fdc6d1d37272559364593 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:00:55 -0700 Subject: [PATCH 203/521] test(tui): expose unbounded shutdown cleanup --- cmux-tui/crates/cmux-tui/src/main.rs | 56 +++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 7bdec39aa63b..e1102861a213 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -968,6 +968,9 @@ struct ServerShutdownCleanup { mux: Arc, state: std::sync::Mutex, changed: std::sync::Condvar, + #[cfg(test)] + shutdown_attempt: + std::sync::Mutex anyhow::Result<()> + Send + Sync + 'static>>>, } impl ServerShutdownCleanup { @@ -982,7 +985,28 @@ impl ServerShutdownCleanup { retry_delay: SERVER_SHUTDOWN_RETRY_INITIAL, }), changed: std::sync::Condvar::new(), + #[cfg(test)] + shutdown_attempt: std::sync::Mutex::new(None), + } + } + + fn attempt_shutdown(&self) -> anyhow::Result<()> { + #[cfg(test)] + if let Some(attempt) = + self.shutdown_attempt.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() + { + return attempt(); } + self.mux.shutdown() + } + + #[cfg(test)] + fn set_shutdown_attempt_for_test( + &self, + attempt: impl Fn() -> anyhow::Result<()> + Send + Sync + 'static, + ) { + *self.shutdown_attempt.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(Arc::new(attempt)); } fn run_until_complete(&self) { @@ -1009,7 +1033,7 @@ impl ServerShutdownCleanup { drop(state); let result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.mux.shutdown())); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_shutdown())); state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); state.running = false; match result { @@ -1610,6 +1634,36 @@ fn usage_exit(msg: &str) -> ! { mod tests { use super::*; + fn assert_shutdown_attempt_returns( + attempt: impl Fn() -> anyhow::Result<()> + Send + Sync + 'static, + ) { + let mux = Mux::new("shutdown-cleanup-test", SurfaceOptions::default()); + let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); + cleanup.set_shutdown_attempt_for_test(attempt); + let worker_cleanup = cleanup.clone(); + let (returned, observed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + worker_cleanup.run_until_complete(); + let _ = returned.send(()); + }); + + let result = observed.recv_timeout(std::time::Duration::from_millis(350)); + if result.is_ok() { + worker.join().unwrap(); + } + assert!(result.is_ok(), "a terminal shutdown result kept the server caller blocked"); + } + + #[test] + fn permanent_shutdown_failure_returns_to_caller() { + assert_shutdown_attempt_returns(|| anyhow::bail!("forced permanent shutdown failure")); + } + + #[test] + fn shutdown_panic_returns_to_caller() { + assert_shutdown_attempt_returns(|| panic!("forced shutdown panic")); + } + fn args(values: &[&str]) -> Args { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } From d331f49cbe18af6d816887811ec1dff4c69657c9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:04:41 -0700 Subject: [PATCH 204/521] fix(tui): bound server shutdown cleanup --- cmux-tui/crates/cmux-tui/src/main.rs | 185 ++++++++++++++------------- 1 file changed, 98 insertions(+), 87 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index e1102861a213..fdfb85e43f06 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -65,8 +65,6 @@ const PROVIDER_WORKSPACE_AUTHORITY_ENV: &str = "CMUX_PROVIDER_WORKSPACE_AUTHORIT // Once core cleanup owns no external processes, give the interactive driver // one final second to unwind before releasing the socket and exiting. const SERVER_SHUTDOWN_EXIT_GRACE: std::time::Duration = std::time::Duration::from_secs(1); -const SERVER_SHUTDOWN_RETRY_INITIAL: std::time::Duration = std::time::Duration::from_millis(100); -const SERVER_SHUTDOWN_RETRY_MAX: std::time::Duration = std::time::Duration::from_secs(5); #[cfg(target_os = "linux")] unsafe extern "C" { @@ -956,34 +954,29 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result anyhow::Result<()> + Send + Sync + 'static>; + struct ServerShutdownCleanup { mux: Arc, state: std::sync::Mutex, changed: std::sync::Condvar, #[cfg(test)] - shutdown_attempt: - std::sync::Mutex anyhow::Result<()> + Send + Sync + 'static>>>, + shutdown_attempt: std::sync::Mutex>, } impl ServerShutdownCleanup { fn new(mux: Arc) -> Self { Self { mux, - state: std::sync::Mutex::new(ServerShutdownCleanupState { - running: false, - complete: false, - blocked: false, - next_attempt: std::time::Instant::now(), - retry_delay: SERVER_SHUTDOWN_RETRY_INITIAL, - }), + state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending), changed: std::sync::Condvar::new(), #[cfg(test)] shutdown_attempt: std::sync::Mutex::new(None), @@ -1009,56 +1002,66 @@ impl ServerShutdownCleanup { Some(Arc::new(attempt)); } - fn run_until_complete(&self) { + fn run_until_complete(&self) -> anyhow::Result<()> { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); loop { - if state.complete { - return; - } - if state.blocked || state.running { - state = self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); - continue; - } - let now = std::time::Instant::now(); - if state.next_attempt > now { - let wait = state.next_attempt.saturating_duration_since(now); - let waited = self - .changed - .wait_timeout(state, wait) - .unwrap_or_else(std::sync::PoisonError::into_inner); - state = waited.0; - continue; - } - state.running = true; - drop(state); - - let result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_shutdown())); - state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - state.running = false; - match result { - Ok(Ok(())) => { - state.complete = true; - self.changed.notify_all(); - return; + match &*state { + ServerShutdownCleanupState::Complete => return Ok(()), + ServerShutdownCleanupState::Failed(message) => { + return Err(anyhow::anyhow!(message.clone())); } - Ok(Err(_)) => { - state.next_attempt = std::time::Instant::now() + state.retry_delay; - state.retry_delay = (state.retry_delay * 2).min(SERVER_SHUTDOWN_RETRY_MAX); + ServerShutdownCleanupState::Running => { + state = + self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); } - Err(_) => { - // A panic means cleanup invariants are unknown. Keep the - // socket and process ownership reserved without invoking - // the damaged path again. - state.blocked = true; + ServerShutdownCleanupState::Pending => { + *state = ServerShutdownCleanupState::Running; + break; } } - self.changed.notify_all(); } + drop(state); + + // Mux::shutdown owns the bounded retry policy. Share its one terminal + // result across the frontend and watchdog instead of retrying it + // indefinitely at this layer. + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_shutdown())); + let outcome = match result { + Ok(Ok(())) => ServerShutdownCleanupState::Complete, + Ok(Err(error)) => ServerShutdownCleanupState::Failed(format!( + "{}: {error:#}", + localization::catalog().server.shutdown_cleanup_incomplete + )), + Err(_) => ServerShutdownCleanupState::Failed( + localization::catalog().server.shutdown_cleanup_incomplete.to_string(), + ), + }; + let return_value = match &outcome { + ServerShutdownCleanupState::Complete => Ok(()), + ServerShutdownCleanupState::Failed(message) => Err(anyhow::anyhow!(message.clone())), + ServerShutdownCleanupState::Pending | ServerShutdownCleanupState::Running => { + unreachable!("shutdown attempt produced no terminal result") + } + }; + state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *state = outcome; + self.changed.notify_all(); + return_value } fn is_complete(&self) -> bool { - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner).complete + matches!( + *self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + ServerShutdownCleanupState::Complete + ) + } + + fn is_terminal(&self) -> bool { + matches!( + *self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner), + ServerShutdownCleanupState::Complete | ServerShutdownCleanupState::Failed(_) + ) } } @@ -1092,10 +1095,13 @@ impl ServerProcessShutdownGuard { } // A blocked frontend cannot own process cleanup. Complete the - // same bounded cleanup path that run_server joins below, and - // retain the socket indefinitely if cleanup cannot prove that - // every external owner was released. - worker_cleanup.run_until_complete(); + // same bounded cleanup path that run_server joins below. A + // terminal cleanup failure retains the socket and leaves the + // process alive instead of claiming a clean exit. + if let Err(error) = worker_cleanup.run_until_complete() { + eprintln!("cmux-tui: {error:#}"); + return; + } if matches!( completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) @@ -1131,8 +1137,8 @@ impl ServerProcessShutdownGuard { Ok(()) } - fn shutdown(&self) { - self.cleanup.run_until_complete(); + fn shutdown(&self) -> anyhow::Result<()> { + self.cleanup.run_until_complete() } fn complete(mut self) { @@ -1140,11 +1146,13 @@ impl ServerProcessShutdownGuard { } fn finish(&mut self) { - if !self.cleanup.is_complete() { + if !self.cleanup.is_terminal() { return; } let Some(completion) = self.completion.take() else { return }; - cleanup_published_socket(&self.published_socket); + if self.cleanup.is_complete() { + cleanup_published_socket(&self.published_socket); + } let _ = completion.send(()); self.shutdown_watch.cancel(); if let Some(worker) = self.worker.take() { @@ -1169,6 +1177,19 @@ fn cleanup_published_socket( } } +fn finish_server_process( + server_process: ServerProcessShutdownGuard, + result: anyhow::Result<()>, +) -> anyhow::Result<()> { + let shutdown_result = server_process.shutdown(); + server_process.complete(); + match (result, shutdown_result) { + (result, Ok(())) => result, + (Ok(()), Err(shutdown_error)) => Err(shutdown_error), + (Err(error), Err(shutdown_error)) => Err(error.context(shutdown_error.to_string())), + } +} + fn server_shutdown_exit_grace() -> std::time::Duration { #[cfg(debug_assertions)] if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS") @@ -1288,9 +1309,7 @@ fn run_server( { Ok(server) => server, Err(error) => { - server_process.shutdown(); - server_process.complete(); - return Err(error.into()); + return finish_server_process(server_process, Err(error.into())); } }; let websocket_server = match ws_addr { @@ -1300,9 +1319,10 @@ fn run_server( Err(error) => { #[cfg(target_os = "linux")] drop(_provider_management); - server_process.shutdown(); - server_process.complete(); - return Err(anyhow::anyhow!("invalid WebSocket address {addr:?}: {error}")); + return finish_server_process( + server_process, + Err(anyhow::anyhow!("invalid WebSocket address {addr:?}: {error}")), + ); } }; match cmux_tui_core::server::serve_websocket( @@ -1315,9 +1335,7 @@ fn run_server( Err(error) => { #[cfg(target_os = "linux")] drop(_provider_management); - server_process.shutdown(); - server_process.complete(); - return Err(error); + return finish_server_process(server_process, Err(error)); } } } @@ -1333,18 +1351,14 @@ fn run_server( drop(websocket_server); #[cfg(target_os = "linux")] drop(_provider_management); - server_process.shutdown(); - server_process.complete(); - return Err(error); + return finish_server_process(server_process, Err(error)); } }; if let Err(error) = server_process.publish_socket(published_socket) { drop(websocket_server); #[cfg(target_os = "linux")] drop(_provider_management); - server_process.shutdown(); - server_process.complete(); - return Err(error.into()); + return finish_server_process(server_process, Err(error.into())); } #[cfg(debug_assertions)] @@ -1366,9 +1380,7 @@ fn run_server( drop(websocket_server); #[cfg(target_os = "linux")] drop(_provider_management); - server_process.shutdown(); - server_process.complete(); - result + finish_server_process(server_process, result) } fn connect_existing_local_session( @@ -1640,10 +1652,9 @@ mod tests { let mux = Mux::new("shutdown-cleanup-test", SurfaceOptions::default()); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); cleanup.set_shutdown_attempt_for_test(attempt); - let worker_cleanup = cleanup.clone(); let (returned, observed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || { - worker_cleanup.run_until_complete(); + let _ = cleanup.run_until_complete(); let _ = returned.send(()); }); @@ -1684,7 +1695,7 @@ mod tests { let guard = ServerProcessShutdownGuard::start(&mux).unwrap(); assert!(cmux_tui_core::server::serve(mux, Some(socket_path.clone())).is_err()); - guard.shutdown(); + guard.shutdown().unwrap(); guard.complete(); let client = UnixStream::connect(&socket_path) @@ -1709,7 +1720,7 @@ mod tests { let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); - cleanup.state.lock().unwrap().complete = true; + *cleanup.state.lock().unwrap() = ServerShutdownCleanupState::Complete; let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || completed.recv().unwrap()); let guard = ServerProcessShutdownGuard { From 69f1ab661830e2482f7e6e4d6030735503dda77a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:05:41 -0700 Subject: [PATCH 205/521] test(tui): expose concurrent socket cleanup --- cmux-tui/crates/cmux-tui-core/src/server.rs | 34 +++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index b102522b6149..ca0414081d52 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -6607,6 +6607,40 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn published_socket_cleanup_waits_for_publication_lock() { + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let directory = std::env::temp_dir() + .join(format!("cmux-serialized-cleanup-{}-{sequence:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("server.sock"); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let published = PublishedSocket::claim(path.clone()).unwrap(); + let publication_lock = + SocketPublicationLock::acquire(&path, Instant::now() + Duration::from_secs(1)).unwrap(); + let (finished, observed) = std::sync::mpsc::channel(); + let cleanup = std::thread::spawn(move || { + published.cleanup(); + let _ = finished.send(()); + }); + + let completed_while_locked = observed.recv_timeout(Duration::from_millis(100)).is_ok(); + drop(publication_lock); + if !completed_while_locked { + observed + .recv_timeout(Duration::from_secs(2)) + .expect("socket cleanup did not resume after the publication lock was released"); + } + cleanup.join().unwrap(); + drop(listener); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(socket_publication_lock_path(&path).unwrap()); + std::fs::remove_dir(&directory).unwrap(); + + assert!(!completed_while_locked, "socket cleanup bypassed the concurrent publication lock"); + } + #[cfg(unix)] #[test] fn concurrent_stale_socket_reclamation_has_one_owner() { From f528a8ff70331d6da6c0fe8ff1ecb4cfe9431394 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:07:14 -0700 Subject: [PATCH 206/521] fix(tui): serialize socket cleanup --- cmux-tui/crates/cmux-tui-core/src/server.rs | 30 +++++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index ca0414081d52..05f147b6e68d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2883,6 +2883,17 @@ impl PublishedSocket { } } + fn cleanup_while_publication_locked(self) { + #[cfg(unix)] + { + let _ = self.cleanup_unix_locked(); + } + #[cfg(not(unix))] + { + cleanup_without_stable_identity(&self.path); + } + } + #[cfg(unix)] fn matches(&self, path: &Path) -> std::io::Result { use std::os::unix::fs::{FileTypeExt, MetadataExt}; @@ -2895,6 +2906,15 @@ impl PublishedSocket { #[cfg(unix)] fn cleanup_unix(&self) -> std::io::Result<()> { + let _publication_lock = SocketPublicationLock::acquire( + &self.path, + Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT + LOCAL_SOCKET_CONNECT_TIMEOUT, + )?; + self.cleanup_unix_locked() + } + + #[cfg(unix)] + fn cleanup_unix_locked(&self) -> std::io::Result<()> { match self.matches(&self.path) { Ok(true) => {} Ok(false) => return Ok(()), @@ -2939,9 +2959,9 @@ impl PublishedSocket { removed } Ok(false) | Err(_) => { - // A replacement won the race between the public identity - // check and rename. Restore it without overwriting any newer - // publication that may already occupy the public path. + // An uncoordinated replacement won the race between the + // public identity check and rename. Restore it without + // overwriting any newer publication at the public path. std::fs::hard_link(&quarantined, &self.path)?; std::fs::remove_file(&quarantined)?; std::fs::remove_dir(&quarantine_dir) @@ -2998,7 +3018,7 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result, path: Option) -> anyhow::Result Date: Wed, 29 Jul 2026 21:58:46 -0700 Subject: [PATCH 207/521] test(tui): preserve macOS 14 package support --- cmux-tui/dist/scripts/test_package_pypi.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 8f44d4f0c6d9..f8cc2343f3b6 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -65,7 +65,7 @@ def test_stable_install_path_is_reused_directly(self) -> None: class DarwinCompatibilityTests(unittest.TestCase): - def test_wheels_require_the_first_safe_macos_wheel_floor(self) -> None: + def test_wheels_preserve_the_supported_macos_floor(self) -> None: tags = { target.rust_target: target.platform_tags for target in TARGETS @@ -75,14 +75,14 @@ def test_wheels_require_the_first_safe_macos_wheel_floor(self) -> None: self.assertEqual( tags, { - "aarch64-apple-darwin": ("macosx_15_0_arm64",), - "x86_64-apple-darwin": ("macosx_15_0_x86_64",), + "aarch64-apple-darwin": ("macosx_14_0_arm64",), + "x86_64-apple-darwin": ("macosx_14_0_x86_64",), }, ) def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: compatibility_file = Path(__file__).resolve().parents[1] / "macos-deployment-target.txt" - self.assertEqual(compatibility_file.read_text().strip(), "15.0") + self.assertEqual(compatibility_file.read_text().strip(), "14.0") workflow = ( Path(__file__).resolve().parents[3] From f333e229e8fdf88692853ef3f29b372a3afd2efa Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:59:46 -0700 Subject: [PATCH 208/521] test(tui): retain failed shutdown owners --- cmux-tui/crates/cmux-tui/src/main.rs | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index fdfb85e43f06..d00bde22bc1e 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1675,6 +1675,45 @@ mod tests { assert_shutdown_attempt_returns(|| panic!("forced shutdown panic")); } + #[test] + fn failed_shutdown_retains_server_until_explicit_retry() { + let mux = Mux::new("shutdown-retry-test", SurfaceOptions::default()); + let server_process = ServerProcessShutdownGuard::start(&mux).unwrap(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + server_process.cleanup.set_shutdown_attempt_for_test({ + let attempts = attempts.clone(); + move || { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + anyhow::bail!("forced first shutdown failure"); + } + Ok(()) + } + }); + let (returned, observed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = finish_server_process(server_process, Ok(())); + let _ = returned.send(result); + }); + + let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { + std::thread::yield_now(); + } + assert_eq!(attempts.load(Ordering::SeqCst), 1); + assert!( + observed.recv_timeout(std::time::Duration::from_millis(75)).is_err(), + "a failed cleanup released the server's process owners" + ); + + mux.request_shutdown(); + observed + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("an explicit shutdown retry did not release the server") + .unwrap(); + worker.join().unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } + fn args(values: &[&str]) -> Args { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } From bcdf822db69814fa49636ce1dab04608444ab19e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:00:04 -0700 Subject: [PATCH 209/521] fix(tui): restore macOS 14 package floor --- cmux-tui/dist/macos-deployment-target.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/dist/macos-deployment-target.txt b/cmux-tui/dist/macos-deployment-target.txt index fe6b09a7d14c..ed9d00f9a5b8 100644 --- a/cmux-tui/dist/macos-deployment-target.txt +++ b/cmux-tui/dist/macos-deployment-target.txt @@ -1 +1 @@ -15.0 +14.0 From 0ea30a2cc40f84cef66ab03398c0083535719a74 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:05:17 -0700 Subject: [PATCH 210/521] fix(tui): retain degraded shutdown owners --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 ++ cmux-tui/crates/cmux-tui/src/main.rs | 225 ++++++++++++++++------- 2 files changed, 178 insertions(+), 67 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2a14a8eed5e5..d86d9d1e2594 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -5357,6 +5357,18 @@ impl Mux { watch } + /// Observe the next shutdown request, even after the server has already + /// accepted an earlier request. Degraded process cleanup uses this to + /// require explicit operator intent before each bounded retry. + pub fn watch_next_shutdown_request(&self) -> ShutdownRequestWatch { + let inner = Arc::new(ShutdownRequestWatchInner::default()); + let watch = ShutdownRequestWatch { inner: inner.clone() }; + let mut watchers = self.shutdown_request_watchers.lock().unwrap(); + watchers.retain(|watcher| watcher.strong_count() != 0); + watchers.push(Arc::downgrade(&inner)); + watch + } + pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { let pending = self.shutdown_owners.len(); let state = self.shutdown_owner_reconciler.state.lock().unwrap(); @@ -14871,6 +14883,14 @@ mod tests { waiter.join().unwrap(); assert!(mux.watch_shutdown_request().wait()); + let next = mux.watch_next_shutdown_request(); + let (next_tx, next_rx) = std::sync::mpsc::sync_channel(1); + let next_waiter = std::thread::spawn(move || next_tx.send(next.wait()).unwrap()); + assert!(next_rx.recv_timeout(Duration::from_millis(50)).is_err()); + mux.request_shutdown(); + assert!(next_rx.recv_timeout(Duration::from_secs(1)).unwrap()); + next_waiter.join().unwrap(); + let fresh = test_mux(); let watch = fresh.watch_shutdown_request(); let cancel = watch.clone(); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index d00bde22bc1e..5645f0566e11 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -955,10 +955,18 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result) -> Self { Self { mux, - state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending), + state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending(0)), changed: std::sync::Condvar::new(), #[cfg(test)] shutdown_attempt: std::sync::Mutex::new(None), @@ -1002,65 +1010,155 @@ impl ServerShutdownCleanup { Some(Arc::new(attempt)); } - fn run_until_complete(&self) -> anyhow::Result<()> { + fn run_once(&self) -> ServerShutdownCleanupOutcome { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - loop { + let (attempt, retry) = loop { match &*state { - ServerShutdownCleanupState::Complete => return Ok(()), - ServerShutdownCleanupState::Failed(message) => { - return Err(anyhow::anyhow!(message.clone())); + ServerShutdownCleanupState::Complete => { + return ServerShutdownCleanupOutcome::Complete; } - ServerShutdownCleanupState::Running => { + ServerShutdownCleanupState::Degraded { attempt, message, retry } => { + return ServerShutdownCleanupOutcome::Degraded { + attempt: *attempt, + message: message.clone(), + retry: retry.clone(), + }; + } + ServerShutdownCleanupState::Poisoned(message) => { + return ServerShutdownCleanupOutcome::Poisoned(message.clone()); + } + ServerShutdownCleanupState::Running(_) => { state = self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); } - ServerShutdownCleanupState::Pending => { - *state = ServerShutdownCleanupState::Running; - break; + ServerShutdownCleanupState::Pending(attempt) => { + let attempt = *attempt; + let retry = self.mux.watch_next_shutdown_request(); + *state = ServerShutdownCleanupState::Running(attempt); + break (attempt, retry); } } - } + }; drop(state); - // Mux::shutdown owns the bounded retry policy. Share its one terminal - // result across the frontend and watchdog instead of retrying it - // indefinitely at this layer. + // Mux::shutdown owns one bounded attempt. A failure retains the mux, + // socket, and exact process owners until another explicit request + // authorizes the next attempt. let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_shutdown())); let outcome = match result { - Ok(Ok(())) => ServerShutdownCleanupState::Complete, - Ok(Err(error)) => ServerShutdownCleanupState::Failed(format!( - "{}: {error:#}", - localization::catalog().server.shutdown_cleanup_incomplete - )), - Err(_) => ServerShutdownCleanupState::Failed( + Ok(Ok(())) => ServerShutdownCleanupOutcome::Complete, + Ok(Err(error)) => ServerShutdownCleanupOutcome::Degraded { + attempt, + message: format!( + "{}: {error:#}", + localization::catalog().server.shutdown_cleanup_incomplete + ), + retry, + }, + Err(_) => ServerShutdownCleanupOutcome::Poisoned( localization::catalog().server.shutdown_cleanup_incomplete.to_string(), ), }; - let return_value = match &outcome { - ServerShutdownCleanupState::Complete => Ok(()), - ServerShutdownCleanupState::Failed(message) => Err(anyhow::anyhow!(message.clone())), - ServerShutdownCleanupState::Pending | ServerShutdownCleanupState::Running => { - unreachable!("shutdown attempt produced no terminal result") + let next_state = match &outcome { + ServerShutdownCleanupOutcome::Complete => ServerShutdownCleanupState::Complete, + ServerShutdownCleanupOutcome::Degraded { attempt, message, retry } => { + ServerShutdownCleanupState::Degraded { + attempt: *attempt, + message: message.clone(), + retry: retry.clone(), + } + } + ServerShutdownCleanupOutcome::Poisoned(message) => { + ServerShutdownCleanupState::Poisoned(message.clone()) } }; state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - *state = outcome; + debug_assert!( + matches!(*state, ServerShutdownCleanupState::Running(current) if current == attempt) + ); + *state = next_state; self.changed.notify_all(); - return_value + outcome } - fn is_complete(&self) -> bool { - matches!( - *self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner), - ServerShutdownCleanupState::Complete - ) + #[cfg(test)] + fn run_until_complete(&self) -> anyhow::Result<()> { + match self.run_once() { + ServerShutdownCleanupOutcome::Complete => Ok(()), + ServerShutdownCleanupOutcome::Degraded { message, .. } + | ServerShutdownCleanupOutcome::Poisoned(message) => Err(anyhow::anyhow!(message)), + } + } + + fn prepare_retry(&self, attempt: u64) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if matches!( + &*state, + ServerShutdownCleanupState::Degraded { + attempt: current, + .. + } if *current == attempt + ) { + *state = ServerShutdownCleanupState::Pending(attempt.saturating_add(1)); + self.changed.notify_all(); + } } - fn is_terminal(&self) -> bool { + fn wait_for_complete(&self) { + let mut reported_attempt = None; + let mut reported_poison = false; + loop { + match self.run_once() { + ServerShutdownCleanupOutcome::Complete => return, + ServerShutdownCleanupOutcome::Degraded { attempt, message, retry } => { + if reported_attempt != Some(attempt) { + eprintln!("cmux-tui: {message}; retry with `cmux-tui server stop`"); + reported_attempt = Some(attempt); + } + if retry.wait() { + self.prepare_retry(attempt); + } else { + let state = + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _state = self + .changed + .wait_while(state, |state| { + matches!( + state, + ServerShutdownCleanupState::Degraded { + attempt: current, + .. + } if *current == attempt + ) + }) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + ServerShutdownCleanupOutcome::Poisoned(message) => { + if !reported_poison { + eprintln!( + "cmux-tui: {message}; retaining the server because cleanup panicked" + ); + reported_poison = true; + } + let state = + self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let _state = self + .changed + .wait_while(state, |state| { + matches!(state, ServerShutdownCleanupState::Poisoned(_)) + }) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + } + } + } + + fn is_complete(&self) -> bool { matches!( *self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner), - ServerShutdownCleanupState::Complete | ServerShutdownCleanupState::Failed(_) + ServerShutdownCleanupState::Complete ) } } @@ -1095,13 +1193,10 @@ impl ServerProcessShutdownGuard { } // A blocked frontend cannot own process cleanup. Complete the - // same bounded cleanup path that run_server joins below. A - // terminal cleanup failure retains the socket and leaves the - // process alive instead of claiming a clean exit. - if let Err(error) = worker_cleanup.run_until_complete() { - eprintln!("cmux-tui: {error:#}"); - return; - } + // same bounded cleanup path that run_server joins below. + // Failed attempts keep serving the control socket and require + // another explicit stop request before retrying. + worker_cleanup.wait_for_complete(); if matches!( completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) @@ -1137,16 +1232,21 @@ impl ServerProcessShutdownGuard { Ok(()) } + #[cfg(test)] fn shutdown(&self) -> anyhow::Result<()> { self.cleanup.run_until_complete() } + fn wait_for_shutdown(&self) { + self.cleanup.wait_for_complete(); + } + fn complete(mut self) { self.finish(); } fn finish(&mut self) { - if !self.cleanup.is_terminal() { + if !self.cleanup.is_complete() { return; } let Some(completion) = self.completion.take() else { return }; @@ -1181,13 +1281,9 @@ fn finish_server_process( server_process: ServerProcessShutdownGuard, result: anyhow::Result<()>, ) -> anyhow::Result<()> { - let shutdown_result = server_process.shutdown(); + server_process.wait_for_shutdown(); server_process.complete(); - match (result, shutdown_result) { - (result, Ok(())) => result, - (Ok(()), Err(shutdown_error)) => Err(shutdown_error), - (Err(error), Err(shutdown_error)) => Err(error.context(shutdown_error.to_string())), - } + result } fn server_shutdown_exit_grace() -> std::time::Duration { @@ -1302,6 +1398,17 @@ fn run_server( mux.set_default_colors(config.terminal_defaults); mux.configure_sidebar_plugin(config.sidebar.plugin.clone()); let server_process = ServerProcessShutdownGuard::start(&mux)?; + let published_socket = + match cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())) { + Ok(published) => published, + Err(error) => { + return finish_server_process(server_process, Err(error)); + } + }; + if let Err(error) = server_process.publish_socket(published_socket) { + return finish_server_process(server_process, Err(error.into())); + } + #[cfg(target_os = "linux")] let _provider_management = match provider_management_listener .map(|listener| cmux_tui_core::provider_management::serve(listener, mux.clone())) @@ -1344,22 +1451,6 @@ fn run_server( if let Some(server) = &websocket_server { eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } - let published_socket = - match cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())) { - Ok(published) => published, - Err(error) => { - drop(websocket_server); - #[cfg(target_os = "linux")] - drop(_provider_management); - return finish_server_process(server_process, Err(error)); - } - }; - if let Err(error) = server_process.publish_socket(published_socket) { - drop(websocket_server); - #[cfg(target_os = "linux")] - drop(_provider_management); - return finish_server_process(server_process, Err(error.into())); - } #[cfg(debug_assertions)] if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { From 43a80766a7a9cec4bc917b4cca1b27660c043341 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:31:08 -0700 Subject: [PATCH 211/521] test(tui): expose Python 3.8 launcher breakage --- cmux-tui/dist/scripts/test_package_pypi.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index f8cc2343f3b6..72833a651403 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -20,6 +20,12 @@ def launcher_command(version: str, argv0: str) -> str: class LauncherCommandTests(unittest.TestCase): + def test_launcher_remains_compatible_with_python_3_8(self) -> None: + files = wheel_bytes("1.2.3", "test-platform", b"binary") + source = next(data for name, data, _ in files if name == "cmux_tui/_main.py") + + self.assertNotIn(b".removeprefix(", source) + def test_uv_cache_restarts_the_same_version(self) -> None: self.assertEqual( launcher_command( From 6870129b926632e83f973f80cae909632ce18758 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:32:16 -0700 Subject: [PATCH 212/521] test(tui): expose unreachable startup retry --- cmux-tui/crates/cmux-tui/src/main.rs | 46 ++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 5645f0566e11..f767215f2bff 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1770,6 +1770,14 @@ mod tests { fn failed_shutdown_retains_server_until_explicit_retry() { let mux = Mux::new("shutdown-retry-test", SurfaceOptions::default()); let server_process = ServerProcessShutdownGuard::start(&mux).unwrap(); + let suffix = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let socket_dir = std::env::temp_dir() + .join(format!("cmux-shutdown-retry-{}-{suffix:x}", std::process::id())); + let socket_path = socket_dir.join("server.sock"); + let published = + cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())).unwrap(); + server_process.publish_socket(published).unwrap(); let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); server_process.cleanup.set_shutdown_attempt_for_test({ let attempts = attempts.clone(); @@ -1803,6 +1811,44 @@ mod tests { .unwrap(); worker.join().unwrap(); assert_eq!(attempts.load(Ordering::SeqCst), 2); + let _ = std::fs::remove_dir_all(socket_dir); + } + + #[test] + fn unpublished_shutdown_failure_returns_without_waiting_for_retry() { + let mux = Mux::new("unpublished-shutdown-test", SurfaceOptions::default()); + let server_process = ServerProcessShutdownGuard::start(&mux).unwrap(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + server_process.cleanup.set_shutdown_attempt_for_test({ + let attempts = attempts.clone(); + move || { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + anyhow::bail!("forced unpublished shutdown failure"); + } + Ok(()) + } + }); + let (returned, observed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = finish_server_process(server_process, Ok(())); + let _ = returned.send(result); + }); + + let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { + std::thread::yield_now(); + } + assert_eq!(attempts.load(Ordering::SeqCst), 1); + let early = observed.recv_timeout(std::time::Duration::from_millis(75)); + if early.is_err() { + mux.request_shutdown(); + let _ = observed.recv_timeout(std::time::Duration::from_secs(1)); + } + worker.join().unwrap(); + + assert!(early.is_ok(), "an unpublished server waited on its unreachable control socket"); + assert!(early.unwrap().is_err(), "failed unpublished cleanup reported success"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); } fn args(values: &[&str]) -> Args { From 4d7d6264279dec879f3db140ed79df7327f94832 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:32:41 -0700 Subject: [PATCH 213/521] fix(tui): keep launcher compatible with Python 3.8 --- cmux-tui/dist/scripts/package_pypi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/dist/scripts/package_pypi.py b/cmux-tui/dist/scripts/package_pypi.py index a3884d5ee80a..d738416171a8 100644 --- a/cmux-tui/dist/scripts/package_pypi.py +++ b/cmux-tui/dist/scripts/package_pypi.py @@ -133,7 +133,7 @@ def _launcher_command(argv0: str) -> str: else: if any( candidate.startswith("archive-v") - and candidate.removeprefix("archive-v").isdigit() + and candidate[len("archive-v") :].isdigit() for candidate in relative.parts ): return "uvx cmux==" + PACKAGE_VERSION @@ -142,7 +142,7 @@ def _launcher_command(argv0: str) -> str: continue if any( candidate.startswith("archive-v") - and candidate.removeprefix("archive-v").isdigit() + and candidate[len("archive-v") :].isdigit() for candidate in parts[index + 1 :] ): return "uvx cmux==" + PACKAGE_VERSION From 8a669224fa068d3c871cf7f5b14415b6bad47233 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:34:59 -0700 Subject: [PATCH 214/521] fix(tui): bound unpublished server cleanup --- cmux-tui/crates/cmux-tui/src/main.rs | 48 +++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index f767215f2bff..6e2c923d6bb5 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -960,6 +960,7 @@ enum ServerShutdownCleanupState { Complete, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, Poisoned(String), + Abandoned, } #[derive(Clone)] @@ -967,6 +968,7 @@ enum ServerShutdownCleanupOutcome { Complete, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, Poisoned(String), + Abandoned, } #[cfg(test)] @@ -1027,6 +1029,9 @@ impl ServerShutdownCleanup { ServerShutdownCleanupState::Poisoned(message) => { return ServerShutdownCleanupOutcome::Poisoned(message.clone()); } + ServerShutdownCleanupState::Abandoned => { + return ServerShutdownCleanupOutcome::Abandoned; + } ServerShutdownCleanupState::Running(_) => { state = self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1072,6 +1077,7 @@ impl ServerShutdownCleanup { ServerShutdownCleanupOutcome::Poisoned(message) => { ServerShutdownCleanupState::Poisoned(message.clone()) } + ServerShutdownCleanupOutcome::Abandoned => ServerShutdownCleanupState::Abandoned, }; state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); debug_assert!( @@ -1082,15 +1088,26 @@ impl ServerShutdownCleanup { outcome } - #[cfg(test)] fn run_until_complete(&self) -> anyhow::Result<()> { match self.run_once() { ServerShutdownCleanupOutcome::Complete => Ok(()), ServerShutdownCleanupOutcome::Degraded { message, .. } | ServerShutdownCleanupOutcome::Poisoned(message) => Err(anyhow::anyhow!(message)), + ServerShutdownCleanupOutcome::Abandoned => { + Err(anyhow::anyhow!("server shutdown cleanup was abandoned")) + } } } + fn abandon(&self) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let ServerShutdownCleanupState::Degraded { retry, .. } = &*state { + retry.cancel(); + } + *state = ServerShutdownCleanupState::Abandoned; + self.changed.notify_all(); + } + fn prepare_retry(&self, attempt: u64) { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if matches!( @@ -1151,6 +1168,7 @@ impl ServerShutdownCleanup { }) .unwrap_or_else(std::sync::PoisonError::into_inner); } + ServerShutdownCleanupOutcome::Abandoned => return, } } } @@ -1232,7 +1250,10 @@ impl ServerProcessShutdownGuard { Ok(()) } - #[cfg(test)] + fn has_published_socket(&self) -> bool { + self.published_socket.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_some() + } + fn shutdown(&self) -> anyhow::Result<()> { self.cleanup.run_until_complete() } @@ -1245,6 +1266,17 @@ impl ServerProcessShutdownGuard { self.finish(); } + fn abandon_unpublished(mut self) { + debug_assert!(!self.has_published_socket()); + self.cleanup.abandon(); + let Some(completion) = self.completion.take() else { return }; + let _ = completion.send(()); + self.shutdown_watch.cancel(); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } + fn finish(&mut self) { if !self.cleanup.is_complete() { return; @@ -1281,6 +1313,15 @@ fn finish_server_process( server_process: ServerProcessShutdownGuard, result: anyhow::Result<()>, ) -> anyhow::Result<()> { + if !server_process.has_published_socket() { + let shutdown_result = server_process.shutdown(); + server_process.abandon_unpublished(); + return match (result, shutdown_result) { + (result, Ok(())) => result, + (Ok(()), Err(shutdown_error)) => Err(shutdown_error), + (Err(error), Err(shutdown_error)) => Err(error.context(shutdown_error.to_string())), + }; + } server_process.wait_for_shutdown(); server_process.complete(); result @@ -1775,8 +1816,7 @@ mod tests { let socket_dir = std::env::temp_dir() .join(format!("cmux-shutdown-retry-{}-{suffix:x}", std::process::id())); let socket_path = socket_dir.join("server.sock"); - let published = - cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())).unwrap(); + let published = cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path)).unwrap(); server_process.publish_socket(published).unwrap(); let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); server_process.cleanup.set_shutdown_attempt_for_test({ From 0eff5732f2e2c9bd9300e7d6498578de478b462d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:52:31 -0700 Subject: [PATCH 215/521] test(tui): expose unsafe macOS package floor --- cmux-tui/dist/scripts/test_package_pypi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 72833a651403..5962346c1278 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -71,7 +71,7 @@ def test_stable_install_path_is_reused_directly(self) -> None: class DarwinCompatibilityTests(unittest.TestCase): - def test_wheels_preserve_the_supported_macos_floor(self) -> None: + def test_wheels_require_the_first_representable_safe_macos_floor(self) -> None: tags = { target.rust_target: target.platform_tags for target in TARGETS @@ -81,14 +81,14 @@ def test_wheels_preserve_the_supported_macos_floor(self) -> None: self.assertEqual( tags, { - "aarch64-apple-darwin": ("macosx_14_0_arm64",), - "x86_64-apple-darwin": ("macosx_14_0_x86_64",), + "aarch64-apple-darwin": ("macosx_15_0_arm64",), + "x86_64-apple-darwin": ("macosx_15_0_x86_64",), }, ) def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: compatibility_file = Path(__file__).resolve().parents[1] / "macos-deployment-target.txt" - self.assertEqual(compatibility_file.read_text().strip(), "14.0") + self.assertEqual(compatibility_file.read_text().strip(), "15.0") workflow = ( Path(__file__).resolve().parents[3] @@ -97,6 +97,8 @@ def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: / "cmux-tui-build-package.yml" ).read_text() self.assertGreaterEqual(workflow.count("dist/macos-deployment-target.txt"), 2) + readme = Path(__file__).resolve().parents[2] / "README.md" + self.assertIn("macOS 15 or newer", readme.read_text()) def test_intel_release_uses_the_apple_linker_deployment_target(self) -> None: workflow = ( From 9653ac69ee44176d7bace96af96fdaa9ad16de47 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:52:56 -0700 Subject: [PATCH 216/521] fix(tui): declare macOS 15 package floor --- cmux-tui/README.md | 3 +++ cmux-tui/dist/RELEASING-TUI.md | 3 +++ cmux-tui/dist/macos-deployment-target.txt | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmux-tui/README.md b/cmux-tui/README.md index d3ea191c7c98..84cca996386e 100644 --- a/cmux-tui/README.md +++ b/cmux-tui/README.md @@ -43,6 +43,9 @@ Linux requires kernel 5.1 or newer, a mounted `/proc`, and permission to call `pidfd_send_signal`. Linux 5.3 and newer use `pidfd_open`; Linux 5.1 and 5.2 use an exact `/proc/` process handle. +Published `cmux-tui` packages require macOS 15 or newer. This TUI package +requirement is separate from the cmux macOS app's system requirement. + Pane layout stays tiled by default. Press `Ctrl-b g` to append a terminal to the right at two-thirds of the viewport width. The existing layout keeps its width, so a continuous horizontal scrollbar appears in the status bar. Focusing a pane reveals it with an animated viewport movement. `Alt-n` reapplies Zellij's automatic layout inside the focused horizontal column. `Ctrl-b U` undoes the latest structural layout action on the focused screen; undoing pane creation asks for confirmation before closing the pane. `attach --surface ` attaches one PTY terminal by numeric or short surface id. It uses the full host terminal without the sidebar, status bar, pane border, or other tabs. diff --git a/cmux-tui/dist/RELEASING-TUI.md b/cmux-tui/dist/RELEASING-TUI.md index f63ebcacd9d9..0942715a93ff 100644 --- a/cmux-tui/dist/RELEASING-TUI.md +++ b/cmux-tui/dist/RELEASING-TUI.md @@ -34,6 +34,9 @@ musllinux wheel tags so installers on both runtime families can resolve it. The runtime requires Linux 5.1 or newer, a mounted `/proc`, and permission to call `pidfd_send_signal`. +macOS packages require macOS 15 or newer. The shared deployment-target file +sets both wheel compatibility tags and the minimum version in each Mach-O. + ## One-time registry setup Add npm Trusted Publishers for all five npm package names: diff --git a/cmux-tui/dist/macos-deployment-target.txt b/cmux-tui/dist/macos-deployment-target.txt index ed9d00f9a5b8..fe6b09a7d14c 100644 --- a/cmux-tui/dist/macos-deployment-target.txt +++ b/cmux-tui/dist/macos-deployment-target.txt @@ -1 +1 @@ -14.0 +15.0 From 6127f75a118e01f7a1c77f793d516c2fd49d8628 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:54:19 -0700 Subject: [PATCH 217/521] test(tui): expose publication lock symlink attack --- cmux-tui/crates/cmux-tui-core/src/server.rs | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 05f147b6e68d..c1c065dfdab1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -6661,6 +6661,35 @@ mod tests { assert!(!completed_while_locked, "socket cleanup bypassed the concurrent publication lock"); } + #[cfg(unix)] + #[test] + fn publication_lock_rejects_symlink_without_changing_target_permissions() { + use std::os::unix::fs::{PermissionsExt, symlink}; + + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let directory = std::env::temp_dir() + .join(format!("cmux-publication-symlink-{}-{sequence:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let socket_path = directory.join("server.sock"); + let target = directory.join("target"); + std::fs::write(&target, b"target").unwrap(); + std::fs::set_permissions(&target, std::fs::Permissions::from_mode(0o644)).unwrap(); + let lock_path = socket_publication_lock_path(&socket_path).unwrap(); + symlink(&target, &lock_path).unwrap(); + + let result = + SocketPublicationLock::acquire(&socket_path, Instant::now() + Duration::from_secs(1)); + let target_mode = std::fs::metadata(&target).unwrap().permissions().mode() & 0o777; + let rejected = result.is_err(); + drop(result); + std::fs::remove_file(&lock_path).unwrap(); + std::fs::remove_file(&target).unwrap(); + std::fs::remove_dir(&directory).unwrap(); + + assert!(rejected, "publication lock followed a symbolic link"); + assert_eq!(target_mode, 0o644, "publication lock changed the symlink target mode"); + } + #[cfg(unix)] #[test] fn concurrent_stale_socket_reclamation_has_one_owner() { From c0502d35a0a8fff10e204cad58dcb9010abcdb3a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:58:55 -0700 Subject: [PATCH 218/521] fix(tui): harden publication lock creation --- cmux-tui/crates/cmux-tui-core/src/server.rs | 38 +++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index c1c065dfdab1..b5e2448e6b80 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -2793,12 +2793,38 @@ struct SocketPublicationLock { impl SocketPublicationLock { fn acquire(path: &Path, deadline: Instant) -> std::io::Result { let lock_path = socket_publication_lock_path(path)?; - let file = std::fs::OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path)?; + let mut options = std::fs::OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.mode(0o600).custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + } + let file = options.open(&lock_path)?; + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let metadata = file.metadata()?; + if !metadata.file_type().is_file() || metadata.nlink() != 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "server socket publication lock is not a single regular file", + )); + } + // SAFETY: geteuid has no preconditions and returns the effective + // user that owns files created by this process. + let current_user = unsafe { libc::geteuid() }; + if metadata.uid() != current_user { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "server socket publication lock is owned by another user", + )); + } + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + } + #[cfg(not(unix))] platform::restrict_file(&lock_path)?; loop { match FileExt::try_lock(&file) { From 11947eb5996f13359934577af1969d76e8ec86d8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:03:12 -0700 Subject: [PATCH 219/521] test(tui): cover frontend finalization paths --- cmux-tui/crates/cmux-tui/src/main.rs | 4 ++ cmux-tui/crates/cmux-tui/tests/cli.rs | 74 +++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 6e2c923d6bb5..4f08d3a3e096 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1259,6 +1259,10 @@ impl ServerProcessShutdownGuard { } fn wait_for_shutdown(&self) { + // `wait_for_complete` runs the first bounded cleanup attempt + // synchronously. It waits for another mux request only after that + // attempt fails, while the published socket remains available for an + // explicit `server stop` retry. self.cleanup.wait_for_complete(); } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 36d2bd8e44be..eeac5d980f21 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1387,6 +1387,80 @@ fn server_status_and_stop_control_a_compatible_headless_session() { assert!(!server.socket.exists()); } +#[cfg(unix)] +#[test] +fn signal_termination_completes_server_shutdown() { + let dir = unique_temp_dir("server-signal-shutdown"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let mut server = Command::new(bin()) + .args(["--headless", "--ephemeral", "--socket"]) + .arg(&socket) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + wait_for_socket_path(&socket); + + assert_eq!(unsafe { libc::kill(server.id() as libc::pid_t, libc::SIGTERM) }, 0); + let deadline = Instant::now() + Duration::from_secs(3); + let status = loop { + if let Some(status) = server.try_wait().unwrap() { + break Some(status); + } + if Instant::now() >= deadline { + server.kill().unwrap(); + let _ = server.wait(); + break None; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let socket_remained = socket.exists(); + fs::remove_dir_all(dir).unwrap(); + + let status = status.expect("SIGTERM left the server waiting on an unrequested mux shutdown"); + assert!(status.success(), "signal-driven shutdown exited with {status}"); + assert!(!socket_remained, "signal-driven shutdown retained its control socket"); +} + +#[cfg(unix)] +#[test] +fn invalid_websocket_startup_returns_after_publishing_the_local_socket() { + let dir = unique_temp_dir("invalid-websocket-startup"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let mut server = Command::new(bin()) + .args(["--headless", "--ephemeral", "--socket"]) + .arg(&socket) + .args(["--ws", "not-an-address"]) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + + let deadline = Instant::now() + Duration::from_secs(3); + let status = loop { + if let Some(status) = server.try_wait().unwrap() { + break Some(status); + } + if Instant::now() >= deadline { + server.kill().unwrap(); + let _ = server.wait(); + break None; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let socket_remained = socket.exists(); + let mut stderr = String::new(); + server.stderr.take().unwrap().read_to_string(&mut stderr).unwrap(); + fs::remove_dir_all(dir).unwrap(); + + let status = status.expect("invalid WebSocket setup entered the running-server shutdown wait"); + assert!(!status.success(), "invalid WebSocket setup unexpectedly succeeded"); + assert!(stderr.contains("invalid WebSocket address"), "{stderr}"); + assert!(!socket_remained, "failed startup retained its local control socket"); +} + #[cfg(unix)] #[test] fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { From 44d397423c1d8f94e7c9e7c83ff87dd15b82d8a4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:20:30 -0700 Subject: [PATCH 220/521] test(tui): expose blocking adoption records --- .../src/terminal_host_runtime.rs | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 8fc44beeef4b..f28a15ceb944 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4674,6 +4674,40 @@ mod unix { assert!(completed.unwrap().is_err(), "FIFO was accepted as a discovery record"); } + #[test] + fn adoption_record_loader_rejects_fifo_without_blocking() { + let root = std::env::temp_dir().join(format!( + "cmux-host-adoption-record-fifo-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + prepare_private_dir(&root).unwrap(); + let path = root.join(format!("{}.json", TerminalId::random().unwrap().to_hex())); + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: c_path is a valid, NUL-terminated path owned by this test. + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + + let loader_root = root.clone(); + let (result_tx, result_rx) = sync_channel(1); + let loader = thread::spawn(move || { + let _ = result_tx.send(load_terminal_host_records(&loader_root)); + }); + let completed = match result_rx.recv_timeout(Duration::from_millis(500)) { + Ok(result) => Some(result), + Err(RecvTimeoutError::Timeout) => { + let mut writer = OpenOptions::new().write(true).open(&path).unwrap(); + writer.write_all(b"{}").unwrap(); + None + } + Err(RecvTimeoutError::Disconnected) => panic!("record loader stopped"), + }; + loader.join().unwrap(); + let _ = fs::remove_dir_all(&root); + + assert!(completed.is_some(), "terminal-host adoption blocked while opening a FIFO"); + assert!(completed.unwrap().unwrap().is_empty(), "FIFO was adopted as a host record"); + } + #[test] fn strict_record_loader_rejects_oversized_file_before_decoding() { let root = std::env::temp_dir().join(format!( From bd319ad51553da7c45c6ce49fbd277666f1e7acc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:22:32 -0700 Subject: [PATCH 221/521] fix(tui): bound terminal host record discovery --- .../src/terminal_host_runtime.rs | 87 ++++++++++--------- 1 file changed, 48 insertions(+), 39 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index f28a15ceb944..c1be112d88da 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -328,6 +328,8 @@ mod unix { const HOST_PROCESS_REAPER_POLL: Duration = Duration::from_millis(25); const HOST_PROCESS_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); const MAX_TERMINAL_HOST_RECORD_BYTES: usize = MAX_LAUNCH_PAYLOAD; + const MAX_TERMINAL_HOST_RECORDS: usize = 4_096; + const TERMINAL_HOST_RECORD_SCAN_TIMEOUT: Duration = Duration::from_secs(2); static HOST_PROCESS_REAPER: OnceLock>> = OnceLock::new(); #[cfg(test)] static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); @@ -1808,35 +1810,12 @@ mod unix { pub fn load_terminal_host_records( root: &Path, ) -> anyhow::Result> { - let mut records = Vec::new(); - let mut identities = HashSet::new(); - let entries = match fs::read_dir(root) { - Ok(entries) => entries, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(records), - Err(error) => return Err(error.into()), - }; - for entry in entries { - let entry = entry?; - let path = entry.path(); - if path.extension().and_then(|value| value.to_str()) != Some("json") { - continue; - } - let bytes = match fs::read(&path) { - Ok(bytes) => bytes, - Err(_) => continue, - }; - let Ok(record) = serde_json::from_slice::(&bytes) else { - continue; - }; - if validate_terminal_host_record(&path, &record).is_err() - || !identities.insert((record.terminal_id.clone(), record.incarnation.clone())) - { - continue; - } - records.push((path, record)); - } - records.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(records) + load_terminal_host_records_until( + root, + MAX_TERMINAL_HOST_RECORDS, + Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT, + TerminalHostRecordLoadMode::Tolerant, + ) } /// Load every discovery record without omission. Adoption tolerates @@ -1846,10 +1825,31 @@ mod unix { root: &Path, max_records: usize, deadline: Instant, + ) -> anyhow::Result> { + load_terminal_host_records_until( + root, + max_records, + deadline, + TerminalHostRecordLoadMode::Strict, + ) + } + + #[derive(Clone, Copy, PartialEq, Eq)] + enum TerminalHostRecordLoadMode { + Tolerant, + Strict, + } + + fn load_terminal_host_records_until( + root: &Path, + max_records: usize, + deadline: Instant, + mode: TerminalHostRecordLoadMode, ) -> anyhow::Result> { ensure_terminal_host_scan_before(deadline)?; let mut records = Vec::new(); let mut identities = HashSet::new(); + let mut candidates = 0usize; let entries = match fs::read_dir(root) { Ok(entries) => entries, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(records), @@ -1862,18 +1862,27 @@ mod unix { if path.extension().and_then(|value| value.to_str()) != Some("json") { continue; } - if records.len() >= max_records { - anyhow::bail!( - "terminal-host record count exceeds shutdown owner capacity {max_records}" - ); + candidates = candidates.saturating_add(1); + if candidates > max_records { + anyhow::bail!("terminal-host record count exceeds capacity {max_records}"); } - let record = read_terminal_host_record_until(&path, deadline)?; + let record = match read_terminal_host_record_until(&path, deadline) { + Ok(record) => record, + Err(_) if mode == TerminalHostRecordLoadMode::Tolerant => { + ensure_terminal_host_scan_before(deadline)?; + continue; + } + Err(error) => return Err(error), + }; if !identities.insert((record.terminal_id.clone(), record.incarnation.clone())) { - anyhow::bail!( - "duplicate terminal-host identity {}:{}", - record.terminal_id, - record.incarnation - ); + if mode == TerminalHostRecordLoadMode::Strict { + anyhow::bail!( + "duplicate terminal-host identity {}:{}", + record.terminal_id, + record.incarnation + ); + } + continue; } records.push((path, record)); } From 1b9df5f9bd99d366f81ff2074609a44d96b3242b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:23:42 -0700 Subject: [PATCH 222/521] test(tui): expose terminal host directory symlink --- .../src/terminal_host_runtime.rs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index c1be112d88da..ce2bb1a706c3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4645,6 +4645,45 @@ mod unix { let _ = fs::remove_dir_all(root); } + #[test] + fn private_directory_setup_rejects_symlink_without_changing_target_permissions() { + let root = std::env::temp_dir().join(format!( + "cmux-host-private-dir-symlink-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root).unwrap(); + let target = root.join("target"); + fs::create_dir(&target).unwrap(); + fs::set_permissions(&target, fs::Permissions::from_mode(0o755)).unwrap(); + let link = root.join("endpoint"); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let result = prepare_private_dir(&link); + let target_mode = fs::metadata(&target).unwrap().mode() & 0o777; + let _ = fs::remove_dir_all(&root); + + assert!(result.is_err(), "private directory setup accepted a symlink"); + assert_eq!(target_mode, 0o755, "private directory setup changed the symlink target"); + } + + #[test] + fn private_directory_setup_tightens_an_owned_directory() { + let root = std::env::temp_dir().join(format!( + "cmux-host-private-dir-owned-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o755)).unwrap(); + + prepare_private_dir(&root).unwrap(); + let mode = fs::metadata(&root).unwrap().mode() & 0o777; + let _ = fs::remove_dir_all(&root); + + assert_eq!(mode, 0o700); + } + #[test] fn strict_record_loader_rejects_fifo_without_blocking() { let root = std::env::temp_dir().join(format!( From e75951aa0765a51f7858554db405113c0217514f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:25:03 -0700 Subject: [PATCH 223/521] fix(tui): harden terminal host endpoint directory --- .../src/terminal_host_runtime.rs | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index ce2bb1a706c3..0c9c66d2de09 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2256,8 +2256,51 @@ mod unix { } fn prepare_private_dir(path: &Path) -> anyhow::Result<()> { - fs::create_dir_all(path)?; - fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + match fs::create_dir(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let parent = path.parent().ok_or_else(|| { + anyhow::anyhow!("terminal-host directory has no parent: {}", path.display()) + })?; + fs::create_dir_all(parent)?; + match fs::create_dir(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + Err(error) => return Err(error.into()), + } + + let directory = OpenOptions::new() + .read(true) + .custom_flags(libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(path) + .with_context(|| format!("open private terminal-host directory {}", path.display()))?; + let metadata = directory.metadata().with_context(|| { + format!("inspect private terminal-host directory {}", path.display()) + })?; + // SAFETY: geteuid has no preconditions and returns the effective user + // that must own this process-private endpoint directory. + let current_user = unsafe { libc::geteuid() }; + if !metadata.file_type().is_dir() || metadata.uid() != current_user { + anyhow::bail!( + "private terminal-host directory is not an owned directory: {}", + path.display() + ); + } + directory.set_permissions(fs::Permissions::from_mode(0o700))?; + + let current = fs::symlink_metadata(path).with_context(|| { + format!("reinspect private terminal-host directory {}", path.display()) + })?; + if !current.file_type().is_dir() + || current.dev() != metadata.dev() + || current.ino() != metadata.ino() + { + anyhow::bail!("private terminal-host directory changed: {}", path.display()); + } Ok(()) } From 6a7712d752b4c69e3b51121377624bface2c78d2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:29:26 -0700 Subject: [PATCH 224/521] test(tui): expose stalled host handshakes --- .../src/terminal_host_runtime.rs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 0c9c66d2de09..4ac8c7601628 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4112,6 +4112,79 @@ mod unix { mod tests { use super::*; + fn spawn_sleeping_host_for_test(name: &str) -> (Arc, PathBuf) { + let root = std::env::temp_dir() + .join(format!("cmux-{name}-{}", crate::workspace_registry::new_uuid_v4())); + fs::create_dir_all(&root).unwrap(); + let bootstrap = HostBootstrap { + min_version: PROTOCOL_VERSION, + max_version: PROTOCOL_VERSION, + terminal_id: TerminalId::random().unwrap(), + owner_token: CapabilityToken::random().unwrap(), + }; + let mut bootstrap_bytes = Vec::new(); + write_frame(&mut bootstrap_bytes, &bootstrap.into_frame(1)).unwrap(); + let bootstrapped = crate::terminal_host::bootstrap_stdio_once( + &mut bootstrap_bytes.as_slice(), + &mut Vec::new(), + ) + .unwrap(); + let launch = HostLaunch { + endpoint: root.join("host.sock").to_string_lossy().into_owned(), + record_path: root.join("host.json").to_string_lossy().into_owned(), + term: "xterm-256color".into(), + cols: 80, + rows: 24, + scrollback: 100, + cwd: Some("/tmp".into()), + command: vec!["/bin/sh".into(), "-c".into(), "exec /bin/sleep 60".into()], + extra_env: Vec::new(), + default_colors: DefaultColors::default(), + }; + (spawn_host_runtime(&launch, &bootstrapped).unwrap(), root) + } + + fn stop_test_host(host: &Arc, root: &Path) { + host.request_termination(); + assert!( + host.wait_for_child_exit(Duration::from_secs(3)), + "test terminal host did not stop" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn unauthenticated_host_client_handshake_is_time_bounded() { + let _guard = HOST_REAP_TEST_LOCK.lock().unwrap(); + let (host, root) = spawn_sleeping_host_for_test("host-handshake-timeout"); + let (server, client) = cmux_tui_process::unix::pair_stream().unwrap(); + let client_host = host.clone(); + let (done_sender, done_receiver) = sync_channel(1); + let worker = thread::spawn(move || { + done_sender.send(serve_client(client_host, server)).unwrap(); + }); + + let result = + done_receiver.recv_timeout(HOST_HANDSHAKE_TIMEOUT + Duration::from_secs(1)); + drop(client); + if result.is_err() { + let _ = done_receiver.recv_timeout(Duration::from_secs(1)); + } + worker.join().unwrap(); + stop_test_host(&host, &root); + + match result { + Ok(Err(_)) => {} + Ok(Ok(())) => panic!("an empty client completed a terminal-host handshake"), + Err(RecvTimeoutError::Timeout) => { + panic!("an unauthenticated client retained a host thread past the deadline") + } + Err(RecvTimeoutError::Disconnected) => { + panic!("the terminal-host client worker disconnected without a result") + } + } + } + #[cfg(target_os = "macos")] #[test] fn host_pty_creation_waits_for_process_barrier() { From 2b188dc8f04f1f0a7397ab9a5a3c3cac9606081a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:32:09 -0700 Subject: [PATCH 225/521] fix(tui): bound terminal host clients --- .../src/terminal_host_runtime.rs | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 4ac8c7601628..28a532bd372c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -329,6 +329,7 @@ mod unix { const HOST_PROCESS_REAPER_RETRY_MAX: Duration = Duration::from_secs(1); const MAX_TERMINAL_HOST_RECORD_BYTES: usize = MAX_LAUNCH_PAYLOAD; const MAX_TERMINAL_HOST_RECORDS: usize = 4_096; + const MAX_TERMINAL_HOST_CLIENTS: usize = 64; const TERMINAL_HOST_RECORD_SCAN_TIMEOUT: Duration = Duration::from_secs(2); static HOST_PROCESS_REAPER: OnceLock>> = OnceLock::new(); #[cfg(test)] @@ -2421,6 +2422,27 @@ mod unix { } } + struct HostClientAdmission { + active: Arc, + } + + impl HostClientAdmission { + fn try_acquire(active: &Arc) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < MAX_TERMINAL_HOST_CLIENTS).then_some(count + 1) + }) + .ok()?; + Some(Self { active: active.clone() }) + } + } + + impl Drop for HostClientAdmission { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } + } + struct HostShared { terminal_id: TerminalId, incarnation: HostIncarnation, @@ -2440,6 +2462,7 @@ mod unix { broadcast_lock: Mutex<()>, sequence: AtomicU64, next_client: AtomicU64, + active_clients: Arc, dead: AtomicBool, child_exit: (Mutex, Condvar), child_waitable: AtomicBool, @@ -3263,12 +3286,20 @@ mod unix { // on macOS. Client protocol threads use blocking framed // reads, so normalize the accepted descriptor here. stream.set_nonblocking(false)?; + let Some(admission) = HostClientAdmission::try_acquire(&shared.active_clients) + else { + continue; + }; let host = shared.clone(); - thread::Builder::new().name("terminal-host-client".into()).spawn( + // A transient thread creation failure rejects this client + // without taking down the terminal host. Dropping the + // unstarted closure releases its admission automatically. + let _ = thread::Builder::new().name("terminal-host-client".into()).spawn( move || { + let _admission = admission; let _ = serve_client(host, stream); }, - )?; + ); } Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { thread::sleep(Duration::from_millis(20)); @@ -3375,6 +3406,7 @@ mod unix { broadcast_lock: Mutex::new(()), sequence: AtomicU64::new(0), next_client: AtomicU64::new(1), + active_clients: Arc::new(AtomicUsize::new(0)), dead: AtomicBool::new(false), child_exit: (Mutex::new(false), Condvar::new()), child_waitable: AtomicBool::new(false), @@ -3560,6 +3592,7 @@ mod unix { } fn serve_client(host: Arc, mut stream: UnixStream) -> anyhow::Result<()> { + stream.set_read_timeout(Some(HOST_HANDSHAKE_TIMEOUT))?; let hello_frame = read_required_frame(&mut stream, "client hello")?; if hello_frame.kind != MessageKind::ClientHello || hello_frame.sequence != 0 @@ -3592,7 +3625,6 @@ mod unix { write_frame(&mut stream, &hello_response)?; if terminate_only { - stream.set_read_timeout(Some(HOST_HANDSHAKE_TIMEOUT))?; let terminate = read_required_frame(&mut stream, "terminate-only request")?; if terminate.kind != MessageKind::Terminate || terminate.flags != 0 @@ -3606,6 +3638,10 @@ mod unix { return Ok(()); } + // Authenticated interactive clients are long-lived. Clear the + // unauthenticated handshake deadline before entering their command + // and live-output loops. + stream.set_read_timeout(None)?; let client = host.next_client.fetch_add(1, Ordering::Relaxed); let (sender, receiver) = sync_channel(256); let tap = HostTap { @@ -4185,6 +4221,29 @@ mod unix { } } + #[test] + fn host_client_admission_is_bounded_and_reusable() { + let active = Arc::new(AtomicUsize::new(0)); + let mut admissions = (0..MAX_TERMINAL_HOST_CLIENTS) + .map(|_| { + HostClientAdmission::try_acquire(&active) + .expect("a client below the configured limit was rejected") + }) + .collect::>(); + + assert!( + HostClientAdmission::try_acquire(&active).is_none(), + "the terminal host admitted more than its configured client limit" + ); + admissions.pop(); + let replacement = HostClientAdmission::try_acquire(&active) + .expect("a released client admission was not reusable"); + drop(replacement); + drop(admissions); + + assert_eq!(active.load(Ordering::Acquire), 0); + } + #[cfg(target_os = "macos")] #[test] fn host_pty_creation_waits_for_process_barrier() { From bf8c67044e322780227f37d81221cb5801bc11d9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:36:04 -0700 Subject: [PATCH 226/521] test(tui): expose polling host listeners --- .../src/terminal_host_runtime.rs | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 28a532bd372c..38122d0b943e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2422,6 +2422,15 @@ mod unix { } } + fn wait_for_host_service_activity( + _listener_fd: RawFd, + _exit_waiter: &mut UnixStream, + dead: &AtomicBool, + ) -> std::io::Result { + thread::sleep(Duration::from_millis(20)); + Ok(!dead.load(Ordering::Acquire)) + } + struct HostClientAdmission { active: Arc, } @@ -3279,6 +3288,7 @@ mod unix { // record and connects through the already-listening Unix socket. let _ = write_frame(writer, &response); + let (_service_waker, mut service_waiter) = cmux_tui_process::unix::pair_stream()?; while !shared.dead.load(Ordering::Acquire) { match cmux_tui_process::unix::accept_stream(&listener) { Ok((stream, _)) => { @@ -3302,7 +3312,13 @@ mod unix { ); } Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(20)); + if !wait_for_host_service_activity( + listener.as_raw_fd(), + &mut service_waiter, + &shared.dead, + )? { + break; + } } Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} Err(error) => return Err(error.into()), @@ -4244,6 +4260,54 @@ mod unix { assert_eq!(active.load(Ordering::Acquire), 0); } + #[test] + fn idle_host_service_wait_is_event_driven() { + let root = std::env::temp_dir().join(format!( + "cmux-hsw-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&root).unwrap(); + let endpoint = root.join("host.sock"); + let listener = UnixListener::bind(&endpoint).unwrap(); + listener.set_nonblocking(true).unwrap(); + let (mut service_waker, mut service_waiter) = + cmux_tui_process::unix::pair_stream().unwrap(); + let dead = Arc::new(AtomicBool::new(false)); + let waiter_dead = dead.clone(); + let listener_fd = listener.as_raw_fd(); + let (done_sender, done_receiver) = sync_channel(1); + let worker = thread::spawn(move || { + done_sender + .send(wait_for_host_service_activity( + listener_fd, + &mut service_waiter, + &waiter_dead, + )) + .unwrap(); + }); + + let first = done_receiver.recv_timeout(Duration::from_millis(75)); + dead.store(true, Ordering::Release); + let _ = service_waker.write_all(&[1]); + let (waited_for_event, result) = match first { + Ok(result) => (false, result), + Err(RecvTimeoutError::Timeout) => { + (true, done_receiver.recv_timeout(Duration::from_secs(1)).unwrap()) + } + Err(RecvTimeoutError::Disconnected) => { + panic!("the host service waiter disconnected without a result") + } + }; + worker.join().unwrap(); + drop(listener); + let _ = fs::remove_file(endpoint); + let _ = fs::remove_dir(root); + + assert!(waited_for_event, "an idle host service woke without socket activity"); + assert!(!result.unwrap(), "the exit wake was reported as client activity"); + } + #[cfg(target_os = "macos")] #[test] fn host_pty_creation_waits_for_process_barrier() { From 30ae6f37fbc0295d5452bb96e52243f41884e6a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:39:09 -0700 Subject: [PATCH 227/521] fix(tui): block on terminal host events --- .../src/terminal_host_runtime.rs | 96 +++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 38122d0b943e..fcf32ca56584 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2423,12 +2423,57 @@ mod unix { } fn wait_for_host_service_activity( - _listener_fd: RawFd, - _exit_waiter: &mut UnixStream, + listener_fd: RawFd, + exit_waiter: &mut UnixStream, dead: &AtomicBool, ) -> std::io::Result { - thread::sleep(Duration::from_millis(20)); - Ok(!dead.load(Ordering::Acquire)) + loop { + if dead.load(Ordering::Acquire) { + return Ok(false); + } + let mut poll_fds = [ + libc::pollfd { + fd: listener_fd, + events: libc::POLLIN | libc::POLLHUP | libc::POLLERR, + revents: 0, + }, + libc::pollfd { + fd: exit_waiter.as_raw_fd(), + events: libc::POLLIN | libc::POLLHUP | libc::POLLERR, + revents: 0, + }, + ]; + // SAFETY: poll_fds contains two initialized descriptors that + // remain owned by the service loop for the duration of this call. + let ready = + unsafe { libc::poll(poll_fds.as_mut_ptr(), poll_fds.len() as libc::nfds_t, -1) }; + if ready < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if poll_fds[0].revents & libc::POLLNVAL != 0 { + return Err(std::io::Error::other("terminal host listener became invalid")); + } + if poll_fds[1].revents & libc::POLLIN != 0 { + let mut wake = [0u8; 64]; + let _ = exit_waiter.read(&mut wake); + if dead.load(Ordering::Acquire) { + return Ok(false); + } + } + if poll_fds[0].revents & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0 { + return Ok(true); + } + if poll_fds[1].revents & (libc::POLLHUP | libc::POLLERR | libc::POLLNVAL) != 0 { + if dead.load(Ordering::Acquire) { + return Ok(false); + } + return Err(std::io::Error::other("terminal host exit waiter became invalid")); + } + } } struct HostClientAdmission { @@ -2479,6 +2524,8 @@ mod unix { exit_published: AtomicBool, force_pty_drain: AtomicBool, pty_drain_waker: Mutex, + service_exit_waker: Mutex, + service_exit_waiter: Mutex>, termination_started: AtomicBool, child_signal_lock: Mutex<()>, child_reaped: AtomicBool, @@ -2964,6 +3011,9 @@ mod unix { &self.exit_published, ) { self.dead.store(true, Ordering::Release); + // Wake the blocking listener poll so host service cleanup + // follows terminal exit without periodic timer retries. + let _ = self.service_exit_waker.lock().unwrap().write_all(&[1]); self.broadcast(MessageKind::Exit, Vec::new()); } } @@ -3288,7 +3338,10 @@ mod unix { // record and connects through the already-listening Unix socket. let _ = write_frame(writer, &response); - let (_service_waker, mut service_waiter) = cmux_tui_process::unix::pair_stream()?; + let mut service_waiter = + shared.service_exit_waiter.lock().unwrap().take().ok_or_else(|| { + anyhow::anyhow!("terminal host service waiter was already claimed") + })?; while !shared.dead.load(Ordering::Acquire) { match cmux_tui_process::unix::accept_stream(&listener) { Ok((stream, _)) => { @@ -3324,7 +3377,6 @@ mod unix { Err(error) => return Err(error.into()), } } - thread::sleep(Duration::from_millis(20)); drop(guard); Ok(()) } @@ -3376,6 +3428,7 @@ mod unix { let mut pty_reader = pty.master.try_clone_reader()?; let pty_writer = pty.master.take_writer()?; let (pty_drain_waker, pty_drain_waiter) = cmux_tui_process::unix::pair_stream()?; + let (service_exit_waker, service_exit_waiter) = cmux_tui_process::unix::pair_stream()?; let pending_responses = Arc::new(Mutex::new(Vec::::new())); let title_changed = Arc::new(AtomicBool::new(false)); @@ -3430,6 +3483,8 @@ mod unix { exit_published: AtomicBool::new(false), force_pty_drain: AtomicBool::new(false), pty_drain_waker: Mutex::new(pty_drain_waker), + service_exit_waker: Mutex::new(service_exit_waker), + service_exit_waiter: Mutex::new(Some(service_exit_waiter)), termination_started: AtomicBool::new(false), child_signal_lock: Mutex::new(()), child_reaped: AtomicBool::new(false), @@ -4308,6 +4363,35 @@ mod unix { assert!(!result.unwrap(), "the exit wake was reported as client activity"); } + #[test] + fn host_service_wait_reports_listener_activity() { + let root = std::env::temp_dir().join(format!( + "cmux-hsc-{}-{}", + std::process::id(), + RECORD_TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&root).unwrap(); + let endpoint = root.join("host.sock"); + let listener = UnixListener::bind(&endpoint).unwrap(); + listener.set_nonblocking(true).unwrap(); + let (_service_waker, mut service_waiter) = + cmux_tui_process::unix::pair_stream().unwrap(); + let dead = AtomicBool::new(false); + let listener_fd = listener.as_raw_fd(); + let worker = thread::spawn(move || { + wait_for_host_service_activity(listener_fd, &mut service_waiter, &dead) + }); + + let client = UnixStream::connect(&endpoint).unwrap(); + let activity = worker.join().unwrap().unwrap(); + drop(client); + drop(listener); + let _ = fs::remove_file(endpoint); + let _ = fs::remove_dir(root); + + assert!(activity, "a waiting host service ignored a client connection"); + } + #[cfg(target_os = "macos")] #[test] fn host_pty_creation_waits_for_process_barrier() { From b2a8cf89bece21c13e10c803f642d4047e1f7fdd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 00:44:53 -0700 Subject: [PATCH 228/521] test(tui): align launcher restart guidance --- cmux-tui/crates/cmux-tui/tests/cli.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index eeac5d980f21..66e9249a998b 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1354,10 +1354,7 @@ fn packaged_launcher_name_is_used_in_upgrade_instructions() { assert_eq!(output.status.code(), Some(1)); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("cmux server stop"), "{stderr}"); - assert!( - stderr.contains(&format!("then run `cmux ping --socket {}` again", socket.display())), - "{stderr}" - ); + assert!(stderr.contains("then rerun the previous command using `cmux`."), "{stderr}"); server.join().unwrap(); let _ = fs::remove_file(&socket); let _ = fs::remove_dir_all(&dir); From 5718027d4211b3021dbeb09ec527a8afb049a305 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:00:47 -0700 Subject: [PATCH 229/521] test(tui): keep blocked launch portable --- cmux-tui/crates/cmux-tui/tests/cli.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 66e9249a998b..faaa9d653dd8 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1606,7 +1606,10 @@ fn server_stop_cancels_a_blocked_terminal_host_launch_write() { false, &[("CMUX_TUI_TEST_STALL_AFTER_BOOTSTRAP_READY_MS", "3000")], ); - let command = format!(": #{}", "x".repeat(128 * 1024)); + // Linux caps one execve argument below 128 KiB even when the aggregate + // ARG_MAX is larger. This still exceeds the bootstrap pipe capacity, so + // the host blocks on the Launch write without failing the CLI spawn first. + let command = format!(": #{}", "x".repeat(96 * 1024)); let mut create = Command::new(bin()) .args(["--socket"]) .arg(&server.socket) From 43276d79e154b16de74d4b245c0cac5183b2b6d3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:53:33 -0700 Subject: [PATCH 230/521] test(tui): expose degraded reaper shutdown delay --- .../cmux-tui-core/src/process_session.rs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 8c3390415d06..88fa885aa573 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1721,6 +1721,51 @@ mod tests { ); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn state_change_wake_preempts_degraded_retry_before_shutdown_deadline() { + let now = Instant::now(); + let shutdown_deadline = now + Duration::from_secs(5); + let (sender, _receiver) = mpsc::channel(); + let degraded = Arc::new(AtomicUsize::new(1)); + let mut pending = vec![NaturalReapRequest { + session: 41, + cleanup_timeout: Duration::ZERO, + observe_child: Box::new(|| Ok(false)), + child_observed: false, + needs_cleanup: Box::new(|| false), + prepare_cleanup: Box::new(|| true), + finish: Box::new(|_| NaturalReapFinish::Pending), + _lease: ReservedChildReaperLease { + sender, + active: Arc::new(AtomicUsize::new(1)), + degraded: degraded.clone(), + }, + next_attempt: now + Duration::from_secs(30), + retry_delay: NATURAL_REAP_RETRY_MAX, + attempts: NATURAL_REAP_MAX_ATTEMPTS, + degraded: true, + }]; + let wake_pending = AtomicBool::new(true); + + accept_natural_reaper_command(NaturalReaperCommand::Wake, &mut pending, &wake_pending); + + assert!( + pending[0].next_attempt <= shutdown_deadline, + "the production degraded retry outlived the server shutdown deadline" + ); + assert!(pending[0].degraded); + assert_eq!(degraded.load(Ordering::Acquire), 1); + + let rescheduled_at = Instant::now(); + pending[0].schedule_failure(); + assert!( + pending[0].next_attempt.saturating_duration_since(rescheduled_at) + >= NATURAL_REAP_DEGRADED_RETRY.saturating_sub(Duration::from_millis(10)), + "an unsuccessful state-change retry disabled degraded backoff" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn ignored_sigchld_fails_before_reserving_a_child_reaper() { From 40556eda9862cb08cb1415610da7ddf9210e1a6f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 01:55:18 -0700 Subject: [PATCH 231/521] fix(tui): wake degraded reaper on shutdown changes --- .../cmux-tui-core/src/process_session.rs | 7 ++++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 27 +++++++++++------ .../src/terminal_host_runtime.rs | 29 ++++++++++++------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 88fa885aa573..338bc1a3a238 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -298,6 +298,11 @@ fn require_waitable_child_disposition() -> io::Result<()> { Ok(()) } +/// Wake every reserved child after a cleanup-relevant state transition. +/// +/// This deliberately preempts degraded retry delays once. Callers must emit +/// it only when the state actually changed; a failed attempt keeps its normal +/// degraded backoff. pub(crate) fn wake_child_reaper() { let Some(slot) = NATURAL_REAPER.get() else { return }; let slot = slot.lock().unwrap(); @@ -607,7 +612,7 @@ fn accept_natural_reaper_command( NaturalReaperCommand::Wake => { wake_pending.store(false, Ordering::Release); let now = Instant::now(); - for request in pending.iter_mut().filter(|request| !request.degraded) { + for request in pending { request.next_attempt = now; } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 5b98169708ef..55469e14500b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1134,9 +1134,11 @@ impl LocalPtyProcess { fn mark_pty_drained(&self) { #[cfg(unix)] { - self.pty_drained.store(true, Ordering::Release); + let changed = !self.pty_drained.swap(true, Ordering::AcqRel); self.exited.1.notify_all(); - crate::process_session::wake_child_reaper(); + if changed { + crate::process_session::wake_child_reaper(); + } } } @@ -1178,9 +1180,11 @@ impl LocalPtyProcess { #[cfg(unix)] fn abandon_termination(&self) { - self.termination_started.store(false, Ordering::Release); + let changed = self.termination_started.swap(false, Ordering::AcqRel); self.exited.1.notify_all(); - crate::process_session::wake_child_reaper(); + if changed { + crate::process_session::wake_child_reaper(); + } } #[cfg(unix)] @@ -1330,16 +1334,19 @@ impl LocalPtyProcess { ) -> bool { #[cfg(unix)] { - let (child_reaped, wait_ownership_lost) = { + let (termination_changed, child_reaped, wait_ownership_lost) = { let _signal = self.child_signal_lock.lock().unwrap(); - self.termination_started.store(true, Ordering::Release); + let termination_changed = !self.termination_started.swap(true, Ordering::AcqRel); ( + termination_changed, self.child_reaped.load(Ordering::Acquire), self.child_wait_ownership_lost.load(Ordering::Acquire), ) }; self.exited.1.notify_all(); - crate::process_session::wake_child_reaper(); + if termination_changed { + crate::process_session::wake_child_reaper(); + } if wait_ownership_lost { self.abandon_termination(); return self.wait_for_exit_until(deadline); @@ -1377,9 +1384,11 @@ impl LocalPtyProcess { self.abandon_termination(); return false; } - self.group_escalation_complete.store(true, Ordering::Release); + let escalation_changed = !self.group_escalation_complete.swap(true, Ordering::AcqRel); self.exited.1.notify_all(); - crate::process_session::wake_child_reaper(); + if escalation_changed { + crate::process_session::wake_child_reaper(); + } } #[cfg(not(unix))] { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index fcf32ca56584..ca6f2d3de9f9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2976,10 +2976,10 @@ mod unix { self.termination_started.swap(true, Ordering::AcqRel) }; self.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); if already_started { return; } + crate::process_session::wake_child_reaper(); let worker = self.clone(); if thread::Builder::new() .name("terminal-host-terminate".into()) @@ -2993,15 +2993,19 @@ mod unix { } fn finish_group_escalation(&self) { - self.group_escalation_complete.store(true, Ordering::Release); + let changed = !self.group_escalation_complete.swap(true, Ordering::AcqRel); self.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); + if changed { + crate::process_session::wake_child_reaper(); + } } fn abandon_group_escalation(&self) { - self.termination_started.store(false, Ordering::Release); + let changed = self.termination_started.swap(false, Ordering::AcqRel); self.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); + if changed { + crate::process_session::wake_child_reaper(); + } } fn publish_exit_if_drained(&self) { @@ -3019,16 +3023,19 @@ mod unix { } fn terminate_and_wait(&self) { - let (child_reaped, wait_ownership_lost) = { + let (termination_changed, child_reaped, wait_ownership_lost) = { let _signal = self.child_signal_lock.lock().unwrap(); - self.termination_started.store(true, Ordering::Release); + let termination_changed = !self.termination_started.swap(true, Ordering::AcqRel); ( + termination_changed, self.child_reaped.load(Ordering::Acquire), self.child_wait_ownership_lost.load(Ordering::Acquire), ) }; self.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); + if termination_changed { + crate::process_session::wake_child_reaper(); + } if wait_ownership_lost { self.abandon_group_escalation(); return; @@ -3571,9 +3578,11 @@ mod unix { // The reader publishes every final PTY byte before declaring the // stream drained. Exit is emitted only after this flag and the // child wait rendezvous, so clients can safely stop at Exit. - reader_host.pty_drained.store(true, Ordering::Release); + let drain_changed = !reader_host.pty_drained.swap(true, Ordering::AcqRel); reader_host.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); + if drain_changed { + crate::process_session::wake_child_reaper(); + } reader_host.publish_exit_if_drained(); })?; let observe_host = shared.clone(); From e38e31c807a28d53410baf06fdd2dbf67c1dba9a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 02:03:01 -0700 Subject: [PATCH 232/521] test(tui): expose cross-session reaper wake --- .../cmux-tui-core/src/process_session.rs | 63 ++++++++++++++++--- 1 file changed, 53 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 338bc1a3a238..4cfe54f492a6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1727,14 +1727,14 @@ mod tests { } #[cfg(any(target_os = "macos", target_os = "linux"))] - #[test] - fn state_change_wake_preempts_degraded_retry_before_shutdown_deadline() { - let now = Instant::now(); - let shutdown_deadline = now + Duration::from_secs(5); + fn degraded_reap_request_for_test( + session: libc::pid_t, + next_attempt: Instant, + degraded: Arc, + ) -> NaturalReapRequest { let (sender, _receiver) = mpsc::channel(); - let degraded = Arc::new(AtomicUsize::new(1)); - let mut pending = vec![NaturalReapRequest { - session: 41, + NaturalReapRequest { + session, cleanup_timeout: Duration::ZERO, observe_child: Box::new(|| Ok(false)), child_observed: false, @@ -1746,14 +1746,36 @@ mod tests { active: Arc::new(AtomicUsize::new(1)), degraded: degraded.clone(), }, - next_attempt: now + Duration::from_secs(30), + next_attempt, retry_delay: NATURAL_REAP_RETRY_MAX, attempts: NATURAL_REAP_MAX_ATTEMPTS, degraded: true, - }]; + } + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn accept_session_state_change_for_test( + _session: libc::pid_t, + pending: &mut Vec, + wake_pending: &AtomicBool, + ) { + accept_natural_reaper_command(NaturalReaperCommand::Wake, pending, wake_pending); + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn state_change_wake_preempts_degraded_retry_before_shutdown_deadline() { + let now = Instant::now(); + let shutdown_deadline = now + Duration::from_secs(5); + let degraded = Arc::new(AtomicUsize::new(1)); + let mut pending = vec![degraded_reap_request_for_test( + 41, + now + Duration::from_secs(30), + degraded.clone(), + )]; let wake_pending = AtomicBool::new(true); - accept_natural_reaper_command(NaturalReaperCommand::Wake, &mut pending, &wake_pending); + accept_session_state_change_for_test(41, &mut pending, &wake_pending); assert!( pending[0].next_attempt <= shutdown_deadline, @@ -1771,6 +1793,27 @@ mod tests { ); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn state_change_wake_preserves_unrelated_degraded_backoff() { + let now = Instant::now(); + let shutdown_deadline = now + Duration::from_secs(5); + let degraded = Arc::new(AtomicUsize::new(2)); + let mut pending = vec![ + degraded_reap_request_for_test(41, now + Duration::from_secs(30), degraded.clone()), + degraded_reap_request_for_test(73, now + Duration::from_secs(30), degraded.clone()), + ]; + let wake_pending = AtomicBool::new(true); + + accept_session_state_change_for_test(41, &mut pending, &wake_pending); + + assert!(pending[0].next_attempt <= shutdown_deadline); + assert!( + pending[1].next_attempt > shutdown_deadline, + "one session state change defeated another degraded session's backoff" + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn ignored_sigchld_fails_before_reserving_a_child_reaper() { From 904076ccbaf17c847068516bce95bd4cf339fd46 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 02:06:03 -0700 Subject: [PATCH 233/521] fix(tui): target shutdown reaper wakes --- .../cmux-tui-core/src/process_session.rs | 59 +++++++------------ cmux-tui/crates/cmux-tui-core/src/surface.rs | 18 ++++-- .../src/terminal_host_runtime.rs | 19 ++++-- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 4cfe54f492a6..2f3e458d0773 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -126,7 +126,7 @@ enum NaturalChildObservation { enum NaturalReaperCommand { Add(NaturalReapRequest), - Wake, + WakeSession(libc::pid_t), } struct NaturalReapRequest { @@ -185,7 +185,6 @@ struct NaturalReaper { sender: mpsc::Sender, active: Arc, degraded: Arc, - wake_pending: Arc, _worker: std::thread::JoinHandle<()>, } @@ -206,18 +205,16 @@ impl NaturalReaper { let (sender, receiver) = mpsc::channel(); let active = Arc::new(AtomicUsize::new(0)); let degraded = Arc::new(AtomicUsize::new(0)); - let wake_pending = Arc::new(AtomicBool::new(false)); - let worker_wake_pending = wake_pending.clone(); let worker = std::thread::Builder::new().name("cmux-pty-session-reaper".into()).spawn( move || { #[cfg(test)] NATURAL_REAP_WORKERS.fetch_add(1, Ordering::AcqRel); - run_natural_reaper(receiver, worker_wake_pending); + run_natural_reaper(receiver); #[cfg(test)] NATURAL_REAP_WORKERS.fetch_sub(1, Ordering::AcqRel); }, )?; - Ok(Self { sender, active, degraded, wake_pending, _worker: worker }) + Ok(Self { sender, active, degraded, _worker: worker }) } fn lease(&self) -> io::Result { @@ -298,18 +295,16 @@ fn require_waitable_child_disposition() -> io::Result<()> { Ok(()) } -/// Wake every reserved child after a cleanup-relevant state transition. +/// Wake one reserved child after a cleanup-relevant state transition. /// -/// This deliberately preempts degraded retry delays once. Callers must emit -/// it only when the state actually changed; a failed attempt keeps its normal -/// degraded backoff. -pub(crate) fn wake_child_reaper() { +/// This deliberately preempts that session's degraded retry delay once. +/// Callers must emit it only when the state actually changed; a failed attempt +/// keeps its normal degraded backoff. +pub(crate) fn wake_child_reaper(session: libc::pid_t) { let Some(slot) = NATURAL_REAPER.get() else { return }; let slot = slot.lock().unwrap(); let Some(reaper) = slot.as_ref() else { return }; - if !reaper.wake_pending.swap(true, Ordering::AcqRel) { - let _ = reaper.sender.send(NaturalReaperCommand::Wake); - } + let _ = reaper.sender.send(NaturalReaperCommand::WakeSession(session)); } pub(crate) fn enqueue_reserved_session_leader( @@ -370,10 +365,7 @@ pub(crate) fn poll_reserved_session_leader( true } -fn run_natural_reaper( - receiver: mpsc::Receiver, - wake_pending: Arc, -) { +fn run_natural_reaper(receiver: mpsc::Receiver) { let mut pending = Vec::::new(); loop { let received = if pending.is_empty() { @@ -389,7 +381,7 @@ fn run_natural_reaper( }; match received { Ok(command) => { - accept_natural_reaper_command(command, &mut pending, wake_pending.as_ref()); + accept_natural_reaper_command(command, &mut pending); } Err(RecvTimeoutError::Timeout) => {} Err(RecvTimeoutError::Disconnected) if pending.is_empty() => return, @@ -402,7 +394,7 @@ fn run_natural_reaper( while let Some(remaining) = batch_deadline.checked_duration_since(Instant::now()) { match receiver.recv_timeout(remaining) { Ok(command) => { - accept_natural_reaper_command(command, &mut pending, wake_pending.as_ref()); + accept_natural_reaper_command(command, &mut pending); } Err(RecvTimeoutError::Timeout | RecvTimeoutError::Disconnected) => break, } @@ -605,14 +597,12 @@ fn kill_sessions_until_only_leaders_from_snapshot( fn accept_natural_reaper_command( command: NaturalReaperCommand, pending: &mut Vec, - wake_pending: &AtomicBool, ) { match command { NaturalReaperCommand::Add(request) => pending.push(request), - NaturalReaperCommand::Wake => { - wake_pending.store(false, Ordering::Release); + NaturalReaperCommand::WakeSession(session) => { let now = Instant::now(); - for request in pending { + for request in pending.iter_mut().filter(|request| request.session == session) { request.next_attempt = now; } } @@ -1676,7 +1666,7 @@ mod tests { let retried_while_degraded = attempt_receiver.recv_timeout(Duration::from_millis(250)).is_ok(); release.store(true, Ordering::Release); - wake_child_reaper(); + wake_child_reaper(session); finished_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); assert!( @@ -1717,7 +1707,7 @@ mod tests { let retried_while_degraded = attempt_receiver.recv_timeout(Duration::from_millis(250)).is_ok(); release.store(true, Ordering::Release); - wake_child_reaper(); + wake_child_reaper(session); finished_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); assert!( @@ -1744,7 +1734,7 @@ mod tests { _lease: ReservedChildReaperLease { sender, active: Arc::new(AtomicUsize::new(1)), - degraded: degraded.clone(), + degraded, }, next_attempt, retry_delay: NATURAL_REAP_RETRY_MAX, @@ -1755,11 +1745,10 @@ mod tests { #[cfg(any(target_os = "macos", target_os = "linux"))] fn accept_session_state_change_for_test( - _session: libc::pid_t, + session: libc::pid_t, pending: &mut Vec, - wake_pending: &AtomicBool, ) { - accept_natural_reaper_command(NaturalReaperCommand::Wake, pending, wake_pending); + accept_natural_reaper_command(NaturalReaperCommand::WakeSession(session), pending); } #[cfg(any(target_os = "macos", target_os = "linux"))] @@ -1773,9 +1762,7 @@ mod tests { now + Duration::from_secs(30), degraded.clone(), )]; - let wake_pending = AtomicBool::new(true); - - accept_session_state_change_for_test(41, &mut pending, &wake_pending); + accept_session_state_change_for_test(41, &mut pending); assert!( pending[0].next_attempt <= shutdown_deadline, @@ -1801,11 +1788,9 @@ mod tests { let degraded = Arc::new(AtomicUsize::new(2)); let mut pending = vec![ degraded_reap_request_for_test(41, now + Duration::from_secs(30), degraded.clone()), - degraded_reap_request_for_test(73, now + Duration::from_secs(30), degraded.clone()), + degraded_reap_request_for_test(73, now + Duration::from_secs(30), degraded), ]; - let wake_pending = AtomicBool::new(true); - - accept_session_state_change_for_test(41, &mut pending, &wake_pending); + accept_session_state_change_for_test(41, &mut pending); assert!(pending[0].next_attempt <= shutdown_deadline); assert!( diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 55469e14500b..a0ba5bd83d1c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1131,13 +1131,21 @@ impl LocalPtyProcess { self.killer.lock().unwrap().kill().is_ok() } + #[cfg(unix)] + fn wake_reserved_child_reaper(&self) { + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return; + }; + crate::process_session::wake_child_reaper(session); + } + fn mark_pty_drained(&self) { #[cfg(unix)] { let changed = !self.pty_drained.swap(true, Ordering::AcqRel); self.exited.1.notify_all(); if changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } } } @@ -1183,7 +1191,7 @@ impl LocalPtyProcess { let changed = self.termination_started.swap(false, Ordering::AcqRel); self.exited.1.notify_all(); if changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } } @@ -1345,7 +1353,7 @@ impl LocalPtyProcess { }; self.exited.1.notify_all(); if termination_changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } if wait_ownership_lost { self.abandon_termination(); @@ -1387,7 +1395,7 @@ impl LocalPtyProcess { let escalation_changed = !self.group_escalation_complete.swap(true, Ordering::AcqRel); self.exited.1.notify_all(); if escalation_changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } } #[cfg(not(unix))] @@ -4649,7 +4657,7 @@ mod tests { process.group_escalation_complete.store(true, Ordering::Release); process.exited.1.notify_all(); - crate::process_session::wake_child_reaper(); + process.wake_reserved_child_reaper(); let reap_deadline = Instant::now() + Duration::from_secs(2); while !process.child_reaped.load(Ordering::Acquire) && Instant::now() < reap_deadline { std::thread::sleep(Duration::from_millis(10)); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index ca6f2d3de9f9..91a5316820a4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2967,6 +2967,13 @@ mod unix { let _ = self.pty_drain_waker.lock().unwrap().write_all(&[1]); } + fn wake_reserved_child_reaper(&self) { + let Some(session) = self.pid.and_then(|pid| libc::pid_t::try_from(pid).ok()) else { + return; + }; + crate::process_session::wake_child_reaper(session); + } + fn request_termination(self: &Arc) { let already_started = { // Serialize the ownership transition with WNOWAIT's final @@ -2979,7 +2986,7 @@ mod unix { if already_started { return; } - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); let worker = self.clone(); if thread::Builder::new() .name("terminal-host-terminate".into()) @@ -2996,7 +3003,7 @@ mod unix { let changed = !self.group_escalation_complete.swap(true, Ordering::AcqRel); self.child_exit.1.notify_all(); if changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } } @@ -3004,7 +3011,7 @@ mod unix { let changed = self.termination_started.swap(false, Ordering::AcqRel); self.child_exit.1.notify_all(); if changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } } @@ -3034,7 +3041,7 @@ mod unix { }; self.child_exit.1.notify_all(); if termination_changed { - crate::process_session::wake_child_reaper(); + self.wake_reserved_child_reaper(); } if wait_ownership_lost { self.abandon_group_escalation(); @@ -3581,7 +3588,7 @@ mod unix { let drain_changed = !reader_host.pty_drained.swap(true, Ordering::AcqRel); reader_host.child_exit.1.notify_all(); if drain_changed { - crate::process_session::wake_child_reaper(); + reader_host.wake_reserved_child_reaper(); } reader_host.publish_exit_if_drained(); })?; @@ -5267,7 +5274,7 @@ mod unix { host.group_escalation_complete.store(true, Ordering::Release); host.child_exit.1.notify_all(); - crate::process_session::wake_child_reaper(); + host.wake_reserved_child_reaper(); let _ = host.wait_for_child_exit(Duration::from_secs(2)); let _ = fs::remove_dir_all(root); From ae334214b656caae2cdeb29c25ea4c245a689eb2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:14:56 -0700 Subject: [PATCH 234/521] test(tui): expose worker-start child leak --- .../cmux-tui-core/src/process_session.rs | 7 ++ cmux-tui/crates/cmux-tui-core/src/surface.rs | 88 ++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2f3e458d0773..4bdbea777da4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -48,6 +48,13 @@ pub(crate) fn dedicated_natural_reap_workers_for_test() -> usize { NATURAL_REAP_WORKERS.load(Ordering::Acquire) } +#[cfg(test)] +pub(crate) fn reserved_child_reaper_active_for_test() -> usize { + let Some(slot) = NATURAL_REAPER.get() else { return 0 }; + let slot = slot.lock().unwrap(); + slot.as_ref().map_or(0, |reaper| reaper.active.load(Ordering::Acquire)) +} + #[cfg(test)] pub(crate) fn set_process_session_preflight_failure_for_test(enabled: bool) { FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.set(enabled); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index a0ba5bd83d1c..6e67d415717d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -920,6 +920,12 @@ type LocalChildReaperLease = crate::process_session::ReservedChildReaperLease; #[cfg(all(unix, test))] static DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS: AtomicUsize = AtomicUsize::new(0); +#[cfg(all(unix, test))] +static FAIL_NEXT_FRAME_PRODUCER_SPAWN: AtomicBool = AtomicBool::new(false); + +#[cfg(all(unix, test))] +static FAIL_NEXT_LOCAL_READER_SPAWN: AtomicBool = AtomicBool::new(false); + #[cfg(all(unix, test))] fn dedicated_local_child_observer_spawns_for_test() -> usize { DEDICATED_LOCAL_CHILD_OBSERVER_SPAWNS.load(Ordering::Acquire) @@ -1938,7 +1944,7 @@ impl Surface { } // PTY reader: pty bytes -> terminal state -> SurfaceOutput events. - let reader = std::thread::Builder::new().name(format!("surface-{id}-reader")).spawn({ + let reader = spawn_local_pty_reader(id, { let surface = surface.clone(); move || { let mut buf = [0u8; 64 * 1024]; @@ -4334,6 +4340,10 @@ fn terminal_color_override_delta( const RENDER_FRAME_CADENCE: Duration = Duration::from_millis(8); fn spawn_frame_producer(surface: &Arc, requests: Receiver) -> anyhow::Result<()> { + #[cfg(all(unix, test))] + if FAIL_NEXT_FRAME_PRODUCER_SPAWN.swap(false, Ordering::AcqRel) { + return Err(std::io::Error::other("forced frame producer spawn failure").into()); + } let weak = Arc::downgrade(surface); let id = surface.id; std::thread::Builder::new().name(format!("surface-{id}-frames")).spawn(move || { @@ -4371,6 +4381,17 @@ fn spawn_frame_producer(surface: &Arc, requests: Receiver) -> anyh Ok(()) } +fn spawn_local_pty_reader( + id: SurfaceId, + run: impl FnOnce() + Send + 'static, +) -> std::io::Result> { + #[cfg(all(unix, test))] + if FAIL_NEXT_LOCAL_READER_SPAWN.swap(false, Ordering::AcqRel) { + return Err(std::io::Error::other("forced local PTY reader spawn failure")); + } + std::thread::Builder::new().name(format!("surface-{id}-reader")).spawn(run) +} + fn broadcast_render_scroll_locked(pty: &PtySurface, position: (u64, bool)) { let (offset, at_bottom) = position; let mut render = pty.render.lock().unwrap(); @@ -4555,6 +4576,71 @@ mod tests { ); } + #[cfg(unix)] + fn assert_failed_local_worker_start_releases_reaper( + child_env: &str, + test_name: &str, + failure: &AtomicBool, + expected_error: &str, + ) { + if std::env::var_os(child_env).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", test_name]) + .env(child_env, "1") + .status() + .unwrap(); + assert!(status.success(), "worker-start cleanup subprocess failed: {status}"); + return; + } + + drop(reserve_local_child_reaper().unwrap()); + let baseline = crate::process_session::reserved_child_reaper_active_for_test(); + failure.store(true, Ordering::Release); + let mux = Mux::new_for_test("local-worker-start-failure", SurfaceOptions::default()); + let options = SurfaceOptions { + command: Some(vec!["/bin/sleep".into(), "60".into()]), + ..SurfaceOptions::default() + }; + + let error = Surface::spawn(1, options, Arc::downgrade(&mux)) + .expect_err("forced local worker-start failure unexpectedly succeeded"); + assert!(error.to_string().contains(expected_error), "unexpected spawn error: {error:#}"); + + let deadline = Instant::now() + Duration::from_secs(4); + while crate::process_session::reserved_child_reaper_active_for_test() != baseline + && Instant::now() < deadline + { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!( + crate::process_session::reserved_child_reaper_active_for_test(), + baseline, + "failed local worker startup retained a child-reaper reservation" + ); + } + + #[cfg(unix)] + #[test] + fn failed_frame_producer_start_releases_local_child() { + assert_failed_local_worker_start_releases_reaper( + "CMUX_TUI_TEST_FRAME_PRODUCER_START_FAILURE", + "surface::tests::failed_frame_producer_start_releases_local_child", + &FAIL_NEXT_FRAME_PRODUCER_SPAWN, + "forced frame producer spawn failure", + ); + } + + #[cfg(unix)] + #[test] + fn failed_pty_reader_start_releases_local_child() { + assert_failed_local_worker_start_releases_reaper( + "CMUX_TUI_TEST_PTY_READER_START_FAILURE", + "surface::tests::failed_pty_reader_start_releases_local_child", + &FAIL_NEXT_LOCAL_READER_SPAWN, + "forced local PTY reader spawn failure", + ); + } + #[derive(Clone, Default)] struct CapturingWriter(Arc>>); From 531099932ca465e48f319c355ad20329e1b5a1c7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:17:33 -0700 Subject: [PATCH 235/521] fix(tui): retain child through worker startup --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 22 +++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 6e67d415717d..22b78afe6da2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -693,6 +693,7 @@ impl LocalProcess { } } + #[cfg(any(not(unix), test))] fn terminate_and_wait(&self, deadline: Instant) -> bool { match self { Self::Owned(process) => process.terminate_and_wait(deadline), @@ -804,6 +805,7 @@ enum SurfaceShutdownOwnerKind { } impl SurfaceShutdownOwner { + #[cfg(any(not(unix), test))] pub(crate) fn terminate_until(&self, deadline: Instant) -> bool { match &self.kind { SurfaceShutdownOwnerKind::Local(process) => process.terminate_and_wait(deadline), @@ -1323,6 +1325,7 @@ impl LocalPtyProcess { true } + #[cfg(any(not(unix), test))] fn terminate_and_wait(&self, deadline: Instant) -> bool { #[cfg(unix)] let snapshot = None; @@ -1934,14 +1937,7 @@ impl Surface { frame_requests, })); - #[cfg(unix)] - process.spawn_reaper(child)?; - #[cfg(not(unix))] - process.spawn_reaper(child, reaper)?; - if let Err(error) = spawn_frame_producer(&surface, frame_rx) { - let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); - return Err(error); - } + spawn_frame_producer(&surface, frame_rx)?; // PTY reader: pty bytes -> terminal state -> SurfaceOutput events. let reader = spawn_local_pty_reader(id, { @@ -2035,11 +2031,12 @@ impl Surface { } } }); - if let Err(error) = reader { - let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(2)); - return Err(error.into()); - } + reader?; + #[cfg(unix)] + process.spawn_reaper(child)?; + #[cfg(not(unix))] + process.spawn_reaper(child, reaper)?; Ok(surface) } @@ -3689,6 +3686,7 @@ impl Surface { } } + #[cfg(test)] pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { self.shutdown_owner().is_none_or(|owner| owner.terminate_until(deadline)) } From efc08d969e8a1d1627424c0a6f6e7326d89fea26 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:20:06 -0700 Subject: [PATCH 236/521] test(tui): expose blocking child cleanup --- .../cmux-tui-core/src/spawned_pty_child.rs | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 273b3a970cae..7b525d1f4632 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -158,3 +158,102 @@ impl Drop for SpawnedPtyChild { let _ = child.wait(); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::{Duration, Instant}; + + #[derive(Debug)] + struct FailedKiller; + + impl portable_pty::ChildKiller for FailedKiller { + fn kill(&mut self) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "forced child kill failure", + )) + } + + fn clone_killer(&self) -> Box { + Box::new(Self) + } + } + + #[derive(Debug)] + struct BlockingWaitChild { + try_waits: Arc, + wait_called: Arc, + } + + impl portable_pty::ChildKiller for BlockingWaitChild { + fn kill(&mut self) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "forced child kill failure", + )) + } + + fn clone_killer(&self) -> Box { + Box::new(FailedKiller) + } + } + + impl portable_pty::Child for BlockingWaitChild { + fn try_wait(&mut self) -> std::io::Result> { + self.try_waits.fetch_add(1, Ordering::AcqRel); + Ok(None) + } + + fn wait(&mut self) -> std::io::Result { + self.wait_called.store(true, Ordering::Release); + loop { + std::thread::park(); + } + } + + fn process_id(&self) -> Option { + None + } + } + + #[test] + fn failed_kill_does_not_block_spawned_child_drop() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_NONBLOCKING_CHILD_DROP"; + const TEST_NAME: &str = + "spawned_pty_child::tests::failed_kill_does_not_block_spawned_child_drop"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "nonblocking child-drop subprocess failed: {status}"); + return; + } + + let try_waits = Arc::new(AtomicUsize::new(0)); + let wait_called = Arc::new(AtomicBool::new(false)); + let child = SpawnedPtyChild::new(Box::new(BlockingWaitChild { + try_waits: try_waits.clone(), + wait_called: wait_called.clone(), + })); + let (dropped_sender, dropped_receiver) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + drop(child); + let _ = dropped_sender.send(()); + }); + + dropped_receiver + .recv_timeout(Duration::from_millis(250)) + .expect("spawned PTY child Drop blocked after kill failed"); + let deadline = Instant::now() + Duration::from_millis(250); + while try_waits.load(Ordering::Acquire) == 0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + assert!(try_waits.load(Ordering::Acquire) > 0, "cleanup owner never polled the child"); + assert!(!wait_called.load(Ordering::Acquire), "cleanup owner called blocking child wait"); + } +} From e50ad84ca038c61f34f11ed0450dc7e42ed66e0f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 03:26:06 -0700 Subject: [PATCH 237/521] fix(tui): retain failed portable children --- .../cmux-tui-core/src/spawned_pty_child.rs | 235 ++++++++++++++++-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 12 +- 2 files changed, 215 insertions(+), 32 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 7b525d1f4632..7a40c67fca24 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -1,10 +1,175 @@ //! Immediate ownership of a spawned PTY child until its reaper completes. -#[cfg(unix)] -use std::time::Duration; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; #[cfg(unix)] const FAILED_SPAWN_SESSION_CLEANUP_TIMEOUT: Duration = Duration::from_secs(2); +const PORTABLE_CHILD_REAP_CAPACITY: usize = 4_096; +const PORTABLE_CHILD_REAP_RETRY_INITIAL: Duration = Duration::from_millis(25); +const PORTABLE_CHILD_REAP_RETRY_MAX: Duration = Duration::from_secs(1); + +static PORTABLE_CHILD_REAPER: OnceLock>> = OnceLock::new(); +static STRANDED_PORTABLE_CHILDREN: OnceLock>> = OnceLock::new(); + +struct PortableChildReaper { + sender: mpsc::Sender, + active: Arc, + _worker: std::thread::JoinHandle<()>, +} + +pub(crate) struct ReservedPortableChildReaperLease { + sender: mpsc::Sender, + active: Arc, +} + +impl Drop for ReservedPortableChildReaperLease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +struct PortableChildReapRequest { + child: Box, + _lease: ReservedPortableChildReaperLease, + next_attempt: Instant, + retry_delay: Duration, +} + +struct StrandedPortableChild { + _child: Box, + _lease: Option, +} + +impl PortableChildReapRequest { + fn schedule_retry(&mut self) { + self.next_attempt = Instant::now() + self.retry_delay; + self.retry_delay = (self.retry_delay * 2).min(PORTABLE_CHILD_REAP_RETRY_MAX); + } +} + +impl PortableChildReaper { + fn start() -> std::io::Result { + let (sender, receiver) = mpsc::channel(); + let active = Arc::new(AtomicUsize::new(0)); + let worker = std::thread::Builder::new() + .name("cmux-portable-child-reaper".into()) + .spawn(move || run_portable_child_reaper(receiver))?; + Ok(Self { sender, active, _worker: worker }) + } + + fn lease(&self) -> std::io::Result { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < PORTABLE_CHILD_REAP_CAPACITY).then_some(active + 1) + }) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "portable child reaper capacity exhausted", + ) + })?; + Ok(ReservedPortableChildReaperLease { + sender: self.sender.clone(), + active: self.active.clone(), + }) + } +} + +pub(crate) fn reserve_portable_child_reaper() -> std::io::Result { + let mut slot = PORTABLE_CHILD_REAPER.get_or_init(|| Mutex::new(None)).lock().unwrap(); + if slot.is_none() { + *slot = Some(PortableChildReaper::start()?); + } + slot.as_ref().expect("portable child reaper initialized").lease() +} + +#[cfg(test)] +fn portable_child_reaper_active_for_test() -> usize { + let Some(slot) = PORTABLE_CHILD_REAPER.get() else { return 0 }; + let slot = slot.lock().unwrap(); + slot.as_ref().map_or(0, |reaper| reaper.active.load(Ordering::Acquire)) +} + +fn enqueue_portable_child_cleanup( + child: Box, + lease: ReservedPortableChildReaperLease, +) { + let sender = lease.sender.clone(); + let request = PortableChildReapRequest { + child, + _lease: lease, + next_attempt: Instant::now(), + retry_delay: PORTABLE_CHILD_REAP_RETRY_INITIAL, + }; + if let Err(mpsc::SendError(request)) = sender.send(request) { + retain_stranded_portable_child(request.child, Some(request._lease)); + } +} + +fn retain_stranded_portable_child( + mut child: Box, + lease: Option, +) { + // A Windows child always reserves the shared worker before spawn, so this + // path requires the already-started worker to have failed unexpectedly. + // Retain the handle without calling blocking wait; bounded capacity then + // prevents later spawns from silently compounding the ownership failure. + let _ = child.kill(); + STRANDED_PORTABLE_CHILDREN + .get_or_init(|| Mutex::new(Vec::new())) + .lock() + .unwrap() + .push(StrandedPortableChild { _child: child, _lease: lease }); +} + +fn run_portable_child_reaper(receiver: mpsc::Receiver) { + let mut pending = Vec::::new(); + loop { + let received = if pending.is_empty() { + receiver.recv().map_err(|_| RecvTimeoutError::Disconnected) + } else { + let now = Instant::now(); + let wait = pending + .iter() + .map(|request| request.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(PORTABLE_CHILD_REAP_RETRY_MAX); + receiver.recv_timeout(wait) + }; + match received { + Ok(request) => pending.push(request), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) if pending.is_empty() => return, + Err(RecvTimeoutError::Disconnected) => { + std::thread::sleep(PORTABLE_CHILD_REAP_RETRY_MAX); + } + } + while let Ok(request) = receiver.try_recv() { + pending.push(request); + } + + let now = Instant::now(); + let mut index = 0; + while index < pending.len() { + if pending[index].next_attempt > now { + index += 1; + continue; + } + let mut request = pending.swap_remove(index); + let _ = request.child.kill(); + match request.child.try_wait() { + Ok(Some(_)) => {} + Ok(None) | Err(_) => { + request.schedule_retry(); + pending.push(request); + } + } + } + } +} #[cfg(unix)] fn signal_owned_child_for_cleanup(session: libc::pid_t) -> std::io::Result<()> { @@ -29,6 +194,7 @@ pub(crate) struct SpawnedPtyChild { child: Option>, #[cfg(unix)] reaper: Option, + portable_reaper: Option, } impl SpawnedPtyChild { @@ -37,6 +203,7 @@ impl SpawnedPtyChild { child: Some(child), #[cfg(unix)] reaper: None, + portable_reaper: None, } } @@ -49,6 +216,15 @@ impl SpawnedPtyChild { self } + #[cfg(not(unix))] + pub(crate) fn with_reaper(mut self, reaper: ReservedPortableChildReaperLease) -> Self { + assert!( + self.portable_reaper.replace(reaper).is_none(), + "spawned PTY child already has a reaper" + ); + self + } + #[cfg(unix)] pub(crate) fn take_reaper( &mut self, @@ -91,7 +267,7 @@ fn enqueue_failed_spawn_cleanup( // shared reaper enumerates and removes every other session member. let _ = signal_owned_child_for_cleanup(session); - let (finished_sender, finished_receiver) = std::sync::mpsc::sync_channel(1); + let (finished_sender, finished_receiver) = mpsc::sync_channel(1); let mut finished_sender = Some(finished_sender); let mut child = Some(child); crate::process_session::enqueue_reserved_session_leader( @@ -137,25 +313,21 @@ impl Drop for SpawnedPtyChild { fn drop(&mut self) { let Some(child) = self.child.take() else { return }; #[cfg(unix)] - let child = match self.reaper.take() { - Some(reaper) => { - let Some(session) = - child.process_id().and_then(|pid| libc::pid_t::try_from(pid).ok()) - else { - drop(reaper); - let mut child = child; - let _ = child.kill(); - let _ = child.wait(); - return; - }; + if let Some(reaper) = self.reaper.take() { + if let Some(session) = + child.process_id().and_then(|pid| libc::pid_t::try_from(pid).ok()) + { enqueue_failed_spawn_cleanup(child, reaper, session); return; } - None => child, - }; - let mut child = child; - let _ = child.kill(); - let _ = child.wait(); + drop(reaper); + } + let reaper = self.portable_reaper.take().or_else(|| reserve_portable_child_reaper().ok()); + if let Some(reaper) = reaper { + enqueue_portable_child_cleanup(child, reaper); + } else { + retain_stranded_portable_child(child, None); + } } } @@ -203,8 +375,8 @@ mod tests { impl portable_pty::Child for BlockingWaitChild { fn try_wait(&mut self) -> std::io::Result> { - self.try_waits.fetch_add(1, Ordering::AcqRel); - Ok(None) + let attempt = self.try_waits.fetch_add(1, Ordering::AcqRel) + 1; + Ok((attempt >= 2).then(|| portable_pty::ExitStatus::with_exit_code(1))) } fn wait(&mut self) -> std::io::Result { @@ -217,6 +389,11 @@ mod tests { fn process_id(&self) -> Option { None } + + #[cfg(windows)] + fn as_raw_handle(&self) -> Option { + None + } } #[test] @@ -240,7 +417,7 @@ mod tests { try_waits: try_waits.clone(), wait_called: wait_called.clone(), })); - let (dropped_sender, dropped_receiver) = std::sync::mpsc::sync_channel(1); + let (dropped_sender, dropped_receiver) = mpsc::sync_channel(1); std::thread::spawn(move || { drop(child); let _ = dropped_sender.send(()); @@ -249,11 +426,19 @@ mod tests { dropped_receiver .recv_timeout(Duration::from_millis(250)) .expect("spawned PTY child Drop blocked after kill failed"); - let deadline = Instant::now() + Duration::from_millis(250); - while try_waits.load(Ordering::Acquire) == 0 && Instant::now() < deadline { + let deadline = Instant::now() + Duration::from_millis(500); + while (try_waits.load(Ordering::Acquire) < 2 + || portable_child_reaper_active_for_test() != 0) + && Instant::now() < deadline + { std::thread::sleep(Duration::from_millis(5)); } - assert!(try_waits.load(Ordering::Acquire) > 0, "cleanup owner never polled the child"); + assert!(try_waits.load(Ordering::Acquire) >= 2, "cleanup owner did not retry the child"); + assert_eq!( + portable_child_reaper_active_for_test(), + 0, + "cleanup owner retained its reservation after the child exited" + ); assert!(!wait_called.load(Ordering::Acquire), "cleanup owner called blocking child wait"); } } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 22b78afe6da2..1045d02ad868 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -934,7 +934,7 @@ fn dedicated_local_child_observer_spawns_for_test() -> usize { } #[cfg(not(unix))] -struct LocalChildReaperLease; +type LocalChildReaperLease = crate::spawned_pty_child::ReservedPortableChildReaperLease; #[cfg(unix)] type LocalProcessSnapshot<'a> = Option<&'a crate::process_session::SessionProcessSnapshot>; @@ -949,7 +949,7 @@ fn reserve_local_child_reaper() -> std::io::Result { #[cfg(not(unix))] fn reserve_local_child_reaper() -> std::io::Result { - Ok(LocalChildReaperLease) + crate::spawned_pty_child::reserve_portable_child_reaper() } impl LocalPtyProcess { @@ -1093,7 +1093,6 @@ impl LocalPtyProcess { fn spawn_reaper( self: &Arc, child: crate::spawned_pty_child::SpawnedPtyChild, - _reaper: LocalChildReaperLease, ) -> std::io::Result<()> { let process = self.clone(); let (child_sender, child_receiver) = @@ -1844,9 +1843,8 @@ impl Surface { cmd.cwd(cwd); } - let child = crate::spawned_pty_child::SpawnedPtyChild::new(pty.slave.spawn_command(cmd)?); - #[cfg(unix)] - let child = child.with_reaper(reaper); + let child = crate::spawned_pty_child::SpawnedPtyChild::new(pty.slave.spawn_command(cmd)?) + .with_reaper(reaper); drop(process_creation); let pid = child.process_id(); drop(pty.slave); @@ -2036,7 +2034,7 @@ impl Surface { #[cfg(unix)] process.spawn_reaper(child)?; #[cfg(not(unix))] - process.spawn_reaper(child, reaper)?; + process.spawn_reaper(child)?; Ok(surface) } From a6cc4508924ffe17cd9349e68e93e4ea34149b47 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:42:50 -0700 Subject: [PATCH 238/521] test(tui): expose serial startup adoption --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 121 ++++++++++++++++++++++- 1 file changed, 118 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d86d9d1e2594..81cc37f902f1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2070,15 +2070,30 @@ impl Mux { // enough for healthy hosts; live or indeterminate failures // continue on the asynchronous adoption loop instead of retrying // serially ahead of every later terminal. + #[cfg(test)] + let adoption_surface_factory = + self.terminal_adoption_surface_factory.lock().unwrap().clone(); + #[cfg(test)] + let adopted = if let Some(factory) = adoption_surface_factory { + factory(id) + } else { + Surface::adopt_hosted( + id, + options.clone(), + Arc::downgrade(self), + record.clone(), + record_path.clone(), + ) + }; + #[cfg(not(test))] let adopted = Surface::adopt_hosted( id, options.clone(), Arc::downgrade(self), record.clone(), record_path.clone(), - ) - .ok(); - let surface = match adopted { + ); + let surface = match adopted.ok() { Some(surface) => surface, None => { if terminal_host_record_liveness(&record_path, &record) @@ -14029,6 +14044,106 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn cold_start_terminal_adoption_never_handshakes_inline() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let root = std::env::temp_dir().join(format!( + "cmux-adoption-startup-budget-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let uid = std::fs::metadata(&root).unwrap().uid(); + let options = SurfaceOptions::default(); + let mux = Mux::new_for_test("adoption-startup-budget", options.clone()); + let workspace = + mux.create_empty_workspace(Some("startup-budget".into()), None, None).unwrap(); + { + let mut registry = mux.workspace_registry.lock().unwrap(); + for index in 0..3 { + let terminal_id = TerminalId::random().unwrap().to_hex(); + let incarnation = crate::terminal_host::HostIncarnation::random().unwrap().to_hex(); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + &format!("startup-adoption-reserve-{index}"), + &RegistryTerminal { + terminal_id: terminal_id.clone(), + workspace_key: workspace.key.clone(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({}), + exit: None, + }, + ) + .unwrap(); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: terminal_id.clone(), + incarnation, + endpoint: format!("/tmp/cmux-th-{uid}/{terminal_id}.sock"), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: workspace.key.clone(), + supports_set_defaults: false, + supports_terminate_only: false, + supports_clear_history: false, + }; + let path = root.join(format!("{terminal_id}.json")); + std::fs::write(&path, serde_json::to_vec(&record).unwrap()).unwrap(); + let mut permissions = std::fs::metadata(&path).unwrap().permissions(); + permissions.set_mode(0o600); + std::fs::set_permissions(path, permissions).unwrap(); + } + } + mux.update_surface_options(|surface_options| { + surface_options.terminal_host_root = Some(root.clone()); + }); + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = true; + + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release_rx = Arc::new(Mutex::new(release_rx)); + *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ + move |_| { + started_tx.send(()).unwrap(); + release_rx.lock().unwrap().recv().unwrap(); + anyhow::bail!("blocked startup handshake") + } + })); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let adoption = std::thread::spawn({ + let mux = mux.clone(); + move || { + done_tx.send(mux.adopt_terminal_hosts()).unwrap(); + } + }); + + let started_inline = started_rx.recv_timeout(Duration::from_millis(100)).is_ok(); + let completed_inline = done_rx.recv_timeout(Duration::from_millis(100)).ok(); + let completed_without_release = completed_inline.is_some(); + for _ in 0..3 { + release_tx.send(()).unwrap(); + } + let result = match completed_inline { + Some(result) => result, + None => done_rx.recv_timeout(Duration::from_secs(2)).unwrap(), + }; + adoption.join().unwrap(); + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = false; + mux.request_daemon_shutdown(); + let _ = std::fs::remove_dir_all(root); + + result.unwrap(); + assert!(!started_inline, "cold-start adoption entered a persisted host handshake inline"); + assert!( + completed_without_release, + "cold-start adoption waited for a persisted host handshake" + ); + } + #[cfg(unix)] #[test] fn terminal_adoption_overflow_uses_one_bounded_worker_pool() { From 7a1548f9e1a3650f658d4841fa81ac6cad99bc6f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:44:24 -0700 Subject: [PATCH 239/521] fix(tui): defer startup host handshakes --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 91 ++---------------------- 1 file changed, 4 insertions(+), 87 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 81cc37f902f1..632b8ef974de 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2056,94 +2056,11 @@ impl Mux { handled_terminals.insert(terminal_id.clone()); continue; } - let _creation = self.begin_surface_creation()?; - let mut owner_reservation = match self.reserve_surface_owner() { - Ok(reservation) => reservation, - Err(_) => { - handled_terminals.insert(terminal_id.clone()); - self.schedule_terminal_adoption(options.clone(), record, record_path); - continue; - } - }; - let id = self.next_id(); - // Bound startup head-of-line blocking per host. One handshake is - // enough for healthy hosts; live or indeterminate failures - // continue on the asynchronous adoption loop instead of retrying - // serially ahead of every later terminal. - #[cfg(test)] - let adoption_surface_factory = - self.terminal_adoption_surface_factory.lock().unwrap().clone(); - #[cfg(test)] - let adopted = if let Some(factory) = adoption_surface_factory { - factory(id) - } else { - Surface::adopt_hosted( - id, - options.clone(), - Arc::downgrade(self), - record.clone(), - record_path.clone(), - ) - }; - #[cfg(not(test))] - let adopted = Surface::adopt_hosted( - id, - options.clone(), - Arc::downgrade(self), - record.clone(), - record_path.clone(), - ); - let surface = match adopted.ok() { - Some(surface) => surface, - None => { - if terminal_host_record_liveness(&record_path, &record) - == TerminalHostLiveness::Dead - { - let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( - &record_path, - &record, - ); - self.mark_terminal_exited_and_materialize( - &terminal_id, - "terminal-host-proven-dead", - "host-process-ended-before-adoption", - &options, - )?; - handled_terminals.insert(terminal_id.clone()); - } else { - // Socket loss and descriptor pressure are not process - // death proof. Keep the capability and retry the same - // host rather than spawning a replacement shell. - handled_terminals.insert(terminal_id.clone()); - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; - } - }; - if self - .finish_terminal_adoption( - &terminal_id, - &record.incarnation, - surface.clone(), - &mut owner_reservation, - ) - .is_err() - { - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - self.mark_terminal_exited_and_materialize( - &terminal_id, - "terminal-adoption-failed", - "host-adoption-topology-failed", - &options, - )?; - handled_terminals.insert(terminal_id.clone()); - if !cleanup_terminal_host_record(&record, &record_path) { - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; - } + // Host handshakes can consume their full timeout. Queue every + // live or indeterminate record before the control socket is + // published so startup time is independent of record count. handled_terminals.insert(terminal_id); - self.reap_if_dead(&surface); + self.schedule_terminal_adoption(options.clone(), record, record_path); } // No launcher survives a daemon restart. A durable lifecycle row From d96a5f80fe93aa6e2b909b3ab362d5278050b54c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:47:16 -0700 Subject: [PATCH 240/521] test(tui): expose blocking shutdown record scan --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 108 ++++++++++++++++++++++- 1 file changed, 104 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 632b8ef974de..c7f445814be9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -51,6 +51,17 @@ type ShutdownAttemptHook = Arc; #[cfg(all(test, unix))] type TerminalAdoptionSurfaceFactory = Arc anyhow::Result> + Send + Sync>; +#[cfg(all(test, unix))] +type TerminalHostRecordLoader = Arc< + dyn Fn( + std::path::PathBuf, + usize, + Instant, + ) -> anyhow::Result< + Vec<(std::path::PathBuf, crate::terminal_host_runtime::TerminalHostRecord)>, + > + Send + + Sync, +>; #[cfg(test)] type NewPaneAfterSpawnHook = Arc) + Send + Sync>; #[cfg(test)] @@ -1638,6 +1649,8 @@ pub struct Mux { terminal_adoption_after_attach: Mutex>>, #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex>, + #[cfg(all(test, unix))] + terminal_host_record_loader: Mutex>, #[cfg(test)] terminal_adoption_workers_started: AtomicUsize, #[cfg(test)] @@ -1897,6 +1910,8 @@ impl Mux { terminal_adoption_after_attach: Mutex::new(None), #[cfg(all(test, unix))] terminal_adoption_surface_factory: Mutex::new(None), + #[cfg(all(test, unix))] + terminal_host_record_loader: Mutex::new(None), #[cfg(test)] terminal_adoption_workers_started: AtomicUsize::new(0), #[cfg(test)] @@ -5319,6 +5334,10 @@ impl Mux { /// closed so a retry cannot race a new PTY into the session. pub fn close_all_surfaces_for_shutdown(&self) -> anyhow::Result { let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; + self.close_all_surfaces_for_shutdown_until(deadline) + } + + fn close_all_surfaces_for_shutdown_until(&self, deadline: Instant) -> anyhow::Result { #[cfg(unix)] crate::process_session::require_stable_process_signaling_until(deadline) .context("preflight process control for server shutdown")?; @@ -5373,10 +5392,9 @@ impl Mux { owner_keys.extend(surface_owner_keys); let records = match root { Some(root) => { - let records = crate::terminal_host_runtime::load_terminal_host_records_strict( - &root, capacity, deadline, - ) - .context("load terminal hosts for server shutdown")?; + let records = self + .load_terminal_host_records_for_shutdown(root, capacity, deadline) + .context("load terminal hosts for server shutdown")?; let available = records .iter() .map(|(_, record)| { @@ -5528,6 +5546,21 @@ impl Mux { Ok(closed_count) } + #[cfg(unix)] + fn load_terminal_host_records_for_shutdown( + &self, + root: std::path::PathBuf, + capacity: usize, + deadline: Instant, + ) -> anyhow::Result> + { + #[cfg(test)] + if let Some(loader) = self.terminal_host_record_loader.lock().unwrap().clone() { + return loader(root, capacity, deadline); + } + crate::terminal_host_runtime::load_terminal_host_records_strict(&root, capacity, deadline) + } + fn lock_shutdown_coordinator_until( &self, deadline: Instant, @@ -14978,6 +15011,73 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn server_shutdown_reuses_one_deadline_bounded_record_scan() { + let root = std::env::temp_dir().join(format!( + "cmux-shutdown-record-scan-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let mux = Mux::new_for_test( + "shutdown-record-scan", + SurfaceOptions { terminal_host_root: Some(root.clone()), ..SurfaceOptions::default() }, + ); + let calls = Arc::new(AtomicUsize::new(0)); + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let release_rx = Arc::new(Mutex::new(release_rx)); + *mux.terminal_host_record_loader.lock().unwrap() = Some(Arc::new({ + let calls = calls.clone(); + move |_, _, _| { + calls.fetch_add(1, Ordering::AcqRel); + started_tx.send(()).unwrap(); + release_rx.lock().unwrap().recv().unwrap(); + Ok(Vec::new()) + } + })); + + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let first = std::thread::spawn({ + let mux = mux.clone(); + move || { + done_tx + .send(mux.close_all_surfaces_for_shutdown_until( + Instant::now() + Duration::from_millis(50), + )) + .unwrap(); + } + }); + started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let first_result = done_rx.recv_timeout(Duration::from_millis(150)).ok(); + let completed_at_deadline = first_result.as_ref().is_some_and(Result::is_err); + let reused_scan = if completed_at_deadline { + let second = mux + .close_all_surfaces_for_shutdown_until(Instant::now() + Duration::from_millis(50)); + let starts = calls.load(Ordering::Acquire); + for _ in 0..2 { + release_tx.send(()).unwrap(); + } + let third = + mux.close_all_surfaces_for_shutdown_until(Instant::now() + Duration::from_secs(1)); + second.is_err() && starts == 1 && third.is_ok() + } else { + for _ in 0..3 { + release_tx.send(()).unwrap(); + } + let _ = done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + false + }; + first.join().unwrap(); + let _ = std::fs::remove_dir_all(root); + + assert!( + completed_at_deadline, + "server shutdown remained blocked inside the terminal-host record scan" + ); + assert!(reused_scan, "a shutdown retry spawned another blocked terminal-host record scan"); + } + #[cfg(unix)] #[test] fn server_shutdown_rejects_malformed_host_records_before_removing_topology() { From 7f4a6419a7d093339b4f3dc32629d36ab27ff257 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:50:27 -0700 Subject: [PATCH 241/521] fix(tui): retain shutdown record scan --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 128 ++++++++++++++++++++--- 1 file changed, 115 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c7f445814be9..580638910e57 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -48,19 +48,15 @@ use crate::{ pub type SurfaceResizeReporter = Arc) + Send + Sync>; #[cfg(test)] type ShutdownAttemptHook = Arc; +#[cfg(unix)] +type TerminalHostRecords = + Vec<(std::path::PathBuf, crate::terminal_host_runtime::TerminalHostRecord)>; #[cfg(all(test, unix))] type TerminalAdoptionSurfaceFactory = Arc anyhow::Result> + Send + Sync>; #[cfg(all(test, unix))] type TerminalHostRecordLoader = Arc< - dyn Fn( - std::path::PathBuf, - usize, - Instant, - ) -> anyhow::Result< - Vec<(std::path::PathBuf, crate::terminal_host_runtime::TerminalHostRecord)>, - > + Send - + Sync, + dyn Fn(std::path::PathBuf, usize, Instant) -> anyhow::Result + Send + Sync, >; #[cfg(test)] type NewPaneAfterSpawnHook = Arc) + Send + Sync>; @@ -178,6 +174,83 @@ struct TerminalAdoptionCoordinator { wake: std::sync::Condvar, } +#[cfg(unix)] +struct TerminalHostRecordScanTask { + root: std::path::PathBuf, + capacity: usize, + result: Receiver>, + worker: std::thread::JoinHandle<()>, +} + +#[cfg(unix)] +#[derive(Default)] +struct TerminalHostRecordScanOwner { + task: Mutex>, +} + +#[cfg(unix)] +impl TerminalHostRecordScanOwner { + fn scan_until( + &self, + root: std::path::PathBuf, + capacity: usize, + deadline: Instant, + loader: F, + ) -> anyhow::Result + where + F: FnOnce(std::path::PathBuf, usize, Instant) -> anyhow::Result + + Send + + 'static, + { + let mut task = self.task.lock().unwrap(); + if let Some(active) = task.as_ref() { + anyhow::ensure!( + active.root == root && active.capacity == capacity, + "terminal-host record scan parameters changed during shutdown" + ); + } else { + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let scan_root = root.clone(); + let worker = std::thread::Builder::new() + .name("terminal-host-record-scan".into()) + .spawn(move || { + let result = loader(scan_root, capacity, deadline); + let _ = result_tx.send(result); + }) + .context("start terminal-host record scan")?; + *task = Some(TerminalHostRecordScanTask { root, capacity, result: result_rx, worker }); + } + + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + anyhow::bail!("terminal-host record scan exceeded the shutdown deadline"); + }; + let received = task + .as_ref() + .expect("terminal-host record scan was installed") + .result + .recv_timeout(remaining); + match received { + Ok(result) => { + let completed = task.take().expect("completed record scan remained installed"); + drop(task); + if completed.worker.join().is_err() { + anyhow::bail!("terminal-host record scan worker panicked"); + } + result + } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + anyhow::bail!("terminal-host record scan exceeded the shutdown deadline") + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + let failed = task.take().expect("failed record scan remained installed"); + drop(task); + let _ = failed.worker.join(); + anyhow::bail!("terminal-host record scan worker stopped without a result") + } + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Hash)] enum ShutdownOwnerKey { Surface(SurfaceId), @@ -1672,6 +1745,8 @@ pub struct Mux { terminal_adoptions: Mutex>, #[cfg(unix)] terminal_adoption_coordinator: Arc, + #[cfg(unix)] + terminal_host_record_scan: TerminalHostRecordScanOwner, /// Fences the interval between accepting a browser daemon-handoff request /// and queueing its acknowledgement. ClientRegistry consults this under /// its own lock so a new native-browser owner cannot race the shutdown. @@ -1933,6 +2008,8 @@ impl Mux { terminal_adoptions: Mutex::new(HashSet::new()), #[cfg(unix)] terminal_adoption_coordinator: Arc::new(TerminalAdoptionCoordinator::default()), + #[cfg(unix)] + terminal_host_record_scan: TerminalHostRecordScanOwner::default(), daemon_handoff_pending: AtomicBool::new(false), shutting_down: AtomicBool::new(false), daemon_shutdown_requested: AtomicBool::new(false), @@ -5552,13 +5629,38 @@ impl Mux { root: std::path::PathBuf, capacity: usize, deadline: Instant, - ) -> anyhow::Result> - { + ) -> anyhow::Result { #[cfg(test)] - if let Some(loader) = self.terminal_host_record_loader.lock().unwrap().clone() { - return loader(root, capacity, deadline); + { + let loader = self.terminal_host_record_loader.lock().unwrap().clone(); + self.terminal_host_record_scan.scan_until( + root, + capacity, + deadline, + move |root, capacity, deadline| { + if let Some(loader) = loader { + loader(root, capacity, deadline) + } else { + crate::terminal_host_runtime::load_terminal_host_records_strict( + &root, capacity, deadline, + ) + } + }, + ) + } + #[cfg(not(test))] + { + self.terminal_host_record_scan.scan_until( + root, + capacity, + deadline, + |root, capacity, deadline| { + crate::terminal_host_runtime::load_terminal_host_records_strict( + &root, capacity, deadline, + ) + }, + ) } - crate::terminal_host_runtime::load_terminal_host_records_strict(&root, capacity, deadline) } fn lock_shutdown_coordinator_until( From d59e8b272deb759ceebdefab98200851a923363f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:58:16 -0700 Subject: [PATCH 242/521] test(tui): satisfy startup adoption lint --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 580638910e57..6db00330cfe0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -14107,8 +14107,7 @@ mod tests { )); std::fs::create_dir_all(&root).unwrap(); let uid = std::fs::metadata(&root).unwrap().uid(); - let options = SurfaceOptions::default(); - let mux = Mux::new_for_test("adoption-startup-budget", options.clone()); + let mux = Mux::new_for_test("adoption-startup-budget", SurfaceOptions::default()); let workspace = mux.create_empty_workspace(Some("startup-budget".into()), None, None).unwrap(); { From c4035828b229e4e7597b826058a5890f565b0bfb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:22:14 -0700 Subject: [PATCH 243/521] test(tui): expose runner Rust drift --- tests/test_tui_workflow_toolchain.py | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/test_tui_workflow_toolchain.py diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py new file mode 100644 index 000000000000..5dda5d25457c --- /dev/null +++ b/tests/test_tui_workflow_toolchain.py @@ -0,0 +1,30 @@ +from pathlib import Path +import re + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github" / "workflows" / "cmux-tui.yml" +RUST_JOBS = ( + "valgrind-leak-check", + "test", + "bindings-e2e", + "windows-experimental", +) + + +def job_block(workflow: str, job: str) -> str: + marker = f"\n {job}:\n" + remainder = workflow.split(marker, 1)[1] + next_job = re.search(r"\n [a-zA-Z0-9_-]+:\n", remainder) + return remainder[: next_job.start()] if next_job else remainder + + +def test_tui_rust_jobs_install_the_release_toolchain() -> None: + workflow = WORKFLOW.read_text() + assert '\nenv:\n RUST_TOOLCHAIN: "1.95.0"\n' in workflow + + for job in RUST_JOBS: + block = job_block(workflow, job) + assert "rustup toolchain install \"$RUST_TOOLCHAIN\" --profile minimal" in block + assert "rustup default \"$RUST_TOOLCHAIN\"" in block + assert "if ! command -v cargo" not in block From b003b6cc04a71952eada55bb501ebd470a8e9af4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:22:48 -0700 Subject: [PATCH 244/521] fix(tui): pin CI Rust toolchain --- .github/workflows/cmux-tui.yml | 42 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index f911ea9d3ae0..f76f82bb6bfc 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -11,6 +11,9 @@ concurrency: permissions: contents: read +env: + RUST_TOOLCHAIN: "1.95.0" + jobs: web-frontend: runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} @@ -61,13 +64,11 @@ jobs: - name: Install zig run: ./scripts/install-zig-ci.sh - - name: Rust version + - name: Install Rust toolchain run: | - rustc --version || true - if ! command -v cargo >/dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - fi + rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal + rustup default "$RUST_TOOLCHAIN" + rustc --version - name: Build test binaries working-directory: cmux-tui @@ -153,13 +154,11 @@ jobs: - name: Install zig run: ./scripts/install-zig-ci.sh - - name: Rust version + - name: Install Rust toolchain run: | - rustc --version || true - if ! command -v cargo >/dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - fi + rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal + rustup default "$RUST_TOOLCHAIN" + rustc --version - name: cargo fmt working-directory: cmux-tui @@ -206,13 +205,11 @@ jobs: - name: Install zig run: ./scripts/install-zig-ci.sh - - name: Rust version + - name: Install Rust toolchain run: | - rustc --version || true - if ! command -v cargo >/dev/null; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - fi + rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal + rustup default "$RUST_TOOLCHAIN" + rustc --version - name: Java version run: | @@ -254,10 +251,17 @@ jobs: with: version: ${{ steps.ghostty-zig-version.outputs.version }} + - name: Install Rust toolchain + shell: bash + run: | + rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal + rustup default "$RUST_TOOLCHAIN" + rustc --version + - name: Install Rust GNU target shell: bash run: | - rustup target add x86_64-pc-windows-gnu + rustup target add --toolchain "$RUST_TOOLCHAIN" x86_64-pc-windows-gnu printf '%s\n' 'C:\msys64\mingw64\bin' >> "$GITHUB_PATH" - name: Build libghostty-vt for Windows GNU From 1a412ab4abd8c370ac563414ef1c2c118c993d96 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:27:46 -0700 Subject: [PATCH 245/521] test(tui): require CI lint components --- tests/test_tui_workflow_toolchain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 5dda5d25457c..a75a3fb95d44 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -28,3 +28,6 @@ def test_tui_rust_jobs_install_the_release_toolchain() -> None: assert "rustup toolchain install \"$RUST_TOOLCHAIN\" --profile minimal" in block assert "rustup default \"$RUST_TOOLCHAIN\"" in block assert "if ! command -v cargo" not in block + + test_job = job_block(workflow, "test") + assert "--component clippy,rustfmt" in test_job From 3213f5f09a3718857ef71716e3f97392827ee1b6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:28:20 -0700 Subject: [PATCH 246/521] fix(tui): install CI lint components --- .github/workflows/cmux-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index f76f82bb6bfc..664fe6e1cc01 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -156,7 +156,7 @@ jobs: - name: Install Rust toolchain run: | - rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal + rustup toolchain install "$RUST_TOOLCHAIN" --profile minimal --component clippy,rustfmt rustup default "$RUST_TOOLCHAIN" rustc --version From 25535a06c5c4ea0bc82c6fcc89ab7b30a5aa9a02 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:43:06 -0700 Subject: [PATCH 247/521] test(tui): await deferred host adoption --- .../cmux-tui/tests/terminal_host_recovery.rs | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 9404a9997f22..998c701dccc8 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -1527,6 +1527,12 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { assert_eq!(wait_for_host_records(&harness.host_root(), 1)[0].1.terminal_id, terminal_id); harness.restart(); + let adopted_surface = wait_for_adopted_terminal( + &harness.socket, + &terminal_id, + &incarnation, + "terminal host was not adopted", + ); let recovered = request(&harness.socket, serde_json::json!({"id": 4, "cmd": "list-workspaces"})); let recovered_workspace = recovered["workspaces"] @@ -1535,8 +1541,7 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { .iter() .find(|item| item["key"].as_str() == Some(&workspace_key)) .expect("durable workspace was not recovered"); - let adopted_surface = - first_surface(recovered_workspace).expect("terminal host was not adopted"); + assert_eq!(first_surface(recovered_workspace), Some(adopted_surface)); let state = wait_for_cursor_visual(&harness.socket, adopted_surface, "block", true); assert_eq!(state["colors"]["cursor_style"], "block"); assert_eq!(state["colors"]["cursor_blink"], true); @@ -1590,6 +1595,12 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { // only in the disposable daemon-side Ghostty mirror. harness.sigkill(); harness.restart(); + let resized_surface = wait_for_adopted_terminal( + &harness.socket, + &terminal_id, + &incarnation, + "resized terminal host was not adopted", + ); let recovered = request(&harness.socket, serde_json::json!({"id": 8, "cmd": "list-workspaces"})); let recovered_workspace = recovered["workspaces"] @@ -1598,8 +1609,7 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { .iter() .find(|item| item["key"].as_str() == Some(&workspace_key)) .expect("workspace was not recovered after the resized host survived again"); - let resized_surface = - first_surface(recovered_workspace).expect("resized terminal host was not adopted"); + assert_eq!(first_surface(recovered_workspace), Some(resized_surface)); let state = request( &harness.socket, serde_json::json!({"id": 9, "cmd": "vt-state", "surface": resized_surface}), @@ -2555,6 +2565,32 @@ fn wait_for_screen(path: &Path, surface: u64, marker: &str) -> String { last } +fn wait_for_adopted_terminal( + path: &Path, + terminal_id: &str, + incarnation: &str, + failure: &str, +) -> u64 { + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let resolved = request( + path, + serde_json::json!({ + "cmd": "resolve-terminal", + "terminal_id": terminal_id, + }), + ); + if resolved["lifecycle"] == "running" + && resolved["terminal_incarnation"].as_str() == Some(incarnation) + && let Some(surface) = resolved["surface"].as_u64() + { + return surface; + } + assert!(Instant::now() < deadline, "{failure}: {resolved}"); + std::thread::sleep(Duration::from_millis(50)); + } +} + fn wait_for_host_records(root: &Path, expected: usize) -> Vec<(PathBuf, TerminalHostRecord)> { let deadline = Instant::now() + Duration::from_secs(10); loop { From 84e37a5ec865e7dfe08d2ac24d18ddd8afcb3f51 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 15:46:46 -0700 Subject: [PATCH 248/521] test(tui): isolate resolver fixtures --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 45 ++++++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 244185fe2e1f..385e491e571c 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -2731,9 +2731,29 @@ mod tests { static RESOLVE_TEST_LOCK: Mutex<()> = Mutex::new(()); + struct TestHostResolverCommandGuard(Option); + + impl TestHostResolverCommandGuard { + fn install(command: TestHostResolverCommand) -> Self { + let previous = TEST_HOST_RESOLVER_COMMAND.lock().unwrap().replace(command); + Self(previous) + } + } + + impl Drop for TestHostResolverCommandGuard { + fn drop(&mut self) { + *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = self.0.take(); + } + } + fn warm_resolver() { - resolve_socket_addr_until("localhost", 0, Instant::now() + Duration::from_secs(5)) - .expect("initialize test resolver"); + let _command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); + resolve_socket_addr_until( + "cmux-warm-resolver.invalid", + 0, + Instant::now() + Duration::from_secs(5), + ) + .expect("initialize test resolver"); } fn resolver_fixture_command(host: &str, port: u16) -> Command { @@ -4214,7 +4234,7 @@ mod tests { fn reconnect_uses_the_cached_peer_without_resolving_again() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); - let listener = TcpListener::bind(("localhost", 0)).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let server = thread::spawn(move || { listener.set_nonblocking(true).unwrap(); @@ -4236,12 +4256,14 @@ mod tests { } connections }); + let command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); let (initial_events, _initial_receiver) = sync_channel(1); let client = CdpClient::connect( - &format!("ws://localhost:{port}/devtools/browser/fake"), + &format!("ws://cmux-cached-peer.invalid:{port}/devtools/browser/fake"), initial_events, ) .unwrap(); + drop(command); let close_deadline = Instant::now() + Duration::from_secs(1); while !client.inner.reader_stopped.load(Ordering::Acquire) { assert!(Instant::now() < close_deadline, "initial CDP reader did not stop"); @@ -4289,13 +4311,20 @@ mod tests { fn expired_resolution_does_not_block_the_next_request() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); + let _command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); - let first = - resolve_socket_addr_until("localhost", 1, Instant::now() + Duration::from_millis(50)); + let first = resolve_socket_addr_until( + "cmux-expired-first.invalid", + 1, + Instant::now() + Duration::from_millis(50), + ); let started = Instant::now(); - let second = - resolve_socket_addr_until("localhost", 2, Instant::now() + Duration::from_millis(100)); + let second = resolve_socket_addr_until( + "cmux-expired-second.invalid", + 2, + Instant::now() + Duration::from_millis(100), + ); let elapsed = started.elapsed(); // Unblock the old single-worker implementation before failing so the From 2ff58a6193b771811d8192f49d6abcaac8693b3c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:19:53 -0700 Subject: [PATCH 249/521] test(tui): stabilize lifecycle checks under load --- .../crates/cmux-tui/src/ui/graphics_writer.rs | 11 +- .../cmux-tui/tests/terminal_host_recovery.rs | 117 +++++++++--------- 2 files changed, 62 insertions(+), 66 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs b/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs index d3d617397330..1fc135989688 100644 --- a/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs +++ b/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs @@ -1456,15 +1456,12 @@ mod tests { 1, vec![test_placement(11, Rect { x: 1, y: 2, width: 3, height: 4 }, 14, Some(8),)] )); - let attempt_deadline = Instant::now() + Duration::from_secs(1); - while attempts.load(Ordering::Acquire) == 0 { - assert!( - Instant::now() < attempt_deadline, - "writer never attempted backpressured output" - ); + let retry_deadline = Instant::now() + Duration::from_secs(1); + while attempts.load(Ordering::Acquire) < 2 { + assert!(Instant::now() < retry_deadline, "writer did not retry backpressured output"); std::thread::sleep(Duration::from_millis(5)); } - let completed_before_shutdown = ready_rx.recv_timeout(Duration::from_millis(50)).is_ok(); + let completed_before_shutdown = ready_rx.try_recv().is_ok(); writer.shutdown(Duration::from_millis(200)); let stopped = writer.handle.as_ref().is_none_or(|handle| handle.is_finished()); diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 998c701dccc8..1fe7bb1ce28f 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -26,6 +26,18 @@ use cmux_tui_core::terminal_host_runtime::{ }; use ghostty_vt::{Rgb, TerminalColorOverrides}; +fn test_timeout_scale() -> u32 { + std::env::var("CMUX_TEST_TIMEOUT_SCALE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|scale| *scale > 0) + .unwrap_or(1) +} + +fn scaled_timeout(timeout: Duration) -> Duration { + timeout.saturating_mul(test_timeout_scale()) +} + struct RecoveryHarness { child: Option, dir: PathBuf, @@ -575,22 +587,27 @@ fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { #[test] fn exit_follows_all_final_pty_bytes_on_the_live_stream() { let harness = RecoveryHarness::start("exit-after-final-bytes"); + let timeout_scale = test_timeout_scale(); + // Instrumentation slows this test's socket consumer more than its native + // terminal-host producer. Keep the native 20,000-line race intact while + // preventing the instrumented reader itself from becoming a lagging client. + let drain_lines = 20_000_u32.div_ceil(timeout_scale.saturating_mul(timeout_scale)); + let script = format!( + concat!( + "IFS= read -r trigger; i=0; ", + "while [ \"$i\" -lt {} ]; do ", + "printf 'drain-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; ", + "i=$((i + 1)); done; ", + "printf 'FINAL-PTY-BYTE-MARKER\\n'", + ), + drain_lines, + ); request( &harness.socket, serde_json::json!({ "id": 1, "cmd": "run", - "argv": [ - "/bin/sh", - "-c", - concat!( - "IFS= read -r trigger; i=0; ", - "while [ \"$i\" -lt 20000 ]; do ", - "printf 'drain-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; ", - "i=$((i + 1)); done; ", - "printf 'FINAL-PTY-BYTE-MARKER\\n'", - ), - ], + "argv": ["/bin/sh", "-c", script], "new_workspace": true, "name": "final-byte-ordering", }), @@ -604,7 +621,7 @@ fn exit_follows_all_final_pty_bytes_on_the_live_stream() { CapabilityRights::ADMIN, ) .unwrap(); - renderer.stream.set_read_timeout(Some(Duration::from_secs(15))).unwrap(); + renderer.stream.set_read_timeout(Some(scaled_timeout(Duration::from_secs(15)))).unwrap(); write_frame(&mut renderer.stream, &Frame::new(MessageKind::Input, b"go\n".to_vec())).unwrap(); let mut output = Vec::new(); @@ -1805,7 +1822,7 @@ fn stalled_renderer_is_disconnected_without_freezing_the_host() { ) .unwrap(); - let done = format!("overflow-done-{}", std::process::id()); + let ready = format!("overflow-ready-{}", std::process::id()); request( &harness.socket, serde_json::json!({ @@ -1813,55 +1830,18 @@ fn stalled_renderer_is_disconnected_without_freezing_the_host() { "cmd": "send", "surface": surface, "text": format!( - // Exceed both the host's bounded client queue and Darwin's - // dynamically sized Unix-socket buffers. A smaller burst can - // fit entirely in the kernel under light load, which does not - // represent a stalled writer and made this assertion timing - // dependent. - "/usr/bin/head -c 64000000 /dev/zero; printf '{done}\\n'\n" + // Keep the producer active until the bounded host queue closes + // this renderer. A finite burst can fit entirely in a + // dynamically sized local-socket buffer on either platform. + "printf '{ready}\\n'; \ + /usr/bin/head -c 1000000000000 /dev/zero & flood=$!\n" ), }), ); - assert!(wait_for_screen(&harness.socket, surface, &done).contains(&done)); - - let disconnected_before_drain = - match stalled.stream.set_read_timeout(Some(Duration::from_secs(5))) { - Ok(()) => false, - // Darwin reports EINVAL when setting SO_RCVTIMEO after the peer has - // already issued shutdown(2); that is the overflow outcome under test. - Err(error) if error.kind() == std::io::ErrorKind::InvalidInput => true, - Err(error) => panic!("set stalled-renderer timeout: {error}"), - }; - let mut complete_frames = 0usize; - if !disconnected_before_drain { - loop { - match read_frame(&mut stalled.stream, MAX_FRAME_PAYLOAD) { - Ok(Some(frame)) => { - assert_eq!(frame.request_id, 0); - assert_eq!(frame.sequence, stalled.next_sequence); - stalled.next_sequence = stalled.next_sequence.wrapping_add(1); - complete_frames += 1; - } - Ok(None) => break, - Err(ProtocolError::Io(error)) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => - { - panic!("stalled renderer silently froze instead of being disconnected") - } - // Shutdown can interrupt a frame already being copied into the - // kernel socket buffer. A clean EOF or truncated final frame are - // both explicit disconnects and require a fresh Snapshot. - Err(ProtocolError::Truncated { .. }) | Err(ProtocolError::Io(_)) => break, - Err(error) => panic!("stalled renderer received invalid protocol data: {error}"), - } - } - } + assert!(wait_for_screen(&harness.socket, surface, &ready).contains(&ready)); assert!( - disconnected_before_drain || complete_frames > 0, - "stalled renderer received no output before disconnect" + wait_for_stream_disconnect(&mut stalled.stream, scaled_timeout(Duration::from_secs(15))), + "stalled renderer silently froze instead of being disconnected" ); // Overflow is isolated to the stalled renderer. The daemon proxy and PTY @@ -1873,7 +1853,10 @@ fn stalled_renderer_is_disconnected_without_freezing_the_host() { "id": 4, "cmd": "send", "surface": surface, - "text": format!("printf '{after}\\n'\n"), + "text": format!( + "kill \"$flood\" 2>/dev/null; wait \"$flood\" 2>/dev/null; \ + printf '{after}\\n'\n" + ), }), ); assert!(wait_for_screen(&harness.socket, surface, &after).contains(&after)); @@ -2565,6 +2548,22 @@ fn wait_for_screen(path: &Path, surface: u64, marker: &str) -> String { last } +fn wait_for_stream_disconnect(stream: &mut UnixStream, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + // ReleaseViewer is valid and idempotent for this renderer but produces + // no host-to-client frame, so it probes only the input half of the + // connection while output remains deliberately unread. + if write_frame(stream, &Frame::new(MessageKind::ReleaseViewer, Vec::new())).is_err() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::sleep(Duration::from_millis(10)); + } +} + fn wait_for_adopted_terminal( path: &Path, terminal_id: &str, From 8ce58e7253b37075292115a987f712e000d660c0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:35:54 -0700 Subject: [PATCH 250/521] test(tui): remove full-suite timing races --- cmux-tui/crates/cmux-tui/src/app.rs | 2 +- .../crates/cmux-tui/src/ui/graphics_writer.rs | 44 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index 1eaf8d07469c..bc8f8eebbca8 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -31553,7 +31553,7 @@ mod tests { command: Some(vec![ "/bin/sh".to_string(), "-c".to_string(), - "sleep 30".to_string(), + "sleep 300".to_string(), ]), cwd: cwd.map(|path| path.to_string_lossy().into_owned()), ..Default::default() diff --git a/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs b/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs index 1fc135989688..79fc19371844 100644 --- a/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs +++ b/cmux-tui/crates/cmux-tui/src/ui/graphics_writer.rs @@ -1173,16 +1173,27 @@ mod tests { } } - #[derive(Clone, Default)] + #[derive(Clone)] struct PartialThenBackpressuredOutput(Arc>); #[derive(Default)] struct PartialThenBackpressuredState { bytes: Vec, blocked_until: Option, + recovery_gate: Option<(std::sync::mpsc::Sender<()>, Receiver<()>)>, } impl PartialThenBackpressuredOutput { + fn with_recovery_gate() -> (Self, Receiver<()>, std::sync::mpsc::Sender<()>) { + let (recovery_started_tx, recovery_started_rx) = std::sync::mpsc::channel(); + let (recovery_release_tx, recovery_release_rx) = std::sync::mpsc::channel(); + let output = Self(Arc::new(Mutex::new(PartialThenBackpressuredState { + recovery_gate: Some((recovery_started_tx, recovery_release_rx)), + ..Default::default() + }))); + (output, recovery_started_rx, recovery_release_tx) + } + fn bytes(&self) -> Vec { self.0.lock().unwrap().bytes.clone() } @@ -1203,6 +1214,13 @@ mod tests { Err(std::io::Error::from(std::io::ErrorKind::WouldBlock)) } Some(_) => { + let recovery_gate = state.recovery_gate.take(); + drop(state); + if let Some((started, release)) = recovery_gate { + started.send(()).unwrap(); + release.recv().unwrap(); + } + let mut state = self.0.lock().unwrap(); state.bytes.extend_from_slice(buf); Ok(buf.len()) } @@ -1482,7 +1500,8 @@ mod tests { #[test] fn partial_output_failure_terminates_apc_and_cleans_possible_images() { let lock = Arc::new(StdoutLock::new(())); - let output = PartialThenBackpressuredOutput::default(); + let (output, recovery_started_rx, recovery_release_tx) = + PartialThenBackpressuredOutput::with_recovery_gate(); let (ready_tx, ready_rx) = std::sync::mpsc::channel(); let mut writer = GraphicsWriter::spawn_with_output(lock.clone(), output.clone(), move || { @@ -1495,13 +1514,16 @@ mod tests { 1, vec![test_placement(15, Rect { x: 1, y: 2, width: 3, height: 4 }, 21, Some(21),)] )); - let partial_deadline = Instant::now() + Duration::from_secs(1); - while output.bytes().is_empty() { - assert!(Instant::now() < partial_deadline, "writer never emitted the partial APC"); - std::thread::sleep(Duration::from_millis(5)); - } - std::thread::sleep(GRAPHICS_OUTPUT_TIMEOUT + Duration::from_millis(20)); - let released_before_recovery = lock.try_lock().is_some(); + recovery_started_rx + .recv_timeout( + GRAPHICS_OUTPUT_TIMEOUT + GRAPHICS_OUTPUT_RECOVERY_TIMEOUT + Duration::from_secs(1), + ) + .expect("writer never entered partial stream recovery"); + assert!( + lock.try_lock().is_none(), + "stdout ownership must remain exclusive until the partial APC is recovered" + ); + recovery_release_tx.send(()).unwrap(); ready_rx .recv_timeout(GRAPHICS_OUTPUT_TIMEOUT + Duration::from_secs(1)) .expect("partial graphics output must settle after bounded stream recovery"); @@ -1517,10 +1539,6 @@ mod tests { occurrences(&bytes, &delete_image(15)) >= 1, "a partially emitted image must be deleted before text fallback can render" ); - assert!( - !released_before_recovery, - "stdout ownership must remain exclusive until the partial APC is recovered" - ); assert!(lock.try_lock().is_some(), "stream recovery must release the shared stdout lock"); } From 1593b1816ecdd753640c2c9c2100e84f533f87bd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:51:33 -0700 Subject: [PATCH 251/521] test(tui): synchronize published host cancellation --- .../cmux-tui-core/src/terminal_host_runtime.rs | 15 +++++++++++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 8 ++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 91a5316820a4..24c1d2b9a0f3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1381,6 +1381,21 @@ mod unix { let mut launch_frame = Frame::new(MessageKind::Launch, launch.encode()?); launch_frame.request_id = 2; write_frame(&mut stdin, &launch_frame)?; + // Integration synchronization seam for cancellation after the host + // has published its durable record but before the launcher consumes + // Ready. The host remains free to serve authenticated termination. + if let Some(barrier) = std::env::var_os("CMUX_TUI_TEST_LAUNCH_ACK_BARRIER") { + let barrier = PathBuf::from(barrier); + while !record_path.exists() && !cancelled() && Instant::now() < launch_deadline { + thread::sleep(Duration::from_millis(5)); + } + if record_path.exists() { + fs::write(&barrier, b"published")?; + while barrier.exists() && !cancelled() && Instant::now() < launch_deadline { + thread::sleep(Duration::from_millis(5)); + } + } + } let launched_frame = read_required_frame(&mut stdout, "launch ready")?; if launched_frame.kind != MessageKind::Ready || launched_frame.request_id != 2 { anyhow::bail!("terminal host did not acknowledge launch"); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index faaa9d653dd8..9b94a8206982 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1641,10 +1641,12 @@ fn server_stop_cancels_a_blocked_terminal_host_launch_write() { #[cfg(unix)] #[test] fn cancelled_published_host_is_terminated_through_its_record() { + let barrier = unique_temp_dir("server-stop-published-launch").with_extension("barrier"); + let barrier_value = barrier.to_string_lossy().into_owned(); let mut server = HeadlessServer::start_with( "server-stop-published-launch", false, - &[("CMUX_TUI_TEST_HOST_READY_DELAY_MS", "1000")], + &[("CMUX_TUI_TEST_LAUNCH_ACK_BARRIER", &barrier_value)], ); let direct_pid_file = server.dir.join("published-direct.pid"); let descendant_pid_file = server.dir.join("published-descendant.pid"); @@ -1666,7 +1668,8 @@ fn cancelled_published_host_is_terminated_through_its_record() { let deadline = Instant::now() + Duration::from_secs(5); while (!direct_pid_file.exists() || !descendant_pid_file.exists() - || terminal_host_pids(&host_root).is_empty()) + || terminal_host_pids(&host_root).is_empty() + || !barrier.exists()) && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(10)); @@ -1679,6 +1682,7 @@ fn cancelled_published_host_is_terminated_through_its_record() { assert_success(&stop); assert!(server.child.wait().unwrap().success()); assert!(!create.wait().unwrap().success()); + let _ = fs::remove_file(&barrier); let deadline = Instant::now() + Duration::from_secs(5); while (process_exists(direct_pid) || process_group_exists(direct_pid) From 791293e22afcc03b8b83f576bc205f65beb7b5a0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 16:51:53 -0700 Subject: [PATCH 252/521] ci(tui): account for Valgrind pidfd gap --- .github/workflows/cmux-tui.yml | 43 +++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 664fe6e1cc01..2180e45b2c53 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -116,8 +116,49 @@ jobs: # their normal deadlines strict while allowing for instrumentation. CMUX_TEST_TIMEOUT_SCALE: "4" run: | + pidfd_probe= + while IFS= read -r bin; do + case "${bin##*/}" in + cmux_tui_core-*) + pidfd_probe="$bin" + break + ;; + esac + done < target/valgrind-test-binaries.txt + test -n "$pidfd_probe" + probe_test=process_session::tests::successful_preflight_survives_a_transient_initializer_failure + "$pidfd_probe" --exact "$probe_test" >/dev/null + valgrind_pidfd_supported=true + if ! valgrind \ + --tool=none \ + --quiet \ + --trace-children=yes \ + --error-exitcode=1 \ + -- "$pidfd_probe" --exact "$probe_test" >/dev/null 2>&1; then + valgrind_pidfd_supported=false + echo "Valgrind cannot emulate pidfd_send_signal; native Linux CI covers process-owning tests" + fi + while IFS= read -r bin; do [ -n "$bin" ] || continue + test_args=() + if [ "$valgrind_pidfd_supported" = false ]; then + # These binaries create PTYs inside the instrumented process. + # The remaining integration binaries exec their runtime + # children outside Valgrind and keep their harness coverage. + case "${bin##*/}" in + cmux_tui-*|cmux_tui_core-*|pty-*) + echo "Skipping pidfd-dependent test binary under Valgrind: $bin" + continue + ;; + websocket_transport-*) + test_args+=( + --skip websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect + --skip clients_list_identify_resize_and_detach_across_transports + ) + ;; + esac + fi echo "Running valgrind for $bin" if ! valgrind \ --error-exitcode=1 \ @@ -125,7 +166,7 @@ jobs: --show-leak-kinds=definite \ --errors-for-leak-kinds=definite \ --track-origins=yes \ - -- "$bin"; then + -- "$bin" "${test_args[@]}"; then echo "Valgrind failed for $bin" >&2 exit 1 fi From 9f803223d8cf11e65111ee5dc1a8f21d94aa72ac Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:20:41 -0700 Subject: [PATCH 253/521] test(tui): stabilize platform verification --- .github/workflows/cmux-tui.yml | 49 ++-- cmux-tui/crates/cmux-tui-core/src/browser.rs | 24 +- .../cmux-tui-core/src/process_session.rs | 11 +- cmux-tui/crates/cmux-tui-core/src/server.rs | 12 +- .../cmux-tui/src/machine_provider_runtime.rs | 1 + cmux-tui/crates/cmux-tui/tests/cli.rs | 78 ++++--- .../cmux-tui/tests/terminal_host_recovery.rs | 36 +-- cmux-tui/dist/valgrind-pidfd-skips.txt | 221 ++++++++++++++++++ tests/test_tui_workflow_toolchain.py | 27 +++ 9 files changed, 377 insertions(+), 82 deletions(-) create mode 100644 cmux-tui/dist/valgrind-pidfd-skips.txt diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 2180e45b2c53..99ca55739398 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -53,6 +53,9 @@ jobs: with: persist-credentials: false + - name: Verify Rust workflow toolchain + run: python3 tests/test_tui_workflow_toolchain.py + - name: Init ghostty submodule run: git submodule update --init --depth 1 ghostty @@ -114,7 +117,7 @@ jobs: CMUX_TEST_WS_LATENCY_BUDGET_MS: "2000" # Process-exit and PTY-reader tests also use bounded polling. Keep # their normal deadlines strict while allowing for instrumentation. - CMUX_TEST_TIMEOUT_SCALE: "4" + CMUX_TEST_TIMEOUT_SCALE: "8" run: | pidfd_probe= while IFS= read -r bin; do @@ -139,25 +142,35 @@ jobs: echo "Valgrind cannot emulate pidfd_send_signal; native Linux CI covers process-owning tests" fi + declare -A seen_skip_scopes=() while IFS= read -r bin; do [ -n "$bin" ] || continue test_args=() if [ "$valgrind_pidfd_supported" = false ]; then - # These binaries create PTYs inside the instrumented process. - # The remaining integration binaries exec their runtime - # children outside Valgrind and keep their harness coverage. - case "${bin##*/}" in - cmux_tui-*|cmux_tui_core-*|pty-*) - echo "Skipping pidfd-dependent test binary under Valgrind: $bin" - continue - ;; - websocket_transport-*) - test_args+=( - --skip websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect - --skip clients_list_identify_resize_and_detach_across_transports - ) + test_list="$("$bin" --list 2>/dev/null || true)" + test_scope= + case "$test_list" in + *"process_session::tests::"*) test_scope=cmux_tui_core ;; + *"app::tests::"*) test_scope=cmux_tui ;; + *"attach_stream_orders_resize_between_output_frames: test"*) test_scope=pty ;; + *"websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect: test"*) + test_scope=websocket_transport ;; esac + if [ -n "$test_scope" ]; then + seen_skip_scopes["$test_scope"]=1 + while read -r skip_scope test_name; do + [ -n "$skip_scope" ] || continue + [[ "$skip_scope" == \#* ]] && continue + [ "$skip_scope" = "$test_scope" ] || continue + if ! grep -Fqx -- "$test_name: test" <<<"$test_list"; then + echo "Unknown $test_scope Valgrind skip: $test_name" >&2 + exit 1 + fi + test_args+=(--skip "$test_name") + done < dist/valgrind-pidfd-skips.txt + echo "Applying ${#test_args[@]} named pidfd skip arguments for $bin" + fi fi echo "Running valgrind for $bin" if ! valgrind \ @@ -171,6 +184,14 @@ jobs: exit 1 fi done < target/valgrind-test-binaries.txt + if [ "$valgrind_pidfd_supported" = false ]; then + while read -r skip_scope; do + if [ -z "${seen_skip_scopes[$skip_scope]:-}" ]; then + echo "No test binary matched Valgrind skip scope: $skip_scope" >&2 + exit 1 + fi + done < <(awk '!/^#/ && NF { print $1 }' dist/valgrind-pidfd-skips.txt | sort -u) + fi test: name: test (${{ matrix.os }}) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index accad4211cef..0dd0648c7876 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -737,9 +737,21 @@ const BROWSER_UPDATED_PAGE_VERIFICATION_FAILED_PREFIX: &str = const BROWSER_VERIFICATION_FAILED_SUFFIX: &str = "; reload to retry"; const AUTHORITY_CAPTURE_ATTEMPTS: usize = 3; #[cfg(not(test))] -const AUTHORITY_CAPTURE_ATTEMPT_BUDGET: Duration = Duration::from_secs(2); +fn authority_capture_attempt_budget() -> Duration { + Duration::from_secs(2) +} +#[cfg(test)] +fn authority_capture_attempt_budget() -> Duration { + Duration::from_millis(150).saturating_mul(test_timeout_scale()) +} #[cfg(test)] -const AUTHORITY_CAPTURE_ATTEMPT_BUDGET: Duration = Duration::from_millis(150); +fn test_timeout_scale() -> u32 { + std::env::var("CMUX_TEST_TIMEOUT_SCALE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|scale| *scale > 0) + .unwrap_or(1) +} const NAVIGATION_AUTHORITY_TIMEOUT: Duration = Duration::from_secs(15); const POINTER_RELEASE_RETRY_DELAY: Duration = Duration::from_millis(250); const BROWSER_RECONFIGURE_RETRY_DELAYS: [Duration; 2] = @@ -4707,7 +4719,7 @@ impl BrowserSurface { { return Ok(BrowserWorkerSuccess::LocallySettled); } - let deadline = Instant::now() + AUTHORITY_CAPTURE_ATTEMPT_BUDGET; + let deadline = Instant::now() + authority_capture_attempt_budget(); match self.capture_main_frame_after_restart(&session, frame_id, loader_id, deadline) { Ok((frame_epoch, captured)) => { let accepted = self.accept_document_paint( @@ -4755,7 +4767,7 @@ impl BrowserSurface { if !self.needs_same_document_paint() { return Ok(BrowserWorkerSuccess::LocallySettled); } - let deadline = Instant::now() + AUTHORITY_CAPTURE_ATTEMPT_BUDGET; + let deadline = Instant::now() + authority_capture_attempt_budget(); match self.capture_main_frame_after_restart(&session, frame_id, loader_id, deadline) { Ok((frame_epoch, captured)) => { let accepted = self.accept_same_document_paint( @@ -4839,7 +4851,7 @@ impl BrowserSurface { ); return Ok(BrowserWorkerSuccess::LocallySettled); } - let deadline = Instant::now() + AUTHORITY_CAPTURE_ATTEMPT_BUDGET; + let deadline = Instant::now() + authority_capture_attempt_budget(); match session .runtime .client @@ -9511,7 +9523,7 @@ mod tests { "document verification must reach the healthy final attempt" ); assert!( - elapsed < Duration::from_millis(600), + elapsed < Duration::from_millis(600).saturating_mul(super::test_timeout_scale()), "bounded attempts monopolized the input worker for {elapsed:?}" ); } diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 4bdbea777da4..c541d082ea19 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -31,6 +31,8 @@ const NATURAL_REAP_MAX_ATTEMPTS: usize = 3; #[cfg(test)] static NATURAL_REAP_WORKERS: AtomicUsize = AtomicUsize::new(0); +#[cfg(test)] +static NATURAL_REAP_TEST_SESSION: AtomicUsize = AtomicUsize::new(1_000_000_000); static NATURAL_REAPER: OnceLock>> = OnceLock::new(); static STABLE_PROCESS_SIGNALING: OnceLock<()> = OnceLock::new(); @@ -48,6 +50,11 @@ pub(crate) fn dedicated_natural_reap_workers_for_test() -> usize { NATURAL_REAP_WORKERS.load(Ordering::Acquire) } +#[cfg(test)] +fn natural_reap_test_session() -> libc::pid_t { + libc::pid_t::try_from(NATURAL_REAP_TEST_SESSION.fetch_add(1, Ordering::Relaxed)).unwrap() +} + #[cfg(test)] pub(crate) fn reserved_child_reaper_active_for_test() -> usize { let Some(slot) = NATURAL_REAPER.get() else { return 0 }; @@ -1646,7 +1653,7 @@ mod tests { let observe_release = release.clone(); let (attempt_sender, attempt_receiver) = mpsc::channel(); let (finished_sender, finished_receiver) = mpsc::channel(); - let session = libc::pid_t::try_from(std::process::id()).unwrap(); + let session = natural_reap_test_session(); enqueue_reserved_session_leader( reserve_child_reaper().unwrap(), session, @@ -1689,7 +1696,7 @@ mod tests { let finish_release = release.clone(); let (attempt_sender, attempt_receiver) = mpsc::channel(); let (finished_sender, finished_receiver) = mpsc::channel(); - let session = libc::pid_t::try_from(std::process::id()).unwrap(); + let session = natural_reap_test_session(); enqueue_reserved_session_leader( reserve_child_reaper().unwrap(), session, diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index b5e2448e6b80..b56bfb9d18d5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -7153,10 +7153,10 @@ mod tests { let nonce = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); - let path = platform::fallback_runtime_dir().join(format!( + let path = std::env::temp_dir().join(format!( "write-eof-drain-{}-{}.sock", std::process::id(), - nonce % 1_000_000_000 + nonce )); let _ = std::fs::remove_file(&path); let listener = transport::listen(&path).unwrap(); @@ -7238,10 +7238,10 @@ mod tests { let nonce = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); - let path = platform::fallback_runtime_dir().join(format!( + let path = std::env::temp_dir().join(format!( "clear-concurrency-{}-{}.sock", std::process::id(), - nonce % 1_000_000_000 + nonce )); let _ = std::fs::remove_file(&path); let listener = transport::listen(&path).unwrap(); @@ -7312,10 +7312,10 @@ mod tests { let nonce = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); - let path = platform::fallback_runtime_dir().join(format!( + let path = std::env::temp_dir().join(format!( "clear-lifecycle-{}-{}.sock", std::process::id(), - nonce % 1_000_000_000 + nonce )); let _ = std::fs::remove_file(&path); let listener = transport::listen(&path).unwrap(); diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs index 80bbe89a51f0..ec231d2bf2cc 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs @@ -3709,6 +3709,7 @@ mod tests { let mut enrolled = catalog.clone(); enrolled.revision = 2; enrolled.selected_machine_id = Some(id("machine-1")); + enrolled.machines[0].connectable = true; let server_catalog = catalog; let server = thread::spawn(move || { let (mut stream, mut reader) = serve_initial_snapshot_with_capabilities( diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 9b94a8206982..b95a2b2b0ff4 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1,8 +1,10 @@ +#[cfg(unix)] +use std::ffi::CStr; use std::fs; #[cfg(unix)] use std::fs::File; use std::io::{BufRead, BufReader, Read, Write}; -#[cfg(target_os = "linux")] +#[cfg(unix)] use std::os::fd::AsRawFd; #[cfg(unix)] use std::os::fd::FromRawFd; @@ -940,6 +942,43 @@ fn ratio_commands_localize_nonfinite_values() { } } +#[cfg(unix)] +fn open_test_pty() -> (File, File) { + static PTY_NAME_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + let _name_lock = PTY_NAME_LOCK.lock().unwrap(); + let master_fd = unsafe { libc::posix_openpt(libc::O_RDWR | libc::O_NOCTTY | libc::O_CLOEXEC) }; + assert_ne!(master_fd, -1, "posix_openpt failed: {}", std::io::Error::last_os_error()); + let master = unsafe { File::from_raw_fd(master_fd) }; + assert_eq!( + unsafe { libc::grantpt(master.as_raw_fd()) }, + 0, + "grantpt failed: {}", + std::io::Error::last_os_error() + ); + assert_eq!( + unsafe { libc::unlockpt(master.as_raw_fd()) }, + 0, + "unlockpt failed: {}", + std::io::Error::last_os_error() + ); + let slave_name = unsafe { libc::ptsname(master.as_raw_fd()) }; + assert!(!slave_name.is_null(), "ptsname failed: {}", std::io::Error::last_os_error()); + let slave_name = unsafe { CStr::from_ptr(slave_name) }.to_owned(); + let slave_fd = + unsafe { libc::open(slave_name.as_ptr(), libc::O_RDWR | libc::O_NOCTTY | libc::O_CLOEXEC) }; + assert_ne!(slave_fd, -1, "open PTY slave failed: {}", std::io::Error::last_os_error()); + let slave = unsafe { File::from_raw_fd(slave_fd) }; + let size = libc::winsize { ws_row: 24, ws_col: 80, ws_xpixel: 0, ws_ypixel: 0 }; + assert_ne!( + unsafe { libc::ioctl(slave.as_raw_fd(), libc::TIOCSWINSZ, &size) }, + -1, + "set PTY size failed: {}", + std::io::Error::last_os_error() + ); + (master, slave) +} + #[cfg(unix)] struct PtyChild { child: Child, @@ -953,21 +992,7 @@ impl PtyChild { } fn start_with_env(args: &[&str], env: &[(&str, &std::ffi::OsStr)]) -> Self { - let mut master = -1; - let mut slave = -1; - let mut size = libc::winsize { ws_row: 24, ws_col: 80, ws_xpixel: 0, ws_ypixel: 0 }; - let opened = unsafe { - libc::openpty( - &mut master, - &mut slave, - std::ptr::null_mut(), - std::ptr::null_mut(), - &raw mut size, - ) - }; - assert_eq!(opened, 0, "openpty failed: {}", std::io::Error::last_os_error()); - let mut master = unsafe { File::from_raw_fd(master) }; - let slave = unsafe { File::from_raw_fd(slave) }; + let (mut master, slave) = open_test_pty(); let output_drain = std::thread::spawn(move || { let mut buffer = [0; 8192]; while master.read(&mut buffer).is_ok_and(|read| read > 0) {} @@ -1007,26 +1032,7 @@ struct DisconnectablePtyChild { #[cfg(unix)] impl DisconnectablePtyChild { fn start(args: &[&str]) -> Self { - let mut master = -1; - let mut slave = -1; - let mut size = libc::winsize { ws_row: 24, ws_col: 80, ws_xpixel: 0, ws_ypixel: 0 }; - let opened = unsafe { - libc::openpty( - &mut master, - &mut slave, - std::ptr::null_mut(), - std::ptr::null_mut(), - &raw mut size, - ) - }; - assert_eq!(opened, 0, "openpty failed: {}", std::io::Error::last_os_error()); - let flags = unsafe { libc::fcntl(master, libc::F_GETFD) }; - assert_ne!(flags, -1, "fcntl(F_GETFD) failed: {}", std::io::Error::last_os_error()); - let cloexec = unsafe { libc::fcntl(master, libc::F_SETFD, flags | libc::FD_CLOEXEC) }; - assert_ne!(cloexec, -1, "fcntl(F_SETFD) failed: {}", std::io::Error::last_os_error()); - - let master = unsafe { File::from_raw_fd(master) }; - let slave = unsafe { File::from_raw_fd(slave) }; + let (master, slave) = open_test_pty(); let child = Command::new(bin()) .args(args) .env_remove("CMUX_TUI_SOCKET") diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 1fe7bb1ce28f..f7e445b5d4bd 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -6,8 +6,8 @@ use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, mpsc}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use cmux_tui_core::platform::transport; @@ -1903,10 +1903,8 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { let mut renderer_writer = renderer.stream.try_clone().unwrap(); let drained = Arc::new(AtomicUsize::new(0)); let drain_count = drained.clone(); - let (overflow_tx, overflow_rx) = mpsc::sync_channel(1); let drain = std::thread::spawn(move || { let mut renderer = renderer; - let mut reported = false; loop { match read_frame(&mut renderer.stream, MAX_FRAME_PAYLOAD) { Ok(Some(frame)) => { @@ -1915,12 +1913,7 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { renderer.next_sequence = renderer.next_sequence.wrapping_add(1); } if frame.kind == MessageKind::Output { - let total = drain_count.fetch_add(frame.payload.len(), Ordering::AcqRel) - + frame.payload.len(); - if total >= 12_000_000 && !reported { - reported = true; - let _ = overflow_tx.send(()); - } + drain_count.fetch_add(frame.payload.len(), Ordering::AcqRel); } } Ok(None) | Err(ProtocolError::Truncated { .. }) | Err(ProtocolError::Io(_)) => { @@ -1935,15 +1928,22 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { // enough PTY output fills the daemon tap's bounded queue and deliberately // disconnects that admin stream. harness.signal_daemon(libc::SIGSTOP); - write_frame( - &mut renderer_writer, - &Frame::new( - MessageKind::Input, - b"/usr/bin/head -c 14000000 /dev/zero; printf '\\nadmin-flood-done\\n'\n".to_vec(), - ), - ) - .unwrap(); - overflow_rx.recv_timeout(Duration::from_secs(15)).unwrap(); + for chunk in 1..=14 { + write_frame( + &mut renderer_writer, + &Frame::new(MessageKind::Input, b"/usr/bin/head -c 1000000 /dev/zero\n".to_vec()), + ) + .unwrap(); + let expected = chunk * 1_000_000; + let deadline = Instant::now() + scaled_timeout(Duration::from_secs(15)); + while drained.load(Ordering::Acquire) < expected && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + drained.load(Ordering::Acquire) >= expected, + "renderer did not drain output chunk {chunk} while the daemon was frozen" + ); + } harness.signal_daemon(libc::SIGCONT); let after = format!("renderer-after-reconnect-{}", std::process::id()); diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt new file mode 100644 index 000000000000..94164d02c9fd --- /dev/null +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -0,0 +1,221 @@ +# Tests that need exact Linux pidfd signaling from the instrumented process. +# Native Linux CI runs every test. This list applies only when Valgrind's +# syscall probe proves that pidfd_send_signal is unavailable. +cmux_tui_core mux::tests::apply_layout_holds_target_workspace_lifecycle_through_commit +cmux_tui_core mux::tests::blocked_terminal_adoption_does_not_stall_an_unrelated_terminal +cmux_tui_core mux::tests::browser_runtime_connection_does_not_hold_the_shared_slot +cmux_tui_core mux::tests::concurrent_browser_and_terminal_share_empty_workspace_screen +cmux_tui_core mux::tests::concurrent_browser_tabs_materialize_one_empty_workspace_screen +cmux_tui_core mux::tests::concurrent_empty_workspace_terminal_inherits_the_first_terminals_cwd +cmux_tui_core mux::tests::daemon_exit_retains_active_local_owner_until_termination_succeeds +cmux_tui_core mux::tests::daemon_exit_retries_until_async_browser_bootstrap_releases_ownership +cmux_tui_core mux::tests::failed_new_pane_attachment_retains_shutdown_ownership +cmux_tui_core mux::tests::failed_surface_shutdown_retains_process_ownership_for_retry +cmux_tui_core mux::tests::key_close_reacquires_replacement_workspace_lifecycle +cmux_tui_core mux::tests::live_authority_install_and_rotation_preserve_open_pty +cmux_tui_core mux::tests::new_browser_tab_materializes_selected_empty_workspace +cmux_tui_core mux::tests::ordinary_surface_close_retains_failed_process_ownership_until_reconciled +cmux_tui_core mux::tests::pane_and_screen_close_wait_for_targeted_terminal_commit +cmux_tui_core mux::tests::persistent_provider_managed_mux_keeps_registry_durable_and_lifecycle_guarded +cmux_tui_core mux::tests::provider_ownership_handoff_waits_for_an_entered_ordinary_close +cmux_tui_core mux::tests::rejected_shutdown_preflight_preserves_browser_ownership +cmux_tui_core mux::tests::restart_marks_reserved_terminal_without_host_record_exited_without_respawn +cmux_tui_core mux::tests::restart_materializes_exited_terminal_until_explicit_close +cmux_tui_core mux::tests::run_materializes_active_empty_workspace +cmux_tui_core mux::tests::run_new_workspace_accepts_a_stable_caller_key +cmux_tui_core mux::tests::server_shutdown_cannot_pass_an_in_flight_surface_retirement +cmux_tui_core mux::tests::server_shutdown_clears_many_surfaces_with_one_tree_event +cmux_tui_core mux::tests::server_shutdown_keeps_topology_when_terminal_batch_commit_fails +cmux_tui_core mux::tests::server_shutdown_rejects_host_records_beyond_owner_capacity +cmux_tui_core mux::tests::server_shutdown_rejects_malformed_host_records_before_removing_topology +cmux_tui_core mux::tests::server_shutdown_reuses_one_deadline_bounded_record_scan +cmux_tui_core mux::tests::server_shutdown_uses_launched_runtime_when_target_confirmation_is_unreachable +cmux_tui_core mux::tests::server_shutdown_waits_for_async_browser_bootstrap +cmux_tui_core mux::tests::server_shutdown_waits_for_in_flight_surface_creation +cmux_tui_core mux::tests::surface_session_subscription_tracks_real_tab_moves_without_layout_churn +cmux_tui_core mux::tests::workspace_mutation_close_waits_for_targeted_terminal_commit_and_replays +cmux_tui_core process_session::tests::natural_reap_batch_refreshes_all_sessions_with_one_scan +cmux_tui_core process_session::tests::session_members_are_never_signaled_through_a_reusable_pid +cmux_tui_core process_session::tests::shutdown_treats_zombie_members_as_drained_before_reserved_parent +cmux_tui_core server::tests::daemon_shutdown_atomically_fences_native_browser_ownership +cmux_tui_core server::tests::daemon_shutdown_is_local_fenced_and_queues_ack_first +cmux_tui_core server::tests::guarded_browser_attach_rejects_a_late_capability_upgrade +cmux_tui_core server::tests::identify_advertises_additive_capabilities +cmux_tui_core server::tests::local_shutdown_exits_after_cleanup_when_the_peer_drops_before_ack +cmux_tui_core server::tests::local_shutdown_responds_before_requesting_mux_exit +cmux_tui_core server::tests::local_shutdown_waits_for_the_acknowledgement_write +cmux_tui_core server::tests::repeated_identify_never_rescans_the_global_process_table +cmux_tui_core server::tests::title_changed_event_includes_authoritative_surface_title +cmux_tui_core surface::tests::explicit_shutdown_deadline_does_not_wait_for_natural_cleanup_sweep +cmux_tui_core surface::tests::failed_local_pty_initialization_reaps_spawned_child +cmux_tui_core surface::tests::local_child_observation_uses_the_reserved_shared_reaper +cmux_tui_core surface::tests::local_same_pair_alt_screen_roundtrip_forces_resolved_cursor_colors +cmux_tui_core surface::tests::local_session_kill_observes_the_waitable_leader_under_its_owner_lock +cmux_tui_core surface::tests::normal_child_exit_cleans_same_session_descendants_before_reaping +cmux_tui_core surface::tests::normal_child_exit_retries_a_failed_session_cleanup +cmux_tui_core terminal_host_runtime::unix::tests::failed_host_pty_initialization_reaps_spawned_child +cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_exit_cleans_descendants_that_keep_the_slave_open +cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_exit_retries_a_failed_session_cleanup +cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_observation_uses_the_reserved_shared_reaper +cmux_tui_core terminal_host_runtime::unix::tests::hosted_session_kill_observes_the_waitable_leader_under_its_owner_lock +cmux_tui_core terminal_host_runtime::unix::tests::spawned_host_process_drop_transfers_waiting_to_the_shared_reaper +cmux_tui_core terminal_host_runtime::unix::tests::unauthenticated_host_client_handshake_is_time_bounded +cmux_tui app::tests::active_scrollbar_drag_rebases_after_terminal_output_changes_geometry +cmux_tui app::tests::alt_n_rejects_a_new_pane_with_no_visible_content +cmux_tui app::tests::alt_n_uses_zellij_default_vertical_distribution +cmux_tui app::tests::attached_tab_mouse_click_uses_rendered_hit_and_survives_routing_refresh +cmux_tui app::tests::attached_tab_selection_is_published_to_owner_mux +cmux_tui app::tests::attached_workspace_mouse_click_uses_both_rendered_rows_and_survives_routing_refresh +cmux_tui app::tests::attached_workspace_release_without_press_is_a_no_op +cmux_tui app::tests::browser_drag_retargets_to_the_animated_pane_geometry +cmux_tui app::tests::browser_graphics_processing_does_not_block_terminal_pointer_input +cmux_tui app::tests::builtin_sidebar_registers_resize_hit_in_both_views +cmux_tui app::tests::captured_pty_motion_rejects_geometry_changed_since_press +cmux_tui app::tests::captured_release_ignores_destination_changes_and_clears_ownership +cmux_tui app::tests::catalog_refresh_preserves_machine_and_workspace_selection_identity_and_scroll +cmux_tui app::tests::clear_history_fallback_operations_remain_ordered +cmux_tui app::tests::clipped_terminal_input_uses_the_logical_source_column +cmux_tui app::tests::close_tab_action_honors_its_explicit_pane_target +cmux_tui app::tests::command_k_clears_prior_output_without_a_session_mutation +cmux_tui app::tests::command_k_matches_the_reported_physical_key_across_layouts +cmux_tui app::tests::command_k_reaches_a_kitty_app_on_the_alternate_screen +cmux_tui app::tests::contended_terminal_semantics_retain_discrete_press_for_replay +cmux_tui app::tests::ctrl_l_reaches_foreground_primary_screen_app +cmux_tui app::tests::deferred_click_fails_closed_after_pointer_map_generation_changes +cmux_tui app::tests::deferred_input_does_not_follow_a_later_routing_mutation +cmux_tui app::tests::deferred_input_follows_a_committed_routing_mutation +cmux_tui app::tests::deferred_input_is_discarded_when_its_destination_changes +cmux_tui app::tests::deferred_scrollbar_press_cannot_reinterpret_changed_thumb_geometry +cmux_tui app::tests::deferred_tab_press_cannot_retarget_a_replacement_surface_at_the_same_index +cmux_tui app::tests::deferred_terminal_press_cannot_retarget_repainted_content +cmux_tui app::tests::deferred_terminal_press_fails_closed_when_live_mouse_ownership_changes +cmux_tui app::tests::deferred_untracked_wheel_fails_closed_when_screen_semantics_change +cmux_tui app::tests::delayed_clear_history_completion_preserves_newer_viewport_and_selection +cmux_tui app::tests::delayed_clear_history_completion_preserves_recreated_selection +cmux_tui app::tests::directional_focus_uses_client_history_and_visible_geometry +cmux_tui app::tests::disabled_mouse_snapshot_does_not_consume_press +cmux_tui app::tests::dispatcher_timeout_preserves_ambiguous_press_for_recovery_release +cmux_tui app::tests::doubled_prefix_keeps_the_focused_sidebar_plugin_target +cmux_tui app::tests::doubled_prefix_runs_the_shared_send_prefix_action +cmux_tui app::tests::event_loop_renders_paint_before_following_pointer_input +cmux_tui app::tests::evicted_known_undelivered_motion_allows_same_cell_retry +cmux_tui app::tests::failed_command_k_preserves_viewport_and_selection +cmux_tui app::tests::failed_machine_switch_preserves_the_current_session +cmux_tui app::tests::failed_mux_recovery_releases_barrier_and_discards_ambiguous_input +cmux_tui app::tests::failed_pty_owned_press_does_not_fall_through_to_cmux_mouse_actions +cmux_tui app::tests::files_filter_exposes_ratatui_cursor_and_accepts_mouse_cursor_placement +cmux_tui app::tests::files_sidebar_renders_temp_directory_and_unread_header_badge +cmux_tui app::tests::focus_loss_purges_pointer_press_waiting_for_a_paint +cmux_tui app::tests::focus_sidebar_focuses_builtin_sidebar +cmux_tui app::tests::focused_sidebar_tab_toggles_builtin_views_and_back +cmux_tui app::tests::focused_sidebar_uses_an_accent_header_and_divider +cmux_tui app::tests::guarded_mouse_encoding_rejects_a_changed_terminal_snapshot +cmux_tui app::tests::guarded_mouse_encoding_rejects_changed_terminal_geometry +cmux_tui app::tests::hiding_then_showing_sidebar_rehydrates_same_size_plugin_status +cmux_tui app::tests::horizontal_status_bar_reserves_terminal_cells_for_wide_messages +cmux_tui app::tests::immediate_terminal_press_fails_closed_before_surface_output_marks_route_stale +cmux_tui app::tests::immediate_terminal_press_rejects_content_changed_before_surface_output +cmux_tui app::tests::immediate_untracked_wheel_fails_closed_before_surface_output_marks_route_stale +cmux_tui app::tests::in_place_machine_switch_preserves_rail_view_focus_and_widths +cmux_tui app::tests::input_never_reasserts_a_viewer_size +cmux_tui app::tests::key_typed_during_pending_sidebar_focus_follows_successful_focus +cmux_tui app::tests::key_typed_during_pending_sidebar_focus_keeps_pane_target_on_failure +cmux_tui app::tests::layout_undo_action_confirms_before_closing_a_created_pane +cmux_tui app::tests::machine_session_replacement_settles_pointer_capture_on_the_old_session +cmux_tui app::tests::mouse_wheel_scrolls_machine_and_workspace_rail_viewports_independently +cmux_tui app::tests::mux_recovery_barrier_defers_input_until_authoritative_tree_is_applied +cmux_tui app::tests::non_switch_machine_action_keeps_the_current_session_and_rails +cmux_tui app::tests::notify_unread_indicators_render_and_clear +cmux_tui app::tests::opening_shortcut_help_releases_an_active_browser_mouse_press +cmux_tui app::tests::opening_shortcut_help_releases_an_active_pty_mouse_press +cmux_tui app::tests::pairing_dialog_captures_live_non_left_pointer_input +cmux_tui app::tests::pane_context_maximize_focuses_the_explicit_inactive_pane +cmux_tui app::tests::pane_context_maximize_preserves_its_explicit_intent_after_remote_state_changes +cmux_tui app::tests::pane_context_new_pane_runs_the_same_smart_layout_action_as_alt_n +cmux_tui app::tests::pane_cursor_and_active_border_yield_to_builtin_and_plugin_sidebars +cmux_tui app::tests::physical_release_queues_behind_a_contended_terminal_press +cmux_tui app::tests::plugin_sidebar_registers_resize_drag_hit +cmux_tui app::tests::pointer_input_waits_for_a_cached_surface_attach_claim +cmux_tui app::tests::pointer_motion_does_not_wait_for_terminal_parsing +cmux_tui app::tests::prefix_bar_and_shortcut_modal_use_the_resolved_action_catalog +cmux_tui app::tests::prefixed_clear_history_never_forwards_only_the_suffix_key +cmux_tui app::tests::prepared_machine_session_events_stay_paused_until_commit_activation +cmux_tui app::tests::pty_mouse_press_does_not_wait_for_terminal_parsing +cmux_tui app::tests::pty_mouse_tracking_forwards_click_release_and_wheel_with_shift_override +cmux_tui app::tests::pty_mouse_uses_the_canonical_rendered_grid_during_resize_margins +cmux_tui app::tests::rejected_motion_enqueue_rolls_back_mouse_encoder_dedupe +cmux_tui app::tests::rejected_release_clears_the_active_pty_drag +cmux_tui app::tests::remote_command_k_uses_authoritative_screen_and_keyboard_modes +cmux_tui app::tests::remote_screen_switch_records_the_new_active_pane +cmux_tui app::tests::remote_workspace_switch_records_the_new_active_pane +cmux_tui app::tests::replaced_session_ignores_old_surface_lane_completion +cmux_tui app::tests::replacement_provider_notice_cannot_mask_missing_workspace_mirror_error +cmux_tui app::tests::retained_right_button_capture_crosses_the_menu_frame_it_opens +cmux_tui app::tests::retiring_surface_state_releases_its_failed_input_lane +cmux_tui app::tests::reverse_viewport_sweep_cancels_a_queued_size_release_and_reasserts +cmux_tui app::tests::right_button_capture_cannot_cross_a_new_pairing_dialog +cmux_tui app::tests::selection_drag_and_release_bypass_a_pending_focus_mutation +cmux_tui app::tests::selection_drag_retargets_to_the_animated_pane_geometry +cmux_tui app::tests::sidebar_context_menu_focus_is_idempotent +cmux_tui app::tests::sidebar_plugin_sync_is_deduped_after_it_is_applied +cmux_tui app::tests::single_surface_client_rejects_hidden_browser_creation +cmux_tui app::tests::single_surface_client_rejects_hidden_pane_closure +cmux_tui app::tests::single_surface_layout_uses_every_terminal_cell_without_chrome +cmux_tui app::tests::single_surface_machine_session_install_does_not_publish_global_cell_metrics +cmux_tui app::tests::single_surface_status_overlays_the_terminal_without_a_status_bar +cmux_tui app::tests::split_ratio_samples_coalesce_without_snapshots_before_final_settlement +cmux_tui app::tests::stale_layout_undo_confirmation_keeps_the_created_pane +cmux_tui app::tests::stale_session_events_are_ignored_after_an_in_place_switch +cmux_tui app::tests::surface_only_context_menu_omits_client_management +cmux_tui app::tests::tab_switch_moves_size_lease_without_dropping_hidden_surface +cmux_tui app::tests::terminal_geometry_mutations_enter_the_pointer_routing_lane +cmux_tui app::tests::terminal_paint_reuses_unchanged_pointer_owner_snapshots +cmux_tui app::tests::terminal_pointer_motion_encoding_stays_inline +cmux_tui app::tests::terminally_failed_retained_motion_does_not_block_a_later_key +cmux_tui app::tests::vertical_split_drag_keeps_nested_pane_tab_bars_visible +cmux_tui app::tests::viewport_columns_resize_with_shortcuts_and_mouse_drag +cmux_tui app::tests::viewport_geometry_changes_reveal_the_active_pane_without_canceling_manual_scroll +cmux_tui app::tests::viewport_pane_overflows_the_existing_tiled_layout +cmux_tui app::tests::viewport_resize_drag_keeps_its_mouse_down_coordinate_origin +cmux_tui app::tests::workspace_rail_scrollbar_is_visible_clickable_and_draggable +cmux_tui config::tests::injected_ghostty_defaults_drive_headless_render_state +cmux_tui pty_input::tests::ambiguous_local_press_keeps_its_recovery_release_reservation +cmux_tui server_lifecycle::legacy_process::tests::frozen_tree_drop_resumes_with_its_original_process_proof +cmux_tui server_lifecycle::legacy_process::tests::process_identity_never_signals_through_a_reusable_pid +cmux_tui server_lifecycle::tests::legacy_stop_terminates_only_the_identified_process +cmux_tui tests::failed_server_publication_preserves_live_socket +pty attach_stream_orders_resize_between_output_frames +pty attach_stream_replays_then_streams_without_duplication +pty byte_attach_cursor_snapshot_does_not_fan_out_a_render_frame +pty control_socket_attach_palette_is_full_sparse_state_and_reset_clears_all_256 +pty control_socket_attach_stream_receives_merged_colors_changed +pty control_socket_attach_vt_state_includes_effective_colors +pty control_socket_attach_vt_state_reports_authoritative_cursor_before_replay +pty control_socket_attach_vt_state_reports_builtin_cursor_without_config +pty control_socket_broadcasts_surface_resized_once_per_changed_size +pty control_socket_read_screen_reports_rendered_viewport_after_scrollback_clear +pty control_socket_round_trip +pty control_socket_wait_for_matches_one_shot_output_already_on_screen +pty create_empty_workspace_is_visible_and_materialized_in_place +pty default_colors_apply_to_existing_and_future_surfaces +pty headless_creation_uses_legacy_default_then_latest_client_size +pty new_tab_on_empty_headless_session_creates_workspace +pty read_scrollback_pages_oldest_rows_and_clamps_bounds +pty render_attach_headless_fans_one_frame_to_render_and_byte_consumers +pty render_attach_resize_is_a_full_replacement_at_the_new_size +pty render_attach_snapshot_and_raced_write_have_no_gap_or_duplicate_frame +pty send_paste_wraps_only_while_dec_mode_2004_is_enabled +pty shared_surface_uses_smallest_viewer_size_until_that_viewer_detaches +pty surface_exit_reaps_tree_and_emits_event +pty surface_resize_reports_whether_the_size_changed +pty surface_runs_command_and_screen_updates +pty tree_event_modes_receive_delta_or_exact_coarse_fallback +websocket_transport clients_list_identify_resize_and_detach_across_transports +websocket_transport websocket_auth_accepts_exact_preamble_and_rejects_missing_or_wrong_tokens +websocket_transport websocket_non_loopback_bind_requires_and_accepts_explicit_insecure_opt_in +websocket_transport websocket_pairing_is_approved_over_trusted_unix_and_credential_reconnects +websocket_transport websocket_rejects_oversized_authentication_frames +websocket_transport websocket_server_allows_pairing_without_a_static_token +websocket_transport websocket_server_rejects_tokens_that_cannot_fit_the_auth_limit +websocket_transport websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect +websocket_transport websocket_subscriber_receives_cross_connection_event_without_poll_delay diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index a75a3fb95d44..1a2ec19985c3 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -4,6 +4,7 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "cmux-tui.yml" +VALGRIND_PIDFD_SKIPS = ROOT / "cmux-tui" / "dist" / "valgrind-pidfd-skips.txt" RUST_JOBS = ( "valgrind-leak-check", "test", @@ -31,3 +32,29 @@ def test_tui_rust_jobs_install_the_release_toolchain() -> None: test_job = job_block(workflow, "test") assert "--component clippy,rustfmt" in test_job + + +def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: + entries = [ + tuple(line.split(maxsplit=1)) + for line in VALGRIND_PIDFD_SKIPS.read_text().splitlines() + if line and not line.startswith("#") + ] + assert all(len(entry) == 2 for entry in entries) + assert len(entries) == len(set(entries)) + assert {scope for scope, _ in entries} == { + "cmux_tui", + "cmux_tui_core", + "pty", + "websocket_transport", + } + + valgrind_job = job_block(WORKFLOW.read_text(), "valgrind-leak-check") + assert "Skipping pidfd-dependent test binary" not in valgrind_job + assert "dist/valgrind-pidfd-skips.txt" in valgrind_job + assert 'test_args+=(--skip "$test_name")' in valgrind_job + + +if __name__ == "__main__": + test_tui_rust_jobs_install_the_release_toolchain() + test_valgrind_fallback_keeps_non_pidfd_tests() From d72ec7ea9cc3ce18b0f1558d069c6ca1e5196642 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:52:09 -0700 Subject: [PATCH 254/521] test(tui): synchronize platform lifecycle checks --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 29 ++++++--- .../cmux-tui-core/tests/browser_runtime.rs | 7 +-- cmux-tui/crates/cmux-tui/tests/cli.rs | 60 +++++++++++++++++-- 3 files changed, 77 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 385e491e571c..4df1bd16f990 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -2731,6 +2731,15 @@ mod tests { static RESOLVE_TEST_LOCK: Mutex<()> = Mutex::new(()); + fn scaled_test_duration(duration: Duration) -> Duration { + let scale = std::env::var("CMUX_TEST_TIMEOUT_SCALE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|scale| *scale > 0) + .unwrap_or(1); + duration.saturating_mul(scale) + } + struct TestHostResolverCommandGuard(Option); impl TestHostResolverCommandGuard { @@ -4312,32 +4321,34 @@ mod tests { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); let _command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); - NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); + let slow_delay = scaled_test_duration(Duration::from_millis(300)); + NEXT_RESOLVE_DELAY_MS.store( + u64::try_from(slow_delay.as_millis()).expect("test delay exceeds u64 milliseconds"), + Ordering::Release, + ); let first = resolve_socket_addr_until( "cmux-expired-first.invalid", 1, - Instant::now() + Duration::from_millis(50), + Instant::now() + scaled_test_duration(Duration::from_millis(50)), ); let started = Instant::now(); let second = resolve_socket_addr_until( "cmux-expired-second.invalid", 2, - Instant::now() + Duration::from_millis(100), + Instant::now() + scaled_test_duration(Duration::from_millis(100)), ); let elapsed = started.elapsed(); - // Unblock the old single-worker implementation before failing so the - // process-wide test resolver cannot contaminate a later test. - if second.is_err() { - thread::sleep(Duration::from_millis(250)); - } NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); + // Let the deliberately expired request leave its worker before the + // process-wide resolver is shared with the next test. + thread::sleep(slow_delay); assert!(first.is_err(), "injected slow resolution unexpectedly completed"); assert!(second.is_ok(), "expired DNS work blocked the next request: {second:?}"); assert!( - elapsed < Duration::from_millis(200), + elapsed < scaled_test_duration(Duration::from_millis(200)), "the next DNS request waited behind expired work: {elapsed:?}" ); } diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 9724c8a1f2ee..041229d7e012 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -1433,11 +1433,8 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { assert_eq!(screencast["params"]["maxWidth"], 100); assert_eq!(screencast["params"]["maxHeight"], 100); - wait_for( - || matches!(surface.browser_status(), Some(BrowserStatus::Live)).then_some(()), - Duration::from_secs(10), - ) - .expect("browser went live"); + wait_for(|| surface.browser_frame_seq(), Duration::from_secs(10)) + .expect("browser never published pointer authority for its first frame"); surface.browser_mouse_event("mousePressed", 5_000.0, 5_000.0, Some("left"), Some(1)).unwrap(); let mouse = recv_method(&seen_rx, "Input.dispatchMouseEvent"); assert_eq!(mouse["sessionId"], "session-1"); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b95a2b2b0ff4..346a17242612 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -950,6 +950,14 @@ fn open_test_pty() -> (File, File) { let master_fd = unsafe { libc::posix_openpt(libc::O_RDWR | libc::O_NOCTTY | libc::O_CLOEXEC) }; assert_ne!(master_fd, -1, "posix_openpt failed: {}", std::io::Error::last_os_error()); let master = unsafe { File::from_raw_fd(master_fd) }; + let master_flags = unsafe { libc::fcntl(master.as_raw_fd(), libc::F_GETFD) }; + assert_ne!( + master_flags, + -1, + "fcntl(F_GETFD) failed for PTY master: {}", + std::io::Error::last_os_error() + ); + assert_ne!(master_flags & libc::FD_CLOEXEC, 0, "PTY master was created without close-on-exec"); assert_eq!( unsafe { libc::grantpt(master.as_raw_fd()) }, 0, @@ -1026,13 +1034,14 @@ impl Drop for PtyChild { #[cfg(unix)] struct DisconnectablePtyChild { child: Child, - master: Option, + disconnect: Option>, + master_drain: Option>, } #[cfg(unix)] impl DisconnectablePtyChild { fn start(args: &[&str]) -> Self { - let (master, slave) = open_test_pty(); + let (mut master, slave) = open_test_pty(); let child = Command::new(bin()) .args(args) .env_remove("CMUX_TUI_SOCKET") @@ -1041,18 +1050,59 @@ impl DisconnectablePtyChild { .stderr(Stdio::from(slave)) .spawn() .unwrap(); - Self { child, master: Some(master) } + let status_flags = unsafe { libc::fcntl(master.as_raw_fd(), libc::F_GETFL) }; + assert_ne!( + status_flags, + -1, + "fcntl(F_GETFL) failed for PTY master: {}", + std::io::Error::last_os_error() + ); + assert_ne!( + unsafe { + libc::fcntl(master.as_raw_fd(), libc::F_SETFL, status_flags | libc::O_NONBLOCK) + }, + -1, + "fcntl(F_SETFL) failed for PTY master: {}", + std::io::Error::last_os_error() + ); + let (disconnect, disconnected) = mpsc::channel(); + let master_drain = std::thread::spawn(move || { + let mut buffer = [0; 8192]; + loop { + if disconnected.try_recv().is_ok() { + break; + } + match master.read(&mut buffer) { + Ok(0) => break, + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + match disconnected.recv_timeout(Duration::from_millis(10)) { + Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } + Err(_) => break, + } + } + }); + Self { child, disconnect: Some(disconnect), master_drain: Some(master_drain) } } fn disconnect_host_terminal(&mut self) { - self.master.take(); + if let Some(disconnect) = self.disconnect.take() { + let _ = disconnect.send(()); + } + if let Some(master_drain) = self.master_drain.take() { + master_drain.join().unwrap(); + } } } #[cfg(unix)] impl Drop for DisconnectablePtyChild { fn drop(&mut self) { - self.master.take(); + self.disconnect_host_terminal(); let _ = self.child.kill(); let _ = self.child.wait(); } From 7a1cc9db30a1d8a8e3ba541c6addbb1748b63146 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:55:47 -0700 Subject: [PATCH 255/521] test(tui): expose macOS test barrier contention --- tests/test_tui_workflow_toolchain.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 1a2ec19985c3..063725243f8a 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -55,6 +55,14 @@ def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: assert 'test_args+=(--skip "$test_name")' in valgrind_job +def test_macos_serializes_process_barrier_tests() -> None: + test_job = job_block(WORKFLOW.read_text(), "test") + assert 'if [[ "$RUNNER_OS" == "macOS" ]]; then' in test_job + assert "args+=(-- --test-threads=1)" in test_job + assert 'cargo test --workspace --locked "${args[@]}"' in test_job + + if __name__ == "__main__": test_tui_rust_jobs_install_the_release_toolchain() test_valgrind_fallback_keeps_non_pidfd_tests() + test_macos_serializes_process_barrier_tests() From c18bae07b86d3410735909691f2ac6cc9cc57fee Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:58:02 -0700 Subject: [PATCH 256/521] ci(tui): serialize macOS process barrier tests --- .github/workflows/cmux-tui.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 99ca55739398..bc7317c436d3 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -232,7 +232,13 @@ jobs: - name: cargo test working-directory: cmux-tui - run: cargo test --workspace --locked + shell: bash + run: | + args=() + if [[ "$RUNNER_OS" == "macOS" ]]; then + args+=(-- --test-threads=1) + fi + cargo test --workspace --locked "${args[@]}" - name: crossterm parser tests working-directory: cmux-tui From 0d6c2092dfecffb33e39be8756ffe16acbcd54d9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:00:08 -0700 Subject: [PATCH 257/521] test(tui): expose Valgrind browser signaling gap --- tests/test_tui_workflow_toolchain.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 063725243f8a..314d5186585d 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -11,6 +11,15 @@ "bindings-e2e", "windows-experimental", ) +BROWSER_RUNTIME_PIDFD_TESTS = { + "browser_capture_scale_applies_to_metrics_screencast_and_input", + "browser_tab_creation_is_async_and_surfaces_bootstrap_failure", + "control_command_reports_backpressure_when_worker_queue_is_full", + "queued_back_and_forward_do_not_collapse_while_worker_is_blocked", + "socket_browser_attach_streams_frames_input_and_cell_pixels", + "stalled_external_browser_nudges_target_once_before_interaction", + "wedged_browser_navigate_does_not_block_same_socket_connection", +} def job_block(workflow: str, job: str) -> str: @@ -43,15 +52,20 @@ def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: assert all(len(entry) == 2 for entry in entries) assert len(entries) == len(set(entries)) assert {scope for scope, _ in entries} == { + "browser_runtime", "cmux_tui", "cmux_tui_core", "pty", "websocket_transport", } + assert { + test_name for scope, test_name in entries if scope == "browser_runtime" + } == BROWSER_RUNTIME_PIDFD_TESTS valgrind_job = job_block(WORKFLOW.read_text(), "valgrind-leak-check") assert "Skipping pidfd-dependent test binary" not in valgrind_job assert "dist/valgrind-pidfd-skips.txt" in valgrind_job + assert "test_scope=browser_runtime" in valgrind_job assert 'test_args+=(--skip "$test_name")' in valgrind_job From 2ef08b6a6b43e71ecf78a50eecea31e390fadfa5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:02:55 -0700 Subject: [PATCH 258/521] ci(tui): scope Valgrind browser signaling exclusions --- .github/workflows/cmux-tui.yml | 3 +++ cmux-tui/dist/valgrind-pidfd-skips.txt | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index bc7317c436d3..1e958dec475e 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -150,6 +150,9 @@ jobs: test_list="$("$bin" --list 2>/dev/null || true)" test_scope= case "$test_list" in + *"browser_capture_scale_applies_to_metrics_screencast_and_input: test"*) + test_scope=browser_runtime + ;; *"process_session::tests::"*) test_scope=cmux_tui_core ;; *"app::tests::"*) test_scope=cmux_tui ;; *"attach_stream_orders_resize_between_output_frames: test"*) test_scope=pty ;; diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index 94164d02c9fd..9026581a0afb 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -1,6 +1,13 @@ # Tests that need exact Linux pidfd signaling from the instrumented process. # Native Linux CI runs every test. This list applies only when Valgrind's # syscall probe proves that pidfd_send_signal is unavailable. +browser_runtime browser_capture_scale_applies_to_metrics_screencast_and_input +browser_runtime browser_tab_creation_is_async_and_surfaces_bootstrap_failure +browser_runtime control_command_reports_backpressure_when_worker_queue_is_full +browser_runtime queued_back_and_forward_do_not_collapse_while_worker_is_blocked +browser_runtime socket_browser_attach_streams_frames_input_and_cell_pixels +browser_runtime stalled_external_browser_nudges_target_once_before_interaction +browser_runtime wedged_browser_navigate_does_not_block_same_socket_connection cmux_tui_core mux::tests::apply_layout_holds_target_workspace_lifecycle_through_commit cmux_tui_core mux::tests::blocked_terminal_adoption_does_not_stall_an_unrelated_terminal cmux_tui_core mux::tests::browser_runtime_connection_does_not_hold_the_shared_slot From dbd41a5c40d3c51fbdec35aa104a31fe4b644ef8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:23:24 -0700 Subject: [PATCH 259/521] fix(tui): drain terminal host exit frames --- .../src/terminal_host_runtime.rs | 33 +++++++++++++++++-- .../cmux-tui/tests/terminal_host_recovery.rs | 16 ++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 24c1d2b9a0f3..0894dc78cba6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -321,6 +321,7 @@ mod unix { const HOST_KILL_WAIT: Duration = Duration::from_secs(2); const HOST_PTY_DRAIN_GRACE: Duration = Duration::from_millis(250); const HOST_FORCED_DRAIN_WINDOW: Duration = Duration::from_millis(100); + const HOST_CLIENT_EXIT_DRAIN: Duration = Duration::from_secs(1); const HOST_LAUNCH_ROLLBACK_WAIT: Duration = Duration::from_secs(4); const HOST_LAUNCH_TIMEOUT: Duration = Duration::from_secs(10); const HOST_LAUNCH_CANCEL_POLL: Duration = Duration::from_millis(25); @@ -2528,6 +2529,7 @@ mod unix { size: Mutex<(u16, u16)>, viewer_sizes: Mutex>, taps: Mutex>, + taps_changed: Condvar, broadcast_lock: Mutex<()>, sequence: AtomicU64, next_client: AtomicU64, @@ -2703,7 +2705,9 @@ mod unix { } fn remove_client(&self, client: u64) { - self.taps.lock().unwrap().remove(&client); + if self.taps.lock().unwrap().remove(&client).is_some() { + self.taps_changed.notify_all(); + } let _ = mutate_viewer_sizes( &self.viewer_sizes, |viewer_sizes| { @@ -3037,10 +3041,31 @@ mod unix { &self.exit_published, ) { self.dead.store(true, Ordering::Release); + self.broadcast(MessageKind::Exit, Vec::new()); // Wake the blocking listener poll so host service cleanup - // follows terminal exit without periodic timer retries. + // follows exit publication without periodic timer retries. let _ = self.service_exit_waker.lock().unwrap().write_all(&[1]); - self.broadcast(MessageKind::Exit, Vec::new()); + } + } + + fn drain_clients_after_exit(&self) { + let deadline = Instant::now() + HOST_CLIENT_EXIT_DRAIN; + let mut taps = self.taps.lock().unwrap(); + while !taps.is_empty() { + let now = Instant::now(); + if now >= deadline { + break; + } + let (next, timeout) = self.taps_changed.wait_timeout(taps, deadline - now).unwrap(); + taps = next; + if timeout.timed_out() && !taps.is_empty() { + break; + } + } + let stalled = taps.values().cloned().collect::>(); + drop(taps); + for tap in stalled { + tap.close(); } } @@ -3406,6 +3431,7 @@ mod unix { Err(error) => return Err(error.into()), } } + shared.drain_clients_after_exit(); drop(guard); Ok(()) } @@ -3501,6 +3527,7 @@ mod unix { size: Mutex::new((launch.cols, launch.rows)), viewer_sizes: Mutex::new(HashMap::new()), taps: Mutex::new(HashMap::new()), + taps_changed: Condvar::new(), broadcast_lock: Mutex::new(()), sequence: AtomicU64::new(0), next_client: AtomicU64::new(1), diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index f7e445b5d4bd..86de11038edf 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -588,19 +588,17 @@ fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { fn exit_follows_all_final_pty_bytes_on_the_live_stream() { let harness = RecoveryHarness::start("exit-after-final-bytes"); let timeout_scale = test_timeout_scale(); - // Instrumentation slows this test's socket consumer more than its native - // terminal-host producer. Keep the native 20,000-line race intact while - // preventing the instrumented reader itself from becoming a lagging client. - let drain_lines = 20_000_u32.div_ceil(timeout_scale.saturating_mul(timeout_scale)); + // Keep enough bytes in flight for the PTY reader to finish after the child, + // but emit them in large writes so this exit-ordering test cannot trip the + // independent 256-frame slow-client guard under scheduler contention. + let drain_chunks = 16_u32.div_ceil(timeout_scale.saturating_mul(timeout_scale)); let script = format!( concat!( - "IFS= read -r trigger; i=0; ", - "while [ \"$i\" -lt {} ]; do ", - "printf 'drain-%05d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; ", - "i=$((i + 1)); done; ", + "IFS= read -r trigger; ", + "dd if=/dev/zero bs=65536 count={} 2>/dev/null; ", "printf 'FINAL-PTY-BYTE-MARKER\\n'", ), - drain_lines, + drain_chunks, ); request( &harness.socket, From c3d44ed23b5c32cf4614396b4cb4ffff47db1e75 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:45:37 -0700 Subject: [PATCH 260/521] test(tui): expose inherited liveness lock --- .../src/terminal_host_runtime.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 0894dc78cba6..9663270dfaa9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -5006,6 +5006,46 @@ mod unix { let _ = fs::remove_dir_all(record_path.parent().unwrap()); } + #[test] + fn liveness_lease_drop_unlocks_an_inherited_file_description() { + let (record_path, record, lease) = record_fixture("inherited-liveness"); + let lease_fd = lease.file.as_raw_fd(); + let mut command = Command::new("/bin/sleep"); + command.arg("60"); + // SAFETY: fcntl is async-signal-safe, and this only clears + // close-on-exec on the test-owned descriptor in the child between + // fork and exec. The parent descriptor remains close-on-exec. + unsafe { + command.pre_exec(move || { + let flags = libc::fcntl(lease_fd, libc::F_GETFD); + if flags == -1 { + return Err(std::io::Error::last_os_error()); + } + if libc::fcntl(lease_fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut inheritor = command.spawn().unwrap(); + + drop(lease); + let observed = terminal_host_record_liveness(&record_path, &record).unwrap(); + + inheritor.kill().unwrap(); + inheritor.wait().unwrap(); + assert!( + remove_stale_terminal_host_record(&record_path, &record).unwrap(), + "test cleanup could not reclaim the released liveness record" + ); + let _ = fs::remove_dir_all(record_path.parent().unwrap()); + assert_eq!( + observed, + TerminalHostLiveness::Dead, + "a forked child retained the terminal host's released liveness lock" + ); + } + #[test] fn record_loader_rejects_noncanonical_filenames_and_identity_spellings() { let (record_path, record, lease) = record_fixture("canonical"); From 359f1284b157ac271187ec5fe325c5efc3f1af19 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:45:57 -0700 Subject: [PATCH 261/521] test(tui): expose Valgrind scheduler gaps --- tests/test_tui_workflow_toolchain.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 314d5186585d..b45214a7fae8 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -20,6 +20,13 @@ "stalled_external_browser_nudges_target_once_before_interaction", "wedged_browser_navigate_does_not_block_same_socket_connection", } +CMUX_TUI_CORE_VALGRIND_INCOMPATIBLE_TESTS = { + "mux::tests::bulk_surface_close_uses_one_shared_termination_deadline", + "mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal", + "mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline", + "mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot", + "server::tests::clear_history_does_not_block_unrelated_surface_input_on_one_connection", +} def job_block(workflow: str, job: str) -> str: @@ -61,6 +68,9 @@ def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: assert { test_name for scope, test_name in entries if scope == "browser_runtime" } == BROWSER_RUNTIME_PIDFD_TESTS + assert CMUX_TUI_CORE_VALGRIND_INCOMPATIBLE_TESTS <= { + test_name for scope, test_name in entries if scope == "cmux_tui_core" + } valgrind_job = job_block(WORKFLOW.read_text(), "valgrind-leak-check") assert "Skipping pidfd-dependent test binary" not in valgrind_job From 08eebef62a8f40990647eb670678bccd18327d28 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:47:51 -0700 Subject: [PATCH 262/521] fix(tui): release inherited liveness lock --- .../cmux-tui-core/src/terminal_host_runtime.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 9663270dfaa9..23ad0a2f3a59 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -3218,6 +3218,22 @@ mod unix { } } + impl Drop for HostLivenessLease { + fn drop(&mut self) { + loop { + // SAFETY: this valid descriptor owns the advisory lock. An + // explicit unlock also releases the shared open-file + // description inherited by a child between fork and exec. + if unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) } == 0 { + break; + } + if std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted { + break; + } + } + } + } + struct HostServiceGuard { shared: Arc, endpoint: PathBuf, From ed63c7d72e032ccb9f87343b5ecd90f3ebdfc8d0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:48:14 -0700 Subject: [PATCH 263/521] ci(tui): scope Valgrind process timing gaps --- cmux-tui/dist/valgrind-pidfd-skips.txt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index 9026581a0afb..745830653197 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -1,6 +1,6 @@ -# Tests that need exact Linux pidfd signaling from the instrumented process. -# Native Linux CI runs every test. This list applies only when Valgrind's -# syscall probe proves that pidfd_send_signal is unavailable. +# Tests whose exact Linux process signaling or scheduling cannot be preserved +# when Valgrind cannot emulate pidfd_send_signal. Native Linux CI runs every +# test. This list applies only after Valgrind's syscall probe fails. browser_runtime browser_capture_scale_applies_to_metrics_screencast_and_input browser_runtime browser_tab_creation_is_async_and_surfaces_bootstrap_failure browser_runtime control_command_reports_backpressure_when_worker_queue_is_full @@ -11,6 +11,7 @@ browser_runtime wedged_browser_navigate_does_not_block_same_socket_connection cmux_tui_core mux::tests::apply_layout_holds_target_workspace_lifecycle_through_commit cmux_tui_core mux::tests::blocked_terminal_adoption_does_not_stall_an_unrelated_terminal cmux_tui_core mux::tests::browser_runtime_connection_does_not_hold_the_shared_slot +cmux_tui_core mux::tests::bulk_surface_close_uses_one_shared_termination_deadline cmux_tui_core mux::tests::concurrent_browser_and_terminal_share_empty_workspace_screen cmux_tui_core mux::tests::concurrent_browser_tabs_materialize_one_empty_workspace_screen cmux_tui_core mux::tests::concurrent_empty_workspace_terminal_inherits_the_first_terminals_cwd @@ -21,6 +22,7 @@ cmux_tui_core mux::tests::failed_surface_shutdown_retains_process_ownership_for_ cmux_tui_core mux::tests::key_close_reacquires_replacement_workspace_lifecycle cmux_tui_core mux::tests::live_authority_install_and_rotation_preserve_open_pty cmux_tui_core mux::tests::new_browser_tab_materializes_selected_empty_workspace +cmux_tui_core mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal cmux_tui_core mux::tests::ordinary_surface_close_retains_failed_process_ownership_until_reconciled cmux_tui_core mux::tests::pane_and_screen_close_wait_for_targeted_terminal_commit cmux_tui_core mux::tests::persistent_provider_managed_mux_keeps_registry_durable_and_lifecycle_guarded @@ -39,13 +41,16 @@ cmux_tui_core mux::tests::server_shutdown_reuses_one_deadline_bounded_record_sca cmux_tui_core mux::tests::server_shutdown_uses_launched_runtime_when_target_confirmation_is_unreachable cmux_tui_core mux::tests::server_shutdown_waits_for_async_browser_bootstrap cmux_tui_core mux::tests::server_shutdown_waits_for_in_flight_surface_creation +cmux_tui_core mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline cmux_tui_core mux::tests::surface_session_subscription_tracks_real_tab_moves_without_layout_churn +cmux_tui_core mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot cmux_tui_core mux::tests::workspace_mutation_close_waits_for_targeted_terminal_commit_and_replays cmux_tui_core process_session::tests::natural_reap_batch_refreshes_all_sessions_with_one_scan cmux_tui_core process_session::tests::session_members_are_never_signaled_through_a_reusable_pid cmux_tui_core process_session::tests::shutdown_treats_zombie_members_as_drained_before_reserved_parent cmux_tui_core server::tests::daemon_shutdown_atomically_fences_native_browser_ownership cmux_tui_core server::tests::daemon_shutdown_is_local_fenced_and_queues_ack_first +cmux_tui_core server::tests::clear_history_does_not_block_unrelated_surface_input_on_one_connection cmux_tui_core server::tests::guarded_browser_attach_rejects_a_late_capability_upgrade cmux_tui_core server::tests::identify_advertises_additive_capabilities cmux_tui_core server::tests::local_shutdown_exits_after_cleanup_when_the_peer_drops_before_ack From fb317f0d9e17c3734ffed062f0269cb9e518276c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:50:08 -0700 Subject: [PATCH 264/521] test(tui): make helper deadline deterministic --- .../crates/cmux-tui/src/server_lifecycle.rs | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a7c3033834e3..64ae82b670c5 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1099,10 +1099,25 @@ fn run_detached_legacy_stop( #[cfg(unix)] fn wait_for_child_until( - mut helper: LegacyHelperChild, + helper: LegacyHelperChild, reaper: LegacyHelperReaperLease, deadline: Instant, ) -> anyhow::Result { + wait_for_child_until_with(helper, reaper, deadline, Instant::now, std::thread::sleep) +} + +#[cfg(unix)] +fn wait_for_child_until_with( + mut helper: LegacyHelperChild, + reaper: LegacyHelperReaperLease, + deadline: Instant, + mut now: Now, + mut sleep: Sleep, +) -> anyhow::Result +where + Now: FnMut() -> Instant, + Sleep: FnMut(Duration), +{ loop { match helper.try_wait() { Ok(Some(status)) => { @@ -1117,15 +1132,15 @@ fn wait_for_child_until( anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } } - if Instant::now() >= deadline { + if now() >= deadline { let pid = libc::pid_t::try_from(helper.id()).ok(); if let Some(pid) = pid { // The helper installs a cooperative handler so Rust cleanup // can thaw any exact process identities currently fenced. let _ = unsafe { libc::kill(pid, libc::SIGTERM) }; } - let cancel_deadline = Instant::now() + LEGACY_HELPER_CANCEL_MARGIN; - while Instant::now() < cancel_deadline { + let cancel_deadline = now() + LEGACY_HELPER_CANCEL_MARGIN; + while now() < cancel_deadline { match helper.try_wait() { Ok(Some(status)) => { helper.settle(status).map_err(|_| { @@ -1141,18 +1156,16 @@ fn wait_for_child_until( anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } } - std::thread::sleep( + sleep( cancel_deadline - .saturating_duration_since(Instant::now()) + .saturating_duration_since(now()) .min(LEGACY_HELPER_POLL_INTERVAL), ); } reaper.retain_shutdown_ownership(helper); anyhow::bail!(crate::localization::catalog().server.legacy_cleanup_failed); } - std::thread::sleep( - deadline.saturating_duration_since(Instant::now()).min(LEGACY_HELPER_POLL_INTERVAL), - ); + sleep(deadline.saturating_duration_since(now()).min(LEGACY_HELPER_POLL_INTERVAL)); } } @@ -1945,16 +1958,24 @@ mod tests { BufReader::new(helper.stdout.take().unwrap()).read_line(&mut ready).unwrap(); assert_eq!(ready, "ready\n"); let helper_pid = libc::pid_t::try_from(helper.id()).unwrap(); - let started = Instant::now(); + let logical_start = Instant::now(); + let logical_now = Arc::new(Mutex::new(logical_start)); + let read_clock = logical_now.clone(); + let advance_clock = logical_now.clone(); let reaper = reserve_legacy_helper_reaper().unwrap(); - let error = wait_for_child_until( + let error = wait_for_child_until_with( LegacyHelperChild { child: helper, quarantine }, reaper, - Instant::now() + Duration::from_millis(50), + logical_start + Duration::from_millis(50), + move || *read_clock.lock().unwrap(), + move |duration| { + let mut clock = advance_clock.lock().unwrap(); + *clock = clock.checked_add(duration).unwrap(); + }, ) .unwrap_err(); - let elapsed = started.elapsed(); + let logical_elapsed = logical_now.lock().unwrap().duration_since(logical_start); // SAFETY: helper_pid names this exact test-owned child. unsafe { libc::kill(helper_pid, libc::SIGKILL); @@ -1975,8 +1996,9 @@ mod tests { assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_cleanup_failed); assert!( - elapsed < Duration::from_millis(250), - "helper wait exceeded its bound: {elapsed:?}" + logical_elapsed + <= Duration::from_millis(50).saturating_add(LEGACY_HELPER_CANCEL_MARGIN), + "helper wait exceeded its logical bound: {logical_elapsed:?}" ); assert_eq!(reaped, -1, "timed-out helper lost its reaping owner"); assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); From 60fa4981fbc8e873be38236ff6550ffc5e5cf196 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:06:00 -0700 Subject: [PATCH 265/521] test(tui): expose remaining Valgrind timing gaps --- tests/test_tui_workflow_toolchain.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index b45214a7fae8..7cef0bfd5d11 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -21,9 +21,12 @@ "wedged_browser_navigate_does_not_block_same_socket_connection", } CMUX_TUI_CORE_VALGRIND_INCOMPATIBLE_TESTS = { + "browser::tests::ordinary_repaint_preserves_captured_drag_motion", + "browser::tests::server_shutdown_bounds_external_target_confirmation", "mux::tests::bulk_surface_close_uses_one_shared_termination_deadline", "mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal", "mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline", + "mux::tests::surface_creation_fence_has_a_bounded_wait", "mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot", "server::tests::clear_history_does_not_block_unrelated_surface_input_on_one_connection", } From c9c72947a02c1285e175b419b3f147649cb00e69 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:06:23 -0700 Subject: [PATCH 266/521] ci(tui): complete Valgrind timing scope --- cmux-tui/dist/valgrind-pidfd-skips.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index 745830653197..c9c5794b1790 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -8,6 +8,8 @@ browser_runtime queued_back_and_forward_do_not_collapse_while_worker_is_blocked browser_runtime socket_browser_attach_streams_frames_input_and_cell_pixels browser_runtime stalled_external_browser_nudges_target_once_before_interaction browser_runtime wedged_browser_navigate_does_not_block_same_socket_connection +cmux_tui_core browser::tests::ordinary_repaint_preserves_captured_drag_motion +cmux_tui_core browser::tests::server_shutdown_bounds_external_target_confirmation cmux_tui_core mux::tests::apply_layout_holds_target_workspace_lifecycle_through_commit cmux_tui_core mux::tests::blocked_terminal_adoption_does_not_stall_an_unrelated_terminal cmux_tui_core mux::tests::browser_runtime_connection_does_not_hold_the_shared_slot @@ -42,6 +44,7 @@ cmux_tui_core mux::tests::server_shutdown_uses_launched_runtime_when_target_conf cmux_tui_core mux::tests::server_shutdown_waits_for_async_browser_bootstrap cmux_tui_core mux::tests::server_shutdown_waits_for_in_flight_surface_creation cmux_tui_core mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline +cmux_tui_core mux::tests::surface_creation_fence_has_a_bounded_wait cmux_tui_core mux::tests::surface_session_subscription_tracks_real_tab_moves_without_layout_churn cmux_tui_core mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot cmux_tui_core mux::tests::workspace_mutation_close_waits_for_targeted_terminal_commit_and_replays From bd5a9c9e901b86af7c35cd09da7020cbb4b1c51d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:39:52 -0700 Subject: [PATCH 267/521] test(tui): pace scripted pointer mutations --- cmux-tui/scripts/smoke-tui.py | 90 ++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/cmux-tui/scripts/smoke-tui.py b/cmux-tui/scripts/smoke-tui.py index c5e0cf9d0237..5f578538dcc2 100644 --- a/cmux-tui/scripts/smoke-tui.py +++ b/cmux-tui/scripts/smoke-tui.py @@ -140,6 +140,25 @@ def tree(): def active_screen(ws): return next(s for s in ws["screens"] if s["active"]) + +def wait_for_active_screen(predicate, seconds=15): + deadline = time.time() + seconds + last = None + while time.time() < deadline: + drain(0.2) + last = active_screen(tree()[0]) + if predicate(last): + return last + raise AssertionError(last) + + +def send_mouse_sequence(*events): + for index, event in enumerate(events): + os.write(fd, event) + if index + 1 < len(events): + drain(0.2) + + def send_prefix_t_until_tab_count(count): last = None for _ in range(5): @@ -673,8 +692,7 @@ def assert_snapshot_parser_controls(): # Alt-n: smart split. In this 75x27 content geometry, width > 2*height, # so the visually longer axis is horizontal and the split is right. os.write(fd, b"\x1bn") -drain(1.0) -screen0 = active_screen(tree()[0]) +screen0 = wait_for_active_screen(lambda screen: len(screen["panes"]) == 2) panes = screen0["panes"] assert len(panes) == 2, screen0 assert screen0["layout"]["type"] == "split" and screen0["layout"]["dir"] == "right", screen0 @@ -683,19 +701,34 @@ def assert_snapshot_parser_controls(): left_pane = panes[0] right_pane = panes[1] tab_order = [t["surface"] for t in left_pane["tabs"]] -os.write(fd, b"\x1b[<0;41;1M\x1b[<32;24;1M\x1b[<0;24;1m") -drain(1.0) -screen0 = active_screen(tree()[0]) +send_mouse_sequence(b"\x1b[<0;41;1M", b"\x1b[<32;24;1M", b"\x1b[<0;24;1m") +expected_order = [tab_order[2], tab_order[0], tab_order[1]] +screen0 = wait_for_active_screen( + lambda screen: [ + t["surface"] + for t in next(p for p in screen["panes"] if p["id"] == left_pane["id"])[ + "tabs" + ] + ] + == expected_order +) panes_by_id = {p["id"]: p for p in screen0["panes"]} left_pane = panes_by_id[left_pane["id"]] right_pane = panes_by_id[right_pane["id"]] reordered = [t["surface"] for t in left_pane["tabs"]] -assert reordered == [tab_order[2], tab_order[0], tab_order[1]], (tab_order, reordered, screen0) +assert reordered == expected_order, (tab_order, reordered, screen0) print("tab drag reorder within pane ok") -os.write(fd, b"\x1b[<0;24;1M\x1b[<32;42;1M\x1b[<0;42;1m") -drain(1.0) -screen0 = active_screen(tree()[0]) +send_mouse_sequence(b"\x1b[<0;24;1M", b"\x1b[<32;42;1M", b"\x1b[<0;42;1m") +screen0 = wait_for_active_screen( + lambda screen: [ + t["surface"] + for t in next(p for p in screen["panes"] if p["id"] == left_pane["id"])[ + "tabs" + ] + ] + == tab_order +) panes_by_id = {p["id"]: p for p in screen0["panes"]} left_pane = panes_by_id[left_pane["id"]] end_reordered = [t["surface"] for t in left_pane["tabs"]] @@ -703,9 +736,23 @@ def assert_snapshot_parser_controls(): print("tab drag past last chip inserts at end ok") moving_surface = left_pane["tabs"][0]["surface"] -os.write(fd, b"\x1b[<0;27;1M\x1b[<32;63;1M\x1b[<0;63;1m") -drain(1.0) -screen0 = active_screen(tree()[0]) +send_mouse_sequence(b"\x1b[<0;27;1M", b"\x1b[<32;63;1M", b"\x1b[<0;63;1m") +screen0 = wait_for_active_screen( + lambda screen: moving_surface + not in [ + t["surface"] + for t in next(p for p in screen["panes"] if p["id"] == left_pane["id"])[ + "tabs" + ] + ] + and moving_surface + in [ + t["surface"] + for t in next(p for p in screen["panes"] if p["id"] == right_pane["id"])[ + "tabs" + ] + ] +) panes_by_id = {p["id"]: p for p in screen0["panes"]} assert moving_surface not in [t["surface"] for t in panes_by_id[left_pane["id"]]["tabs"]], screen0 assert moving_surface in [t["surface"] for t in panes_by_id[right_pane["id"]]["tabs"]], screen0 @@ -715,9 +762,16 @@ def assert_snapshot_parser_controls(): right_pane = panes_by_id[right_pane["id"]] content_surface = left_pane["tabs"][0]["surface"] right_before = [t["surface"] for t in right_pane["tabs"]] -os.write(fd, b"\x1b[<0;27;1M\x1b[<32;82;8M\x1b[<0;82;8m") -drain(1.0) -screen0 = active_screen(tree()[0]) +send_mouse_sequence(b"\x1b[<0;27;1M", b"\x1b[<32;82;8M", b"\x1b[<0;82;8m") +screen0 = wait_for_active_screen( + lambda screen: [ + t["surface"] + for t in next(p for p in screen["panes"] if p["id"] == right_pane["id"])[ + "tabs" + ] + ] + == right_before + [content_surface] +) panes_by_id = {p["id"]: p for p in screen0["panes"]} right_after = [t["surface"] for t in panes_by_id[right_pane["id"]]["tabs"]] assert right_after == right_before + [content_surface], (right_before, right_after, screen0) @@ -793,7 +847,7 @@ def assert_snapshot_parser_controls(): # row 1 blank, rows 2-3 workspace 1, row 4 blank, rows 5-6 workspace 2 # (SGR mouse coordinates are 1-based). original_ws = ws_id -os.write(fd, b"\x1b[<0;2;3M\x1b[<32;2;7M\x1b[<0;2;7m") +send_mouse_sequence(b"\x1b[<0;2;3M", b"\x1b[<32;2;7M", b"\x1b[<0;2;7m") drain(1.0) workspaces = tree() assert [w["id"] for w in workspaces] == [w["id"] for w in workspaces if w["id"] != original_ws] + [original_ws], workspaces @@ -871,7 +925,7 @@ def assert_snapshot_parser_controls(): for s in w["screens"] for p in s["panes"] ) -os.write(fd, b"\x1b[<2;81;6M\x1b[<34;81;10M\x1b[<2;81;10m") +send_mouse_sequence(b"\x1b[<2;81;6M", b"\x1b[<34;81;10M", b"\x1b[<2;81;10m") drain(1.0) tabs_after = sum( len(p["tabs"]) @@ -918,7 +972,7 @@ def assert_snapshot_parser_controls(): for p in s["panes"] ) # "Close tab" is the second row, directly below "Rename tab". -os.write(fd, b"\x1b[<2;81;6M\x1b[<34;81;7M\x1b[<2;81;7m") +send_mouse_sequence(b"\x1b[<2;81;6M", b"\x1b[<34;81;7M", b"\x1b[<2;81;7m") drain(1.0) tabs_after = sum( len(p["tabs"]) From d6a0dd7bdda605f163f9117702b801dca31a66a9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:40:50 -0700 Subject: [PATCH 268/521] test(tui): expose unscaled macOS deadlines --- tests/test_tui_workflow_toolchain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 7cef0bfd5d11..0e8565205386 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -85,6 +85,7 @@ def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: def test_macos_serializes_process_barrier_tests() -> None: test_job = job_block(WORKFLOW.read_text(), "test") assert 'if [[ "$RUNNER_OS" == "macOS" ]]; then' in test_job + assert 'export CMUX_TEST_TIMEOUT_SCALE="4"' in test_job assert "args+=(-- --test-threads=1)" in test_job assert 'cargo test --workspace --locked "${args[@]}"' in test_job From 75368a09fdb1a602ebe466ef8a2d1785edd8dbc4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 00:53:02 -0700 Subject: [PATCH 269/521] ci(tui): scale macOS scheduling budgets --- .github/workflows/cmux-tui.yml | 1 + cmux-tui/crates/cmux-tui-core/src/browser.rs | 67 ++++++++++------- cmux-tui/crates/cmux-tui-core/src/lib.rs | 14 ++++ cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 73 ++++++++++++++----- .../src/terminal_host_runtime.rs | 16 +++- 6 files changed, 127 insertions(+), 51 deletions(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 1e958dec475e..003c5cab7e7c 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -239,6 +239,7 @@ jobs: run: | args=() if [[ "$RUNNER_OS" == "macOS" ]]; then + export CMUX_TEST_TIMEOUT_SCALE="4" args+=(-- --test-threads=1) fi cargo test --workspace --locked "${args[@]}" diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 0dd0648c7876..c4fff275a6bf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -742,24 +742,20 @@ fn authority_capture_attempt_budget() -> Duration { } #[cfg(test)] fn authority_capture_attempt_budget() -> Duration { - Duration::from_millis(150).saturating_mul(test_timeout_scale()) -} -#[cfg(test)] -fn test_timeout_scale() -> u32 { - std::env::var("CMUX_TEST_TIMEOUT_SCALE") - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|scale| *scale > 0) - .unwrap_or(1) + crate::test_timeout(Duration::from_millis(150)) } const NAVIGATION_AUTHORITY_TIMEOUT: Duration = Duration::from_secs(15); const POINTER_RELEASE_RETRY_DELAY: Duration = Duration::from_millis(250); const BROWSER_RECONFIGURE_RETRY_DELAYS: [Duration; 2] = [Duration::from_millis(250), Duration::from_millis(500)]; #[cfg(not(test))] -const NAVIGATION_COMMIT_WAIT: Duration = Duration::from_millis(250); +fn navigation_commit_wait() -> Duration { + Duration::from_millis(250) +} #[cfg(test)] -const NAVIGATION_COMMIT_WAIT: Duration = Duration::from_millis(100); +fn navigation_commit_wait() -> Duration { + crate::test_timeout(Duration::from_millis(100)) +} impl BrowserRuntime { pub fn connect(opts: &SurfaceOptions) -> anyhow::Result> { @@ -3957,7 +3953,8 @@ impl BrowserSurface { // Command acknowledgment does not mean the document committed. The // ingress navigation event owns this barrier and may arrive after the // short synchronous wait on a slow page. - let _ = self.frame_epoch.wait_until_at_least(expected_frame_epoch, NAVIGATION_COMMIT_WAIT); + let _ = + self.frame_epoch.wait_until_at_least(expected_frame_epoch, navigation_commit_wait()); } fn finish_navigation_command( @@ -6511,11 +6508,16 @@ mod tests { }); let owner = browser.shutdown_owner().unwrap(); - assert!(!owner.terminate_until(Instant::now() + Duration::from_millis(200))); - assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 1); + assert!( + !owner + .terminate_until(Instant::now() + crate::test_timeout(Duration::from_millis(200))) + ); + assert_eq!(close_rx.recv_timeout(crate::test_timeout(Duration::from_secs(1))).unwrap(), 1); assert!(browser.session.lock().unwrap().is_none()); - assert!(owner.terminate_until(Instant::now() + Duration::from_secs(1))); - assert_eq!(close_rx.recv_timeout(Duration::from_secs(1)).unwrap(), 2); + assert!( + owner.terminate_until(Instant::now() + crate::test_timeout(Duration::from_secs(1))) + ); + assert_eq!(close_rx.recv_timeout(crate::test_timeout(Duration::from_secs(1))).unwrap(), 2); runtime.shutdown(); server.join().unwrap(); @@ -9384,7 +9386,7 @@ mod tests { let addr = listener.local_addr().unwrap(); let server = thread::spawn(move || { let (stream, _) = listener.accept().unwrap(); - stream.set_read_timeout(Some(Duration::from_millis(500))).unwrap(); + stream.set_read_timeout(Some(crate::test_timeout(Duration::from_millis(500)))).unwrap(); let mut ws = accept(stream).unwrap(); let discover = read_ws_json(&mut ws); assert_eq!(discover["method"], "Target.setDiscoverTargets"); @@ -9523,7 +9525,7 @@ mod tests { "document verification must reach the healthy final attempt" ); assert!( - elapsed < Duration::from_millis(600).saturating_mul(super::test_timeout_scale()), + elapsed < crate::test_timeout(Duration::from_millis(600)), "bounded attempts monopolized the input worker for {elapsed:?}" ); } @@ -9844,7 +9846,9 @@ mod tests { assert_eq!(reload["method"], "Page.reload"); write_ws_json(&mut ws, json!({"id": reload["id"], "result": {}})); - ws.get_ref().set_read_timeout(Some(Duration::from_millis(250))).unwrap(); + ws.get_ref() + .set_read_timeout(Some(crate::test_timeout(Duration::from_millis(250)))) + .unwrap(); let Ok(message) = ws.read() else { superseded_tx.send(false).unwrap(); return; @@ -9880,7 +9884,7 @@ mod tests { }), ); superseded_tx.send(true).unwrap(); - stop_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + stop_rx.recv_timeout(crate::test_timeout(Duration::from_secs(1))).unwrap(); }); let runtime = super::BrowserRuntime::connect_to_endpoint( &format!("ws://{addr}/devtools/browser/fake"), @@ -9903,11 +9907,13 @@ mod tests { browser.reload_blocking().unwrap(); let navigate_result = browser.navigate_blocking("https://next.test"); - let superseded = superseded_rx.recv_timeout(Duration::from_secs(1)).unwrap_or(false); + let superseded = superseded_rx + .recv_timeout(crate::test_timeout(Duration::from_secs(1))) + .unwrap_or(false); let mut admitted = false; if navigate_result.is_ok() { - let deadline = Instant::now() + Duration::from_secs(1); + let deadline = Instant::now() + crate::test_timeout(Duration::from_secs(1)); while Instant::now() < deadline && browser.state.lock().unwrap().pending_navigation_epoch.is_some() { @@ -9955,7 +9961,9 @@ mod tests { assert_eq!(first_reload["method"], "Page.reload"); write_ws_json(&mut ws, json!({"id": first_reload["id"], "result": {}})); - ws.get_ref().set_read_timeout(Some(Duration::from_millis(500))).unwrap(); + ws.get_ref() + .set_read_timeout(Some(crate::test_timeout(Duration::from_millis(500)))) + .unwrap(); let Ok(message) = ws.read() else { superseded_tx.send(false).unwrap(); return; @@ -9974,7 +9982,7 @@ mod tests { } write_ws_json(&mut ws, json!({"id": second_reload["id"], "result": {}})); superseded_tx.send(true).unwrap(); - stop_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + stop_rx.recv_timeout(crate::test_timeout(Duration::from_secs(1))).unwrap(); }); let runtime = super::BrowserRuntime::connect_to_endpoint( &format!("ws://{addr}/devtools/browser/fake"), @@ -9993,7 +10001,9 @@ mod tests { browser.reload_blocking().unwrap(); let second_reload = browser.reload_blocking(); - let superseded = superseded_rx.recv_timeout(Duration::from_secs(1)).unwrap_or(false); + let superseded = superseded_rx + .recv_timeout(crate::test_timeout(Duration::from_secs(1))) + .unwrap_or(false); let _ = stop_tx.send(()); runtime.shutdown(); @@ -10690,8 +10700,9 @@ mod tests { browser.store_frame(test_frame(1)); let result = browser.navigate_blocking("https://example.test#same-document"); - let post_navigate_delay = - post_navigate_delay_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let post_navigate_delay = post_navigate_delay_rx + .recv_timeout(crate::test_timeout(Duration::from_secs(1))) + .unwrap(); let state = browser.state.lock().unwrap(); let pending_frame_epoch = state.pending_frame_epoch; let pending_navigation_epoch = state.pending_navigation_epoch; @@ -10712,7 +10723,7 @@ mod tests { "the unchanged document must regain authority through freshly captured pixels" ); assert!( - post_navigate_delay < super::NAVIGATION_COMMIT_WAIT / 2, + post_navigate_delay < super::navigation_commit_wait() / 2, "loaderless same-document navigation waited {post_navigate_delay:?} for an epoch that cannot advance" ); } diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index cfa06f6fb82b..629a06b678a5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -75,6 +75,20 @@ pub type SplitId = u64; pub type ScreenId = u64; pub type WorkspaceId = u64; +#[cfg(test)] +pub(crate) fn test_timeout_scale() -> u32 { + std::env::var("CMUX_TEST_TIMEOUT_SCALE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|scale| *scale > 0) + .unwrap_or(1) +} + +#[cfg(test)] +pub(crate) fn test_timeout(duration: std::time::Duration) -> std::time::Duration { + duration.saturating_mul(test_timeout_scale()) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SplitDir { /// Split into left/right columns. diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 6db00330cfe0..0c78b046f592 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -15369,7 +15369,7 @@ mod tests { #[test] fn daemon_exit_retries_until_async_browser_bootstrap_releases_ownership() { let mux = test_mux(); - mux.set_shutdown_attempt_timeout_for_test(Duration::from_millis(25)); + mux.set_shutdown_attempt_timeout_for_test(crate::test_timeout(Duration::from_millis(25))); let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); @@ -15390,10 +15390,11 @@ mod tests { shutdown_done_tx.send(()).unwrap(); } }); - let finished_early = shutdown_done_rx.recv_timeout(Duration::from_millis(100)).is_ok(); + let finished_early = + shutdown_done_rx.recv_timeout(crate::test_timeout(Duration::from_millis(100))).is_ok(); release_bootstrap_tx.send(()).unwrap(); if !finished_early { - shutdown_done_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + shutdown_done_rx.recv_timeout(crate::test_timeout(Duration::from_secs(2))).unwrap(); } shutdown.join().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 1045d02ad868..590b3e4f7c6f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4756,11 +4756,25 @@ mod tests { #[cfg(unix)] #[test] fn completed_natural_cleanup_does_not_revalidate_reusable_session_id() { - // A completed child may have its numeric session ID reassigned. Use - // this test process's live session as a deterministic reused value. - // SAFETY: getsid(0) only queries the calling process. - let reused_session = unsafe { libc::getsid(0) }; - assert!(reused_session > 1); + use std::os::unix::process::CommandExt as _; + + // A completed child may have its numeric session ID reassigned. Use a + // dedicated live session instead of assuming the test runner itself + // has a reusable session ID greater than one. + let mut command = std::process::Command::new("sleep"); + command.arg("60"); + unsafe { + command.pre_exec(|| { + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + let mut reused = command.spawn().unwrap(); + let reused_session = libc::pid_t::try_from(reused.id()).unwrap(); + // SAFETY: getsid only queries the live child spawned above. + assert_eq!(unsafe { libc::getsid(reused_session) }, reused_session); let signals = Arc::new(AtomicUsize::new(0)); let process = LocalPtyProcess::new( u32::try_from(reused_session).ok(), @@ -4774,6 +4788,12 @@ mod tests { "exact completed cleanup was rejected through a reused numeric session ID" ); assert_eq!(signals.load(Ordering::Relaxed), 0); + assert!( + reused.try_wait().unwrap().is_none(), + "completed cleanup signaled a reused session" + ); + let _ = reused.kill(); + let _ = reused.wait(); } #[cfg(unix)] @@ -4995,9 +5015,12 @@ mod tests { LocalProcess::Owned(process) => process.clone(), LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), }; - process.normal_cleanup_delay_ms.store(300, Ordering::Release); + process.normal_cleanup_delay_ms.store( + crate::test_timeout(Duration::from_millis(300)).as_millis() as u64, + Ordering::Release, + ); std::fs::write(&release_path, b"ready").unwrap(); - let sweep_deadline = Instant::now() + Duration::from_secs(1); + let sweep_deadline = Instant::now() + crate::test_timeout(Duration::from_secs(1)); while !process.normal_cleanup_started.load(Ordering::Acquire) && Instant::now() < sweep_deadline { @@ -5006,13 +5029,15 @@ mod tests { assert!(process.normal_cleanup_started.load(Ordering::Acquire)); let started = Instant::now(); - let _ = process.terminate_and_wait(started + Duration::from_millis(50)); + let _ = + process.terminate_and_wait(started + crate::test_timeout(Duration::from_millis(50))); let elapsed = started.elapsed(); - let _ = process.terminate_and_wait(Instant::now() + Duration::from_secs(1)); + let _ = process + .terminate_and_wait(Instant::now() + crate::test_timeout(Duration::from_secs(1))); let _ = std::fs::remove_dir_all(root); assert!( - elapsed < Duration::from_millis(150), + elapsed < crate::test_timeout(Duration::from_millis(150)), "explicit shutdown waited behind the natural cleanup sweep: {elapsed:?}" ); } @@ -5283,31 +5308,45 @@ mod tests { #[cfg(unix)] #[test] fn local_same_pair_alt_screen_roundtrip_forces_resolved_cursor_colors() { + let root = std::env::temp_dir().join(format!( + "cmux-local-cursor-activity-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let release_path = root.join("release"); let mux = Mux::new_for_test("local-cursor-activity", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ "/bin/sh".into(), "-c".into(), - "sleep 0.2; printf '\\033[?1049h\\033[?1049l'; sleep 0.2".into(), + format!( + "while [ ! -e {} ]; do sleep 0.01; done; \ + printf '\\033[?1049h\\033[?1049l'; sleep 0.2", + release_path.display() + ), ]), ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); let attach = surface.attach_stream().unwrap(); let expected = (attach.colors.cursor_style, attach.colors.cursor_blink); - let deadline = Instant::now() + Duration::from_secs(2); + std::fs::write(&release_path, b"ready").unwrap(); + let deadline = Instant::now() + crate::test_timeout(Duration::from_secs(2)); let mut output = Vec::new(); let colors = loop { assert!(Instant::now() < deadline, "local cursor activity was not published"); - match attach.stream.recv_timeout(Duration::from_millis(250)).unwrap() { - AttachFrame::Output(bytes) => output.extend_from_slice(&bytes), - AttachFrame::ColorsChanged(colors) => break colors, - AttachFrame::Resized { .. } | AttachFrame::ResizedWithColors { .. } => {} - AttachFrame::OutputWithColors { .. } => { + match attach.stream.recv_timeout(crate::test_timeout(Duration::from_millis(250))) { + Err(RecvTimeoutError::Timeout) => continue, + Err(error) => panic!("local cursor activity stream failed: {error}"), + Ok(AttachFrame::Output(bytes)) => output.extend_from_slice(&bytes), + Ok(AttachFrame::ColorsChanged(colors)) => break colors, + Ok(AttachFrame::Resized { .. } | AttachFrame::ResizedWithColors { .. }) => {} + Ok(AttachFrame::OutputWithColors { .. }) => { panic!("local PTYs must use ordered Output then ColorsChanged") } } }; + let _ = std::fs::remove_dir_all(root); assert!( output.windows(16).any(|window| window == b"\x1b[?1049h\x1b[?1049l"), diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 23ad0a2f3a59..42e689819c27 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -337,6 +337,16 @@ mod unix { static HOST_REAP_TEST_LOCK: Mutex<()> = Mutex::new(()); #[cfg(test)] static NEXT_HOST_NORMAL_CLEANUP_FAILURES: AtomicUsize = AtomicUsize::new(0); + + #[cfg(not(test))] + fn host_forced_drain_window() -> Duration { + HOST_FORCED_DRAIN_WINDOW + } + + #[cfg(test)] + fn host_forced_drain_window() -> Duration { + crate::test_timeout(HOST_FORCED_DRAIN_WINDOW) + } #[cfg(test)] static NEXT_HOST_PROCESS_REAPER_SPAWN_FAILURES: AtomicUsize = AtomicUsize::new(0); #[cfg(test)] @@ -2383,7 +2393,7 @@ mod unix { loop { if force_drain.load(Ordering::Acquire) { let started = forced_at.get_or_insert_with(Instant::now); - if started.elapsed() >= HOST_FORCED_DRAIN_WINDOW { + if started.elapsed() >= host_forced_drain_window() { return Ok(false); } } @@ -2401,7 +2411,7 @@ mod unix { ]; let timeout_ms = forced_at .map(|started| { - let remaining = HOST_FORCED_DRAIN_WINDOW.saturating_sub(started.elapsed()); + let remaining = host_forced_drain_window().saturating_sub(started.elapsed()); remaining.as_millis().clamp(1, i32::MAX as u128) as i32 }) .unwrap_or(-1); @@ -3152,7 +3162,7 @@ mod unix { // the durable host: wake the reader, drain bytes already // readable for a short bounded window, then publish Exit. self.request_forced_pty_drain(); - let _ = self.wait_for_pty_drain(HOST_FORCED_DRAIN_WINDOW * 2); + let _ = self.wait_for_pty_drain(host_forced_drain_window() * 2); } } } From 44cb724f1a48e4c04397817505594a9a0684f6d1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 01:12:52 -0700 Subject: [PATCH 270/521] test(tui): scale adoption worker startup wait --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0c78b046f592..d9432e3b1678 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -14225,7 +14225,7 @@ mod tests { ); } - let deadline = Instant::now() + Duration::from_secs(1); + let deadline = Instant::now() + crate::test_timeout(Duration::from_secs(1)); while mux.terminal_adoption_workers_started.load(Ordering::Acquire) < TERMINAL_ADOPTION_WORKERS && Instant::now() < deadline From 50411609b6a8558a3f48337b0b5b95f592c7962e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 01:12:52 -0700 Subject: [PATCH 271/521] test(tui): serialize Crossterm color state --- .../vendor/crossterm/src/style/types/colored.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmux-tui/vendor/crossterm/src/style/types/colored.rs b/cmux-tui/vendor/crossterm/src/style/types/colored.rs index ab8d769d793c..c20582c4d992 100644 --- a/cmux-tui/vendor/crossterm/src/style/types/colored.rs +++ b/cmux-tui/vendor/crossterm/src/style/types/colored.rs @@ -153,58 +153,68 @@ impl fmt::Display for Colored { #[cfg(test)] mod tests { use crate::style::{Color, Colored}; + use serial_test::serial; fn check_format_color(colored: Colored, expected: &str) { Colored::set_ansi_color_disabled(true); - assert_eq!(colored.to_string(), ""); + let disabled = colored.to_string(); Colored::set_ansi_color_disabled(false); + assert_eq!(disabled, ""); assert_eq!(colored.to_string(), expected); } #[test] + #[serial] fn test_format_fg_color() { let colored = Colored::ForegroundColor(Color::Red); check_format_color(colored, "38;5;9"); } #[test] + #[serial] fn test_format_bg_color() { let colored = Colored::BackgroundColor(Color::Red); check_format_color(colored, "48;5;9"); } #[test] + #[serial] fn test_format_reset_fg_color() { let colored = Colored::ForegroundColor(Color::Reset); check_format_color(colored, "39"); } #[test] + #[serial] fn test_format_reset_bg_color() { let colored = Colored::BackgroundColor(Color::Reset); check_format_color(colored, "49"); } #[test] + #[serial] fn test_format_fg_rgb_color() { let colored = Colored::BackgroundColor(Color::Rgb { r: 1, g: 2, b: 3 }); check_format_color(colored, "48;2;1;2;3"); } #[test] + #[serial] fn test_format_fg_ansi_color() { let colored = Colored::ForegroundColor(Color::AnsiValue(255)); check_format_color(colored, "38;5;255"); } #[test] + #[serial] fn test_parse_ansi_fg() { test_parse_ansi(Colored::ForegroundColor) } #[test] + #[serial] fn test_parse_ansi_bg() { - test_parse_ansi(Colored::ForegroundColor) + test_parse_ansi(Colored::BackgroundColor) } /// Used for test_parse_ansi_fg and test_parse_ansi_bg From a2ab29c4d28620e01e4980c4783486003638c848 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 04:12:00 -0700 Subject: [PATCH 272/521] test(tui): make provider retry fixture deterministic --- cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs index ec231d2bf2cc..680d765f46e8 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_runtime.rs @@ -3915,7 +3915,7 @@ mod tests { &protocol::ResponseEnvelope::success( request.id, protocol::ConnectExternalMachineResult { - machine_id: id("machine-1"), + machine_id: id("machine-2"), revision: server_catalog.revision, notice: None, }, @@ -3957,7 +3957,7 @@ mod tests { controller.provider.reconnect_control().unwrap(); let result = controller.perform_request(provider_connect("PAIR 4J7K")).unwrap(); - assert!(result.ui.request.is_some()); + assert_eq!(result.ui.request, Some(MachineRequest::ReconnectProvider)); controller.close(); server.join().unwrap(); } From 5d3f59a7c847811c0d755f072c48fae2f028e80a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 04:12:00 -0700 Subject: [PATCH 273/521] test(tui): avoid browser fixture ownership cycle --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d9432e3b1678..08f86d66d288 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -11708,9 +11708,10 @@ mod tests { } })); *mux.browser_tab_after_spawn.lock().unwrap() = Some(Arc::new({ - let mux = mux.clone(); + let mux = Arc::downgrade(&mux); let runtime = runtime.clone(); move |surface| { + let mux = mux.upgrade().expect("test mux remained alive through browser spawn"); surface.as_browser().unwrap().install_shutdown_session_for_test( runtime.clone(), "target-race", From 35f190d93a883b241d907cbfafff5e9aeb90f635 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:31:37 -0700 Subject: [PATCH 274/521] test(tui): sync SDK command inventory --- cmux-tui/bindings/ERGONOMICS.md | 4 ++-- cmux-tui/bindings/RELEASING.md | 2 +- cmux-tui/bindings/cpp/tests/test_generated.cpp | 2 +- cmux-tui/bindings/go/raw/README.md | 2 +- cmux-tui/bindings/go/raw/client_test.go | 4 ++-- .../java/tests/com/cmux/raw/GeneratedCoverageTest.java | 2 +- cmux-tui/bindings/python/tests/test_protocol.py | 2 +- cmux-tui/bindings/typescript/test/generated.test.ts | 2 +- cmux-tui/bindings/zig/examples/watch.zig | 2 +- cmux-tui/bindings/zig/src/raw.zig | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmux-tui/bindings/ERGONOMICS.md b/cmux-tui/bindings/ERGONOMICS.md index df8bc3ce4a17..4ccdf2d435d3 100644 --- a/cmux-tui/bindings/ERGONOMICS.md +++ b/cmux-tui/bindings/ERGONOMICS.md @@ -2,7 +2,7 @@ The seven public SDKs expose handwritten resource handles over the reviewed 112-operation `cmux.protocol/1` catalog. The raw protocol inventory is a -separate compatibility surface with 91 commands and 44 events. Deterministic +separate compatibility surface with 92 commands and 44 events. Deterministic generation is limited to those private protocol-10 models under each package's explicit `raw` namespace. Consumers do not run a generator or install a generator runtime. @@ -71,7 +71,7 @@ and secret redaction. The exact-binary live matrix adds one isolated create, run, exit, restart, and cleanup flow per language. TypeScript repeats it over authenticated WebSocket, for eight live transport runs. The separate raw protocol-10 suite runs 266 -compatibility checks over its 91 commands and 44 events. +compatibility checks over its 92 commands and 44 events. Each package suite also opens a stream with a short request deadline, leaves it idle past that deadline, then delivers and cancels normally. This separates diff --git a/cmux-tui/bindings/RELEASING.md b/cmux-tui/bindings/RELEASING.md index 90a84af437e4..e6c9e771a6d0 100644 --- a/cmux-tui/bindings/RELEASING.md +++ b/cmux-tui/bindings/RELEASING.md @@ -23,7 +23,7 @@ from app release tags such as `vX.Y.Z`. Current SDK package versions are | C++ | C++20 and CMake 3.20 | standard library and platform socket APIs | installable CMake package | | Zig | Zig 0.15.2 | standard library only | source package | -All packages target mux protocol 10 and expose the same 91 commands and 44 +All packages target mux protocol 10 and expose the same 92 commands and 44 events. The shared conformance suite verifies their common wire behavior. ## One-time registry setup diff --git a/cmux-tui/bindings/cpp/tests/test_generated.cpp b/cmux-tui/bindings/cpp/tests/test_generated.cpp index d0a1c2249696..dddf6c8eb8eb 100644 --- a/cmux-tui/bindings/cpp/tests/test_generated.cpp +++ b/cmux-tui/bindings/cpp/tests/test_generated.cpp @@ -84,7 +84,7 @@ static_assert(std::is_same_v< static_assert(!std::is_copy_constructible_v); static_assert(std::is_move_constructible_v); -constexpr std::size_t kExpectedRawCommandCount = 91U; +constexpr std::size_t kExpectedRawCommandCount = 92U; constexpr std::array kViewportHistoryCommandNames{ "clear-history", "new-pane-right", diff --git a/cmux-tui/bindings/go/raw/README.md b/cmux-tui/bindings/go/raw/README.md index dad0a3a3ec4b..5ff3f6112294 100644 --- a/cmux-tui/bindings/go/raw/README.md +++ b/cmux-tui/bindings/go/raw/README.md @@ -1,6 +1,6 @@ # cmux Go SDK -The Go SDK covers all 91 protocol-10 commands and 44 event shapes using only +The Go SDK covers all 92 protocol-10 commands and 44 event shapes using only the Go standard library. ```go diff --git a/cmux-tui/bindings/go/raw/client_test.go b/cmux-tui/bindings/go/raw/client_test.go index 241c20f79dc3..f396522f908e 100644 --- a/cmux-tui/bindings/go/raw/client_test.go +++ b/cmux-tui/bindings/go/raw/client_test.go @@ -20,8 +20,8 @@ import ( func TestGeneratedInventoryHasTypedMethodForEveryCommand(t *testing.T) { commands := AllCommandMetadata() - if len(commands) != 91 { - t.Fatalf("generated commands = %d, want 91", len(commands)) + if len(commands) != 92 { + t.Fatalf("generated commands = %d, want 92", len(commands)) } clientType := reflect.TypeOf((*Client)(nil)) commandNames := make(map[string]struct{}, len(commands)) diff --git a/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java b/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java index 571f66465690..5082c546a086 100644 --- a/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java +++ b/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java @@ -20,7 +20,7 @@ public final class GeneratedCoverageTest { public static void main(String[] args) throws Exception { check(Protocol.VERSION == 10, "protocol version"); check("1.0.0".equals(Protocol.SDK_VERSION), "SDK release version"); - check(Commands.ALL.size() == 91, "all 91 commands generated"); + check(Commands.ALL.size() == 92, "all 92 commands generated"); check(Events.ALL.size() == 44, "all 44 events generated"); Map methods = Arrays.stream(GeneratedCmuxClient.class.getDeclaredMethods()) diff --git a/cmux-tui/bindings/python/tests/test_protocol.py b/cmux-tui/bindings/python/tests/test_protocol.py index ec861c08f13c..3b745a12a514 100644 --- a/cmux-tui/bindings/python/tests/test_protocol.py +++ b/cmux-tui/bindings/python/tests/test_protocol.py @@ -22,7 +22,7 @@ class GeneratedProtocolTests(unittest.TestCase): def test_protocol_ten_inventory_is_exhaustive(self) -> None: self.assertEqual(MUX_PROTOCOL, 10) - self.assertEqual(len(COMMANDS), 91) + self.assertEqual(len(COMMANDS), 92) self.assertEqual(set(COMMANDS), set(SCHEMA["commands"])) self.assertEqual(set(EVENTS), set(SCHEMA["events"])) self.assertEqual(len(IR_SHA256), 64) diff --git a/cmux-tui/bindings/typescript/test/generated.test.ts b/cmux-tui/bindings/typescript/test/generated.test.ts index 986f999123cb..be059a326fde 100644 --- a/cmux-tui/bindings/typescript/test/generated.test.ts +++ b/cmux-tui/bindings/typescript/test/generated.test.ts @@ -12,7 +12,7 @@ import { test("generated protocol coverage matches the canonical v10 IR", () => { assert.equal(MUX_PROTOCOL_VERSION, 10); assert.equal(SDK_SCHEMA_VERSION, 2); - assert.equal(Object.keys(COMMAND_METADATA).length, 91); + assert.equal(Object.keys(COMMAND_METADATA).length, 92); assert.equal(Object.keys(EVENT_METADATA).length, 44); assert.equal(SDK_IR_SHA256.length, 64); assert.deepEqual(Object.keys(PROFILES).sort(), [ diff --git a/cmux-tui/bindings/zig/examples/watch.zig b/cmux-tui/bindings/zig/examples/watch.zig index 85ee01610745..ab2cb8c97e76 100644 --- a/cmux-tui/bindings/zig/examples/watch.zig +++ b/cmux-tui/bindings/zig/examples/watch.zig @@ -126,7 +126,7 @@ test "package consumer imports handwritten root and generated raw module" { wheel.pointer_frame_seq, ); try std.testing.expectEqual( - @as(usize, 91), + @as(usize, 92), cmux.raw.protocol.command_count, ); try std.testing.expectEqual( diff --git a/cmux-tui/bindings/zig/src/raw.zig b/cmux-tui/bindings/zig/src/raw.zig index 06dccbd5787d..2d9b9dd82bae 100644 --- a/cmux-tui/bindings/zig/src/raw.zig +++ b/cmux-tui/bindings/zig/src/raw.zig @@ -44,7 +44,7 @@ test { _ = @import("raw/wire_presence_test.zig"); _ = @import("raw/generated/presence_test.zig"); std.testing.refAllDecls(protocol); - try std.testing.expectEqual(@as(usize, 91), protocol.command_count); + try std.testing.expectEqual(@as(usize, 92), protocol.command_count); for ([_][]const u8{ "browser-frame-presented", "browser-key-press", From 5881a74e1569dee6e33d0534f79dd12266ff0717 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:54:59 -0700 Subject: [PATCH 275/521] test(tui): expose projection before terminal adoption --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 21b3bdade953..b942089e5ce6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -15189,6 +15189,44 @@ mod tests { surface.kill(); } + #[test] + fn projected_effect_preserves_restored_terminal_before_surface_adoption() { + let mux = test_mux(); + let surface = mux.new_workspace(Some("Restored".into()), Some((80, 24))).unwrap(); + let identity = surface.resource_identity().unwrap().clone(); + let ContentPublicId::Terminal(terminal_id) = identity.content_id.clone() else { + panic!("workspace fixture must create a terminal"); + }; + let host = mux.resource_terminal_host_identity(&surface).unwrap(); + let removed = mux.state.lock().unwrap().surfaces.remove(&surface.id); + assert!(removed.is_some(), "surface must exist before simulating delayed adoption"); + + let projection = mux + .resource_effect_projection() + .expect("a restored terminal may be projected before its surface is adopted"); + + assert!(projection.patch.changes.iter().any(|change| matches!( + change, + ResourceChange::UpsertTab(tab) + if tab.public_id == identity.tab_id + && tab.content_id == ContentPublicId::Terminal(terminal_id.clone()) + ))); + assert!(projection.patch.changes.iter().any(|change| matches!( + change, + ResourceChange::UpsertTerminal { public_id, terminal } + if public_id == &terminal_id && terminal.terminal_id == host.terminal_id + ))); + assert!(!projection.patch.changes.iter().any(|change| matches!( + change, + ResourceChange::TombstoneTab { tab_id } if tab_id == &identity.tab_id + ))); + assert!(!projection.patch.changes.iter().any(|change| matches!( + change, + ResourceChange::TombstoneTerminal { public_id, .. } if public_id == &terminal_id + ))); + surface.kill(); + } + #[test] fn projected_effect_holds_writer_fence_through_commit_and_publishes_once() { use std::sync::mpsc; From 2efea80b44441bae63337f7caa7b4c1ab5dd89f0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:58:34 -0700 Subject: [PATCH 276/521] fix(tui): preserve resources before surface adoption --- .../cmux-tui-core/src/mux/resource_content.rs | 98 ++++++++++++++++++- .../crates/cmux-tui-core/src/resource_api.rs | 83 +++++++++------- 2 files changed, 141 insertions(+), 40 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs index 568b8ce905a2..c3ef31f4510c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs @@ -7,6 +7,7 @@ use super::{Mux, ResourceMutationMetrics, ResourceMutationPlan}; use crate::browser::{BrowserSource, BrowserStatus}; use crate::model::{Node, State}; use crate::resource::{ContentPublicId, PanePublicId, SplitPublicId, WorkspacePublicId}; +use crate::resource_api::{public_browser_snapshot, public_terminal_snapshot}; use crate::workspace_registry::{ RegistryBrowser, RegistryBrowserLaunch, RegistryBrowserSource, RegistryBrowserStatus, RegistryLayoutNode, RegistryPane, RegistryScreen, RegistryTab, RegistryViewport, @@ -413,6 +414,11 @@ impl Mux { .iter() .map(|browser| (browser.public_id.clone(), browser.clone())) .collect::>(); + let before_tabs = before + .tabs + .iter() + .map(|tab| (tab.public_id.clone(), tab.clone())) + .collect::>(); let before_pane_ordinals = before .panes .iter() @@ -521,9 +527,95 @@ impl Mux { let mut tab_order = Vec::with_capacity(pane.tabs.len()); for (position, surface_slot) in pane.tabs.iter().enumerate() { - let surface = state.surfaces.get(surface_slot).with_context(|| { - format!("pane references missing surface {surface_slot}") - })?; + let Some(surface) = state.surfaces.get(surface_slot) else { + let tab_id = state + .resource_indexes + .tab_ids + .get(surface_slot) + .cloned() + .with_context(|| { + format!( + "pane references missing surface {surface_slot} without a tab identity" + ) + })?; + let content_id = state + .resource_indexes + .content_ids + .get(surface_slot) + .cloned() + .with_context(|| { + format!( + "pane references missing surface {surface_slot} without a content identity" + ) + })?; + let mut tab = before_tabs.get(&tab_id).cloned().with_context(|| { + format!("restored tab {tab_id} has no durable record") + })?; + anyhow::ensure!( + tab.content_id == content_id, + "restored tab {tab_id} changed content identity" + ); + tab.pane_id = pane.public_id.clone(); + tab.position = position; + live_tabs.insert(tab_id.clone()); + tab_order.push(tab_id); + + let public_content = match &tab.content_id { + ContentPublicId::Terminal(terminal_id) => { + live_terminals.insert(terminal_id.clone()); + let host_id = tab.terminal_id.as_deref().context( + "restored terminal tab omitted its durable host identity", + )?; + let terminal = terminal_records + .get(host_id) + .cloned() + .with_context(|| { + format!( + "restored terminal tab references missing {host_id}" + ) + })?; + let value = public_terminal_snapshot(&tab, &terminal, None)?; + changes.push(ResourceChange::UpsertTerminal { + public_id: terminal_id.clone(), + terminal, + }); + ("terminal", terminal_id.to_string(), value) + } + ContentPublicId::Browser(browser_id) => { + live_browsers.insert(browser_id.clone()); + let browser = before_browsers + .get(browser_id) + .cloned() + .with_context(|| { + format!( + "restored browser tab references missing {browser_id}" + ) + })?; + let value = public_browser_snapshot(&tab, &browser, None); + changes.push(ResourceChange::UpsertBrowser(browser)); + ("browser", browser_id.to_string(), value) + } + }; + changes.push(ResourceChange::UpsertTab(tab.clone())); + let content_kind = match &tab.content_id { + ContentPublicId::Terminal(_) => "terminal", + ContentPublicId::Browser(_) => "browser", + }; + public.push(( + "tab", + tab.public_id.to_string(), + json!({ + "id":tab.public_id, + "pane_id":tab.pane_id, + "index":tab.position, + "name":tab.name, + "content_kind":content_kind, + "content_id":tab.content_id.as_str(), + }), + )); + public.push(public_content); + continue; + }; let identity = surface.resource_identity().with_context(|| { format!("pane surface {surface_slot} has no resource identity") })?; diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_api.rs b/cmux-tui/crates/cmux-tui-core/src/resource_api.rs index 82cf755980b2..75b405e41955 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_api.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_api.rs @@ -20,8 +20,8 @@ use crate::resource::{ use crate::sidebar_resource::{sidebar_snapshot, sidebar_view_id}; use crate::workspace_registry::{ RegistryBrowser, RegistryBrowserLaunch, RegistryBrowserSource, RegistryBrowserStatus, - RegistryLayoutNode, RegistryPane, RegistryScreen, RegistryTab, RegistryViewport, - ResourceEffectOutcome, ResourceEffectPreparation, TerminalLifecycle, + RegistryLayoutNode, RegistryPane, RegistryScreen, RegistryTab, RegistryTerminal, + RegistryViewport, ResourceEffectOutcome, ResourceEffectPreparation, TerminalLifecycle, }; use crate::{Mux, ResourceSelectors}; @@ -499,7 +499,7 @@ pub(crate) fn public_session_snapshot(mux: &Mux) -> Result .tabs .iter() .filter_map(|tab| { - let ContentPublicId::Terminal(terminal_id) = &tab.content_id else { + let ContentPublicId::Terminal(_) = &tab.content_id else { return None; }; Some((|| { @@ -510,44 +510,12 @@ pub(crate) fn public_session_snapshot(mux: &Mux) -> Result let durable = terminals_by_id .get(durable_id) .with_context(|| format!("terminal tab references missing {durable_id}"))?; - let lifecycle = match durable.lifecycle { - TerminalLifecycle::Launching | TerminalLifecycle::Adopting => "launching", - TerminalLifecycle::Running => "running", - TerminalLifecycle::Exited => "exited", - TerminalLifecycle::Tombstoned => { - anyhow::bail!("live terminal tab references a tombstoned terminal") - } - }; let surface = state .resource_indexes .content .get(&tab.content_id) .and_then(|slot| state.surfaces.get(slot)); - let (cols, rows) = surface.map(|surface| surface.size()).unwrap_or((80, 24)); - let mut terminal = json!({ - "id": terminal_id, - "tab_id": tab.public_id, - "title": surface.map(|surface| surface.title()).unwrap_or_default(), - "cols": cols.max(1), - "rows": rows.max(1), - "running": durable.lifecycle == TerminalLifecycle::Running, - "lifecycle": lifecycle, - }); - if let Some(cwd) = surface.and_then(|surface| surface.spawn_cwd()) { - terminal["cwd"] = json!(cwd); - } - if durable.lifecycle == TerminalLifecycle::Exited { - terminal["exit"] = durable - .exit - .clone() - .context("exited terminal omitted its durable outcome")?; - } else { - anyhow::ensure!( - durable.exit.is_none(), - "non-exited terminal unexpectedly has a durable outcome" - ); - } - Ok(terminal) + public_terminal_snapshot(tab, durable, surface) })()) }) .collect::>>()?; @@ -764,7 +732,48 @@ fn public_layout_node( } } -fn public_browser_snapshot( +pub(crate) fn public_terminal_snapshot( + tab: &RegistryTab, + durable: &RegistryTerminal, + surface: Option<&Arc>, +) -> anyhow::Result { + let ContentPublicId::Terminal(terminal_id) = &tab.content_id else { + anyhow::bail!("terminal snapshot requires terminal content") + }; + let lifecycle = match durable.lifecycle { + TerminalLifecycle::Launching | TerminalLifecycle::Adopting => "launching", + TerminalLifecycle::Running => "running", + TerminalLifecycle::Exited => "exited", + TerminalLifecycle::Tombstoned => { + anyhow::bail!("live terminal tab references a tombstoned terminal") + } + }; + let (cols, rows) = surface.map(|surface| surface.size()).unwrap_or((80, 24)); + let mut terminal = json!({ + "id": terminal_id, + "tab_id": tab.public_id, + "title": surface.map(|surface| surface.title()).unwrap_or_default(), + "cols": cols.max(1), + "rows": rows.max(1), + "running": durable.lifecycle == TerminalLifecycle::Running, + "lifecycle": lifecycle, + }); + if let Some(cwd) = surface.and_then(|surface| surface.spawn_cwd()) { + terminal["cwd"] = json!(cwd); + } + if durable.lifecycle == TerminalLifecycle::Exited { + terminal["exit"] = + durable.exit.clone().context("exited terminal omitted its durable outcome")?; + } else { + anyhow::ensure!( + durable.exit.is_none(), + "non-exited terminal unexpectedly has a durable outcome" + ); + } + Ok(terminal) +} + +pub(crate) fn public_browser_snapshot( tab: &RegistryTab, durable: &RegistryBrowser, surface: Option<&Arc>, From 8b1eaf9e792a1302e4682d15bf585c6b332f3bbd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:33:08 -0700 Subject: [PATCH 277/521] test(tui): expose acknowledged stream close race --- cmux-tui/bindings/go/resource_api_test.go | 74 +++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/cmux-tui/bindings/go/resource_api_test.go b/cmux-tui/bindings/go/resource_api_test.go index 555f38f1258a..6871ac836fab 100644 --- a/cmux-tui/bindings/go/resource_api_test.go +++ b/cmux-tui/bindings/go/resource_api_test.go @@ -2695,6 +2695,80 @@ func TestTypedStreamEndAndCancellation(t *testing.T) { } } +func TestAcknowledgedStreamOpenSurvivesTerminalTransportClose(t *testing.T) { + clientSide, serverSide := net.Pipe() + releaseWrite := make(chan struct{}) + heldClientSide := &heldWriteReturnConn{ + Conn: clientSide, + release: releaseWrite, + } + go func() { + defer serverSide.Close() + open := readRequest(t, bufio.NewReader(serverSide)) + streamID := requestParams(t, open)["stream_id"] + var batch bytes.Buffer + encoder := json.NewEncoder(&batch) + for _, envelope := range []map[string]any{ + { + "protocol": "cmux.protocol/1", + "type": "response", + "id": open["id"], + "ok": true, + "result": map[string]any{"stream_id": streamID}, + }, + { + "protocol": "cmux.protocol/1", + "type": "stream_end", + "stream_id": streamID, + "reason": "completed", + }, + } { + if err := encoder.Encode(envelope); err != nil { + t.Errorf("encode server batch: %v", err) + return + } + } + if _, err := serverSide.Write(batch.Bytes()); err != nil { + t.Errorf("write server batch: %v", err) + } + }() + client, err := NewClient(context.Background(), ClientOptions{ + DialContext: func(context.Context, string, string) (net.Conn, error) { + return heldClientSide, nil + }, + }) + if err != nil { + t.Fatal(err) + } + go func() { + <-client.done + close(releaseWrite) + }() + session := client.Machine(SelectID(testMachineID)).Session(SelectID(testSessionID)) + stream, err := session.Events(context.Background(), SessionEventsOptions{}) + if err != nil { + t.Fatalf("open acknowledged stream: %v", err) + } + _, err = stream.Recv(context.Background()) + var end *StreamEndError + if !errors.As(err, &end) || end.Reason != "completed" { + t.Fatalf("stream end = %T %#v", err, err) + } +} + +type heldWriteReturnConn struct { + net.Conn + release <-chan struct{} +} + +func (c *heldWriteReturnConn) Write(value []byte) (int, error) { + count, err := c.Conn.Write(value) + if err == nil && count == len(value) { + <-c.release + } + return count, err +} + func TestCancelPreservesOpeningRouteAndServerEnd(t *testing.T) { clientSide, serverSide := net.Pipe() cancelRequests := make(chan map[string]any, 1) From 89c5d087d43fbcbbf3714904de7371c182e229cc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:35:22 -0700 Subject: [PATCH 278/521] fix(tui): preserve completed Go stream opens --- cmux-tui/bindings/go/client.go | 10 ++++++++++ cmux-tui/bindings/go/stream.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmux-tui/bindings/go/client.go b/cmux-tui/bindings/go/client.go index 4fa943400908..63441e147c84 100644 --- a/cmux-tui/bindings/go/client.go +++ b/cmux-tui/bindings/go/client.go @@ -122,6 +122,7 @@ type streamRoute struct { mu sync.Mutex accepting bool terminated bool + serverEnded bool queuedBytes int cancelParams map[string]any openDispatched bool @@ -1266,6 +1267,12 @@ func (r *streamRoute) markOpenAcknowledged() { r.mu.Unlock() } +func (r *streamRoute) hasServerEnd() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.serverEnded +} + func (r *streamRoute) failedOpenCancelParams() (map[string]any, bool) { r.mu.Lock() defer r.mu.Unlock() @@ -1425,6 +1432,9 @@ func (r *streamRoute) deliver(message streamMessage) bool { } select { case r.messages <- message: + if message.envelope.Type == "stream_end" { + r.serverEnded = true + } r.queuedBytes += message.size return true default: diff --git a/cmux-tui/bindings/go/stream.go b/cmux-tui/bindings/go/stream.go index f9cac31f0250..c3d00edbc30d 100644 --- a/cmux-tui/bindings/go/stream.go +++ b/cmux-tui/bindings/go/stream.go @@ -423,7 +423,7 @@ func openStream[T any]( }) } client.mu.Lock() - if client.closed { + if client.closed && !route.hasServerEnd() { openError := client.err if openError == nil { openError = ErrClosed From 8cc6f0ff34c37845aaa252bfbb48ec813e1139ab Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:48:56 -0700 Subject: [PATCH 279/521] test(tui): synchronize Python disconnect fixture --- cmux-tui/bindings/python/tests/test_resource_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/bindings/python/tests/test_resource_api.py b/cmux-tui/bindings/python/tests/test_resource_api.py index c9c1472a671b..4688124fdc04 100644 --- a/cmux-tui/bindings/python/tests/test_resource_api.py +++ b/cmux-tui/bindings/python/tests/test_resource_api.py @@ -2110,13 +2110,13 @@ def deliver_then_fail(encoded): self.assertIs(raised.exception, expected) self.assertTrue(client.closed) self.assertTrue(client._connection._wire.closed) - self.assertTrue(disconnected.wait(0.5)) started = time.monotonic() with self.assertRaises(CmuxConnectionError): session.ping() self.assertLess(time.monotonic() - started, 0.1) + self.assertTrue(disconnected.is_set()) self.assertEqual( [request["operation"] for request in observed], ["session.events"], From bcf48a558d289201f7fcfa0bf792218a34698863 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:54:43 -0700 Subject: [PATCH 280/521] test(tui): record Python broken-pipe disconnects --- .../python/tests/test_resource_api.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cmux-tui/bindings/python/tests/test_resource_api.py b/cmux-tui/bindings/python/tests/test_resource_api.py index 4688124fdc04..ef3001b1bec3 100644 --- a/cmux-tui/bindings/python/tests/test_resource_api.py +++ b/cmux-tui/bindings/python/tests/test_resource_api.py @@ -2076,15 +2076,17 @@ def test_stream_open_send_error_after_delivery_closes_without_cancel( disconnected = threading.Event() def handler(connection, _index): - for request in frames(connection): - observed.append(request) - open_seen.set() - ok( - connection, - request, - {"stream_id": request["params"]["stream_id"]}, - ) - disconnected.set() + try: + for request in frames(connection): + observed.append(request) + open_seen.set() + ok( + connection, + request, + {"stream_id": request["params"]["stream_id"]}, + ) + finally: + disconnected.set() expected = CmuxConnectionError( "synthetic failure after the complete frame was delivered" From dd74247194fa792efe6d8288a70e7eebda25d16d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:55:18 -0700 Subject: [PATCH 281/521] test(tui): expose explicit session socket precedence --- cmux-tui/crates/cmux-tui/src/cli.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 170f05d6091e..ac7a5547d6e3 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -611,14 +611,21 @@ fn print_server_status(output: OutputMode, probe: &crate::server_lifecycle::Serv } fn resolve_server_socket(global: &GlobalArgs) -> PathBuf { + resolve_server_socket_with(global, |name| std::env::var_os(name).map(PathBuf::from)) +} + +fn resolve_server_socket_with( + global: &GlobalArgs, + ambient_socket: impl Fn(&str) -> Option, +) -> PathBuf { if let Some(path) = &global.socket { return path.clone(); } for name in ["CMUX_TUI_SOCKET", "CMUX_MUX_SOCKET"] { - if let Some(path) = std::env::var_os(name) - && !path.is_empty() + if let Some(path) = ambient_socket(name) + && !path.as_os_str().is_empty() { - return PathBuf::from(path); + return path; } } cmux_tui_core::server::default_socket_path(global.session.as_deref().unwrap_or("main")) @@ -659,6 +666,18 @@ mod tests { ); } + #[test] + fn explicit_session_overrides_ambient_server_socket() { + let global = + GlobalArgs { session: Some("explicit-session".into()), ..GlobalArgs::default() }; + let ambient = PathBuf::from("/tmp/cmux-ambient-session.sock"); + + assert_eq!( + resolve_server_socket_with(&global, |_| Some(ambient.clone())), + cmux_tui_core::server::default_socket_path("explicit-session") + ); + } + #[test] fn every_scope_has_dedicated_help() { for scope in PUBLIC_SCOPES { From caba904498e23babf3662b1bca12d76af64d6b74 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:58:54 -0700 Subject: [PATCH 282/521] fix(tui): honor explicit session routing --- cmux-tui/crates/cmux-tui/src/cli.rs | 5 ++++- cmux-tui/crates/cmux-tui/src/cli/raw.rs | 17 +---------------- cmux-tui/crates/cmux-tui/src/cli/wire.rs | 17 +---------------- 3 files changed, 6 insertions(+), 33 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index ac7a5547d6e3..12c3cb9ecc12 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -621,6 +621,9 @@ fn resolve_server_socket_with( if let Some(path) = &global.socket { return path.clone(); } + if let Some(session) = global.session.as_deref() { + return cmux_tui_core::server::default_socket_path(session); + } for name in ["CMUX_TUI_SOCKET", "CMUX_MUX_SOCKET"] { if let Some(path) = ambient_socket(name) && !path.as_os_str().is_empty() @@ -628,7 +631,7 @@ fn resolve_server_socket_with( return path; } } - cmux_tui_core::server::default_socket_path(global.session.as_deref().unwrap_or("main")) + cmux_tui_core::server::default_socket_path("main") } #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui/src/cli/raw.rs b/cmux-tui/crates/cmux-tui/src/cli/raw.rs index 2c418b1dd5c5..2f867d643fb7 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/raw.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/raw.rs @@ -4,7 +4,6 @@ //! internal fields and receives no public compatibility guarantees. use std::io::{self, BufRead, BufReader, Read, Write}; -use std::path::PathBuf; use std::time::Duration; use cmux_tui_core::platform::transport; @@ -31,7 +30,7 @@ pub(super) fn run(global: GlobalArgs, plan: RawCommandPlan) -> i32 { return 2; } }; - let socket = resolve_socket(&global); + let socket = super::resolve_server_socket(&global); let stream = match transport::connect(&socket) { Ok(stream) => stream, Err(error) => { @@ -132,20 +131,6 @@ fn read_line_limited( .map_err(|error| format!("protocol error: raw response is not UTF-8: {error}")) } -fn resolve_socket(global: &GlobalArgs) -> PathBuf { - if let Some(path) = &global.socket { - return path.clone(); - } - for name in ["CMUX_TUI_SOCKET", "CMUX_MUX_SOCKET"] { - if let Some(path) = std::env::var_os(name) - && !path.is_empty() - { - return PathBuf::from(path); - } - } - cmux_tui_core::server::default_socket_path(global.session.as_deref().unwrap_or("main")) -} - #[cfg(test)] mod tests { use super::*; diff --git a/cmux-tui/crates/cmux-tui/src/cli/wire.rs b/cmux-tui/crates/cmux-tui/src/cli/wire.rs index 58c9b6f61a1e..6c2513f2f0cc 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/wire.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/wire.rs @@ -1,5 +1,4 @@ use std::io::{self, BufRead, BufReader, Read, Write}; -use std::path::PathBuf; use std::time::Duration; use cmux_tui_core::platform::transport; @@ -54,7 +53,7 @@ pub(super) fn run(global: GlobalArgs, mut plan: RequestPlan) -> i32 { let request_id = request["id"].as_str().expect("locally built request IDs are strings").to_string(); - let socket = resolve_socket(&global); + let socket = super::resolve_server_socket(&global); let stream = match transport::connect(&socket) { Ok(stream) => stream, Err(error) => { @@ -527,20 +526,6 @@ fn human_key_rank(key: &str) -> usize { } } -fn resolve_socket(global: &GlobalArgs) -> PathBuf { - if let Some(path) = &global.socket { - return path.clone(); - } - for name in ["CMUX_TUI_SOCKET", "CMUX_MUX_SOCKET"] { - if let Some(path) = std::env::var_os(name) - && !path.is_empty() - { - return PathBuf::from(path); - } - } - cmux_tui_core::server::default_socket_path(global.session.as_deref().unwrap_or("main")) -} - #[cfg(test)] mod tests { use super::*; From c7032a4fb302f9d134c17b16425ed1ac83a5e765 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:33:05 -0700 Subject: [PATCH 283/521] test(tui): expose final packaging and startup regressions --- cmux-tui/crates/cmux-tui/src/cli.rs | 15 +++++++++++++++ tests/test_tui_package_launchers.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 12c3cb9ecc12..b56e23c80756 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -669,6 +669,21 @@ mod tests { ); } + #[test] + fn startup_option_values_cannot_select_cli_mode() { + for args in [ + &["--term", "server"][..], + &["--state", "server"][..], + &["--machine-provider-command", "server", "--"][..], + &["--machine-provider-command", "provider", "server", "--"][..], + ] { + assert!( + !is_cli_invocation(&strings(args)), + "startup invocation misclassified: {args:?}" + ); + } + } + #[test] fn explicit_session_overrides_ambient_server_socket() { let global = diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index c59aa9ded4f0..60b62eeba951 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -14,7 +14,7 @@ VERIFY_SCRIPT = ROOT / "cmux-tui/dist/scripts/verify_artifact_identity.py" NPM_LAUNCHER = ROOT / "cmux-tui/dist/npm/cmux/bin/cmux.js" PYPI_PACKAGER = ROOT / "cmux-tui/dist/scripts/package_pypi.py" -EXPECTED_IDENTITY = "cmux-tui 1.2.3 (build123; ghostty ghost456)" +EXPECTED_IDENTITY = "cmux 1.2.3 (build123; ghostty ghost456)" NPM_PACKAGE_BY_PLATFORM = { ("darwin", "arm64"): "cmux-tui-darwin-arm64", From 06382d788e3c232b698935f4edf4bfe0740e19f4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:34:35 -0700 Subject: [PATCH 284/521] fix(tui): align artifact and startup routing contracts --- cmux-tui/crates/cmux-tui/src/cli.rs | 4 +++- cmux-tui/dist/scripts/verify_artifact_identity.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index b56e23c80756..78ef75cb6d05 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -78,7 +78,9 @@ pub fn is_cli_invocation(args: &[String]) -> bool { "-h" | "--help" | "help" => return true, "server" => return true, value if PUBLIC_SCOPES.contains(&value) => return true, - value if value.starts_with('-') => index += 1, + // Any other option belongs to startup parsing (or is invalid). + // Do not inspect its values because they may match a CLI scope. + value if value.starts_with('-') => return false, _ => return false, } } diff --git a/cmux-tui/dist/scripts/verify_artifact_identity.py b/cmux-tui/dist/scripts/verify_artifact_identity.py index 1dc12f2d12d6..81c0a39b259f 100644 --- a/cmux-tui/dist/scripts/verify_artifact_identity.py +++ b/cmux-tui/dist/scripts/verify_artifact_identity.py @@ -21,7 +21,7 @@ def parse_args() -> argparse.Namespace: def main() -> None: args = parse_args() expected = ( - f"cmux-tui {args.version} " + f"cmux {args.version} " f"({args.build_commit}; ghostty {args.ghostty_commit})" ) try: From 6224c73cc6da62199e1f4d0816fece378aff60a8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:05:01 -0700 Subject: [PATCH 285/521] test(tui): expose restored host close leak --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 16 ++++++++++ .../src/resource_router/content.rs | 30 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b942089e5ce6..b1f58864bab6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1932,6 +1932,8 @@ pub struct Mux { resource_projection_before_commit: Mutex>>, #[cfg(test)] resource_close_after_commit: Mutex>>, + #[cfg(test)] + discovered_terminal_termination_requests: Mutex)>>, browser_runtime: BrowserRuntimeSlot, cell_pixels: Mutex<(u16, u16)>, default_colors: Mutex, @@ -2263,6 +2265,8 @@ impl Mux { resource_projection_before_commit: Mutex::new(None), #[cfg(test)] resource_close_after_commit: Mutex::new(None), + #[cfg(test)] + discovered_terminal_termination_requests: Mutex::new(Vec::new()), browser_runtime: BrowserRuntimeSlot::default(), cell_pixels: Mutex::new((8, 16)), default_colors: Mutex::new(default_colors), @@ -6947,6 +6951,11 @@ impl Mux { } fn terminate_discovered_terminal_host(&self, terminal_id: &str, incarnation: Option<&str>) { + #[cfg(test)] + self.discovered_terminal_termination_requests + .lock() + .unwrap() + .push((terminal_id.to_string(), incarnation.map(str::to_string))); #[cfg(unix)] { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); @@ -6967,6 +6976,13 @@ impl Mux { let _ = (terminal_id, incarnation); } + #[cfg(test)] + pub(crate) fn take_discovered_terminal_termination_requests_for_test( + &self, + ) -> Vec<(String, Option)> { + std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) + } + fn terminate_tombstoned_workspace_hosts(&self, workspace_key: &str) { #[cfg(unix)] { diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs index 0865b7739c9c..9208b1aca665 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs @@ -2098,6 +2098,36 @@ mod tests { assert_eq!(mux.terminal_registry_snapshot().unwrap().revision, before_terminal + 1); } + #[cfg(unix)] + #[test] + fn terminal_close_terminates_a_restored_host_before_acknowledgement() { + let (mux, surface, selectors) = terminal_fixture(None); + let host = mux.resource_terminal_host_identity(&surface).unwrap(); + let detached = mux + .remove_surface_runtime_for_test(surface.id) + .expect("fixture surface is live before simulating delayed adoption"); + assert!(mux.take_discovered_terminal_termination_requests_for_test().is_empty()); + + let closed = dispatch( + &mux, + parsed_request( + "terminal.close", + &selectors, + json!({}), + Some("close-restored-terminal-host"), + ), + ) + .unwrap(); + + assert_eq!(closed["replayed"], false); + assert_eq!( + mux.take_discovered_terminal_termination_requests_for_test(), + vec![(host.terminal_id, Some(host.incarnation))], + "terminal.close acknowledged before terminating the exact restored host" + ); + detached.kill(); + } + #[test] fn terminal_viewport_scroll_uses_one_bounded_receipt_without_session_journal_churn() { let (mux, surface, selectors) = terminal_fixture(None); From 8e50e5a2fa8f25d55df4a7401f62d433965a3a8c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:09:22 -0700 Subject: [PATCH 286/521] fix(tui): terminate restored hosts on close --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 28 +++++++++++++++---- .../src/mux/resource_topology.rs | 8 +++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index b1f58864bab6..5aac7e366a79 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6951,11 +6951,21 @@ impl Mux { } fn terminate_discovered_terminal_host(&self, terminal_id: &str, incarnation: Option<&str>) { + self.terminate_discovered_terminal_hosts(&[( + terminal_id.to_string(), + incarnation.map(str::to_string), + )]); + } + + fn terminate_discovered_terminal_hosts(&self, terminals: &[(String, Option)]) { + if terminals.is_empty() { + return; + } #[cfg(test)] self.discovered_terminal_termination_requests .lock() .unwrap() - .push((terminal_id.to_string(), incarnation.map(str::to_string))); + .extend(terminals.iter().cloned()); #[cfg(unix)] { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); @@ -6964,16 +6974,24 @@ impl Mux { else { return; }; + let targets = terminals + .iter() + .map(|(terminal_id, incarnation)| (terminal_id.as_str(), incarnation.as_deref())) + .collect::>(); for (path, record) in records { - if record.terminal_id == terminal_id - && incarnation.is_none_or(|expected| record.incarnation == expected) - { + let Some(expected_incarnation) = targets.get(record.terminal_id.as_str()) else { + continue; + }; + if match *expected_incarnation { + Some(expected) => record.incarnation == expected, + None => true, + } { terminate_host_record(record, path); } } } #[cfg(not(unix))] - let _ = (terminal_id, incarnation); + let _ = terminals; } #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 2e98a916d950..58ecdae8c74a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -43,6 +43,7 @@ struct ResourceClosePlan { state: State, removed: Vec>, terminals: Vec<(String, Option)>, + unmaterialized_terminals: Vec<(String, Option)>, workspace_close: Option, closed_workspace_key: Option, delta: TreeDelta, @@ -2086,6 +2087,7 @@ impl Mux { drop(registry); drop(workspace_lifecycle); + self.terminate_discovered_terminal_hosts(&plan.unmaterialized_terminals); self.notify_terminal_exit_waiters(closed_public_ids); self.publish_resource_event(); for surface in plan.removed { @@ -2170,6 +2172,7 @@ impl Mux { let mut removed = Vec::new(); let mut terminals = Vec::new(); + let mut unmaterialized_terminals = Vec::new(); for surface_id in &surface_ids { if let Some(surface) = state.surfaces.get(surface_id).cloned() { if let Some(identity) = self.resource_terminal_host_identity(&surface) { @@ -2185,7 +2188,9 @@ impl Mux { let incarnation = registry .terminal_record(&terminal_id)? .and_then(|terminal| terminal.incarnation); - terminals.push((terminal_id, incarnation)); + let identity = (terminal_id, incarnation); + terminals.push(identity.clone()); + unmaterialized_terminals.push(identity); } } let mut split_index_changed = false; @@ -2247,6 +2252,7 @@ impl Mux { state: projected, removed, terminals, + unmaterialized_terminals, workspace_close, closed_workspace_key, delta, From ca35eb25a361d089da025a53fe0a1bc79a3a871e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:18:01 -0700 Subject: [PATCH 287/521] fix(tui): keep Linux strict lint clean --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 - cmux-tui/crates/cmux-tui/src/cli/command.rs | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5aac7e366a79..2396e245f280 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -14928,7 +14928,6 @@ mod tests { assert_eq!(resolved_rx.recv().unwrap(), first_id); let concurrent = { let mux = mux.clone(); - let overlap = overlap; let second_id = second_id.clone(); std::thread::spawn(move || { overlap.wait(); diff --git a/cmux-tui/crates/cmux-tui/src/cli/command.rs b/cmux-tui/crates/cmux-tui/src/cli/command.rs index 28c86845ef5f..ad7fe79d0703 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/command.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/command.rs @@ -2147,13 +2147,13 @@ pub(super) fn run_provider_authority(global: GlobalArgs, plan: ProviderAuthority "--authority-file".into(), plan.authority_file, ]; - return match crate::provider_authority::try_run(&args) { + match crate::provider_authority::try_run(&args) { Some(0) => super::wire::print_local_success( &json!({"installed": true, "generation": output_generation}), output, ), Some(_) | None => 1, - }; + } } #[cfg(not(target_os = "linux"))] { From 7bbf40e0f2c3e9c5b54b5a860b5661a741c6e62e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:26:35 -0700 Subject: [PATCH 288/521] fix(tui): restore portable child wait ownership --- cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 1853a483a90d..14aea8e721c4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -240,6 +240,15 @@ impl SpawnedPtyChild { self.child.as_ref().expect("spawned PTY child is present").clone_killer() } + #[cfg(not(unix))] + pub(crate) fn wait(&mut self) -> std::io::Result { + let result = self.child.as_mut().expect("spawned PTY child is present").wait(); + if result.is_ok() { + self.child.take(); + } + result + } + /// Release a child only after stable identity proves that this process no /// longer owns its wait handle and the numeric PID belongs to no live /// process from the original publication. From 1028636bc38bc476931f7ab0a88b3ea4b9c2ebd4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:36:30 -0700 Subject: [PATCH 289/521] test(tui): expose server lifecycle locale drift --- cmux-tui/crates/cmux-tui/tests/cli.rs | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 2abd13b60f30..0df37b11b690 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1456,6 +1456,36 @@ fn server_status_and_stop_control_a_compatible_headless_session() { assert!(!server.socket.exists()); } +#[test] +fn server_lifecycle_help_and_machine_rejection_are_localized() { + let help = Command::new(bin()) + .env("LC_ALL", "ja_JP.UTF-8") + .env("LC_MESSAGES", "ja_JP.UTF-8") + .env("LANG", "ja_JP.UTF-8") + .args(["server", "--help"]) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_success(&help); + let help = String::from_utf8(help.stdout).unwrap(); + assert!(help.contains("使用方法"), "{help}"); + assert!(help.contains("別のリリース"), "{help}"); + assert!(!help.contains("These local lifecycle commands"), "{help}"); + + let rejected = Command::new(bin()) + .env("LC_ALL", "ja_JP.UTF-8") + .env("LC_MESSAGES", "ja_JP.UTF-8") + .env("LANG", "ja_JP.UTF-8") + .args(["--machine", "remote", "server", "status"]) + .env_remove("CMUX_TUI_SOCKET") + .output() + .unwrap(); + assert_eq!(rejected.status.code(), Some(2)); + let rejected = String::from_utf8(rejected.stderr).unwrap(); + assert!(rejected.contains("--machine を使用できません"), "{rejected}"); + assert!(!rejected.contains("lifecycle commands cannot use"), "{rejected}"); +} + #[cfg(unix)] #[test] fn signal_termination_completes_server_shutdown() { From 587a744123f010e3ad333707c3577879702ff13e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:38:08 -0700 Subject: [PATCH 290/521] fix(tui): localize server lifecycle CLI --- cmux-tui/crates/cmux-tui/src/cli.rs | 14 +++----------- cmux-tui/crates/cmux-tui/src/localization.rs | 7 +++++-- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 78ef75cb6d05..d09080e06488 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -446,15 +446,6 @@ USAGE escape for the legacy control protocol and provides no compatibility promise. "; -const SERVER_HELP: &str = "\ -USAGE - cmux server status - cmux server stop - -These local lifecycle commands remain available when the running server has a -different release identity. -"; - fn run_server_lifecycle_if_requested(args: &[String]) -> Option { let (global, command) = match parse_globals(args) { Ok(parsed) => parsed, @@ -471,8 +462,9 @@ fn run_server_lifecycle_if_requested(args: &[String]) -> Option { } let actions = &command[1..]; if actions.iter().any(|value| matches!(value.as_str(), "-h" | "--help" | "help")) { + let messages = &crate::localization::catalog().server; let mut stdout = io::stdout().lock(); - let _ = stdout.write_all(SERVER_HELP.as_bytes()); + let _ = stdout.write_all(messages.help.as_bytes()); let _ = stdout.flush(); return Some(0); } @@ -497,7 +489,7 @@ fn run_server_lifecycle(global: &GlobalArgs, actions: &[String]) -> i32 { } }; if global.machine.is_some() { - eprintln!("cmux: server lifecycle commands cannot use --machine"); + eprintln!("cmux: {}", messages.machine_scope_unsupported); return 2; } let socket_path = resolve_server_socket(global); diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index 51b2d32d4ee6..bb26afe5ced2 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -26,6 +26,7 @@ pub(crate) struct ForeignViewportMessages { #[derive(Debug, PartialEq, Eq)] pub(crate) struct ServerMessages { pub help: &'static str, + pub machine_scope_unsupported: &'static str, pub incompatible_local_server: &'static str, pub server_label: &'static str, pub client_label: &'static str, @@ -547,7 +548,8 @@ static ENGLISH: Catalog = Catalog { }, foreign_viewport: ForeignViewportMessages { terminal_grid: "terminal grid" }, server: ServerMessages { - help: "Inspect or stop the local server.", + help: "USAGE\n cmux server status\n cmux server stop\n\nThese local lifecycle commands remain available when the running server has a\ndifferent release identity.\n", + machine_scope_unsupported: "server lifecycle commands cannot use --machine", incompatible_local_server: "This session is running a different cmux-tui release.", server_label: "server", client_label: "client", @@ -840,7 +842,8 @@ static JAPANESE: Catalog = Catalog { }, foreign_viewport: ForeignViewportMessages { terminal_grid: "端末グリッド" }, server: ServerMessages { - help: "ローカルサーバーの確認または停止。", + help: "使用方法\n cmux server status\n cmux server stop\n\n実行中のサーバーが別のリリースでも、これらのローカルライフサイクル操作を利用できます。\n", + machine_scope_unsupported: "server のライフサイクル操作では --machine を使用できません", incompatible_local_server: "このセッションは別の cmux-tui リリースで実行されています。", server_label: "サーバー", client_label: "クライアント", From 2ccb1783b49cc07e70535fda35a5f68f207982d6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:56:12 -0700 Subject: [PATCH 291/521] test(tui): expose legacy publication race --- .../crates/cmux-tui/src/server_lifecycle.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index d65b7088df3c..4bf0f01dea0c 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1677,6 +1677,11 @@ mod tests { use std::process::{Command, Stdio}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; + #[cfg(unix)] + use std::sync::mpsc; + + #[cfg(unix)] + use cmux_tui_core::{Mux, SurfaceOptions}; use super::*; @@ -1747,6 +1752,45 @@ mod tests { (listener, quarantine, path) } + #[cfg(unix)] + #[test] + fn legacy_quarantine_blocks_replacement_publication() { + let (listener, mut quarantine, path) = quarantined_test_listener("cq-publication"); + let (sender, receiver) = mpsc::channel(); + let publication_path = path.clone(); + let publisher = std::thread::spawn(move || { + let mux = Mux::new("quarantine-publication-test", SurfaceOptions::default()); + let result = cmux_tui_core::server::serve_owned(mux, Some(publication_path)); + sender.send(result).unwrap(); + }); + + match receiver.recv_timeout(Duration::from_millis(250)) { + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(error) => panic!("replacement publication channel failed: {error}"), + Ok(result) => { + if let Ok(publication) = result { + publication.cleanup(); + } + quarantine.restore().unwrap(); + drop(quarantine); + drop(listener); + let _ = std::fs::remove_file(&path); + publisher.join().unwrap(); + panic!("replacement server published while the legacy socket was quarantined"); + } + } + + quarantine.restore().unwrap(); + drop(quarantine); + let result = receiver + .recv_timeout(Duration::from_secs(5)) + .expect("replacement publication did not resume after quarantine ownership ended"); + assert!(result.is_err(), "replacement publication replaced the restored live socket"); + publisher.join().unwrap(); + drop(listener); + std::fs::remove_file(path).unwrap(); + } + #[test] fn mismatch_message_contains_both_release_identities_and_stop_command() { let identity = ServerIdentity { From 7b1d4b1fd47bb255e67f156eb5c5ea8ce2a16487 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:16:34 -0700 Subject: [PATCH 292/521] test(tui): exercise guarded publication path --- cmux-tui/crates/cmux-tui/src/server_lifecycle.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 4bf0f01dea0c..de756a89a3b1 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1755,7 +1755,16 @@ mod tests { #[cfg(unix)] #[test] fn legacy_quarantine_blocks_replacement_publication() { - let (listener, mut quarantine, path) = quarantined_test_listener("cq-publication"); + let directory = std::env::temp_dir().join(format!( + "cq-publication-{}-{}", + std::process::id(), + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() + )); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("server.sock"); + let publication_lock = directory.join(".server.sock.publish.lock"); + let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); + let mut quarantine = LegacySocketQuarantine::acquire(&path).unwrap(); let (sender, receiver) = mpsc::channel(); let publication_path = path.clone(); let publisher = std::thread::spawn(move || { @@ -1776,6 +1785,8 @@ mod tests { drop(listener); let _ = std::fs::remove_file(&path); publisher.join().unwrap(); + let _ = std::fs::remove_file(&publication_lock); + std::fs::remove_dir(&directory).unwrap(); panic!("replacement server published while the legacy socket was quarantined"); } } @@ -1789,6 +1800,8 @@ mod tests { publisher.join().unwrap(); drop(listener); std::fs::remove_file(path).unwrap(); + std::fs::remove_file(publication_lock).unwrap(); + std::fs::remove_dir(directory).unwrap(); } #[test] From ed810ae573cf33993481519c51973110936a5339 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:33:28 -0700 Subject: [PATCH 293/521] fix(tui): fence legacy socket publication --- cmux-tui/crates/cmux-tui-core/src/server.rs | 326 +++++++++++++++++- .../crates/cmux-tui/src/server_lifecycle.rs | 52 ++- 2 files changed, 357 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 1dfb10022349..e1d3fb4b6253 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -3306,12 +3306,79 @@ fn socket_publication_lock_path(path: &Path) -> std::io::Result { Ok(parent.join(lock_name)) } -struct SocketPublicationLock { +#[cfg(unix)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct SocketPublicationLockIdentity { + device: u64, + inode: u64, +} + +#[cfg(unix)] +static SOCKET_PUBLICATION_LOCAL_STATE: std::sync::OnceLock<( + Mutex>, + Condvar, +)> = std::sync::OnceLock::new(); + +#[cfg(unix)] +struct SocketPublicationLocalGuard { + identity: SocketPublicationLockIdentity, +} + +#[cfg(unix)] +impl SocketPublicationLocalGuard { + fn acquire( + identity: SocketPublicationLockIdentity, + path: &Path, + deadline: Instant, + ) -> std::io::Result { + let (held, changed) = SOCKET_PUBLICATION_LOCAL_STATE.get_or_init(Default::default); + let mut held = held.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + if held.insert(identity) { + return Ok(Self { identity }); + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("timed out waiting to publish server socket {}", path.display()), + ) + })?; + held = match changed.wait_timeout(held, remaining) { + Ok((held, _)) => held, + Err(error) => error.into_inner().0, + }; + } + } +} + +#[cfg(unix)] +impl Drop for SocketPublicationLocalGuard { + fn drop(&mut self) { + let (held, changed) = SOCKET_PUBLICATION_LOCAL_STATE.get_or_init(Default::default); + held.lock().unwrap_or_else(std::sync::PoisonError::into_inner).remove(&self.identity); + changed.notify_all(); + } +} + +/// Exclusive authority to mutate one local server socket publication. +/// +/// The lock is released by closing its file descriptor. On Unix, callers can +/// deliberately inherit that descriptor into a helper process so publication +/// ownership survives a parent-process exit without an unlocked interval. +#[must_use = "dropping the guard releases socket publication ownership"] +pub struct SocketPublicationGuard { + // Rust drops fields in declaration order. Close this process's file-lock + // reference before reopening local admission for another thread. file: std::fs::File, + #[cfg(unix)] + _local: SocketPublicationLocalGuard, } -impl SocketPublicationLock { - fn acquire(path: &Path, deadline: Instant) -> std::io::Result { +impl SocketPublicationGuard { + pub fn acquire(path: &Path, deadline: Instant) -> std::io::Result { let lock_path = socket_publication_lock_path(path)?; let mut options = std::fs::OpenOptions::new(); options.create(true).truncate(false).read(true).write(true); @@ -3323,7 +3390,7 @@ impl SocketPublicationLock { } let file = options.open(&lock_path)?; #[cfg(unix)] - { + let local = { use std::os::unix::fs::{MetadataExt, PermissionsExt}; let metadata = file.metadata()?; @@ -3343,12 +3410,23 @@ impl SocketPublicationLock { )); } file.set_permissions(std::fs::Permissions::from_mode(0o600))?; - } + SocketPublicationLocalGuard::acquire( + SocketPublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, + path, + deadline, + )? + }; #[cfg(not(unix))] platform::restrict_file(&lock_path)?; loop { match FileExt::try_lock(&file) { - Ok(()) => return Ok(Self { file }), + Ok(()) => { + return Ok(Self { + file, + #[cfg(unix)] + _local: local, + }); + } Err(fs4::TryLockError::WouldBlock) if Instant::now() < deadline => { std::thread::sleep(Duration::from_millis(10)); } @@ -3362,11 +3440,100 @@ impl SocketPublicationLock { } } } -} -impl Drop for SocketPublicationLock { - fn drop(&mut self) { - let _ = FileExt::unlock(&self.file); + /// Arrange for this guard's lock descriptor to survive `exec` in `command`. + /// + /// The returned descriptor number must be passed to the child, which + /// should validate and adopt it with [`Self::adopt_inherited`]. The parent + /// descriptor remains close-on-exec and continues to own the same lock. + #[cfg(unix)] + pub fn inherit_into(&self, command: &mut std::process::Command) -> std::os::fd::RawFd { + use std::os::fd::AsRawFd as _; + use std::os::unix::process::CommandExt as _; + + let descriptor = self.file.as_raw_fd(); + // SAFETY: fcntl is async-signal-safe, and this only clears + // close-on-exec on the guard-owned descriptor in the child between + // fork and exec. The parent descriptor remains close-on-exec. + unsafe { + command.pre_exec(move || { + let flags = libc::fcntl(descriptor, libc::F_GETFD); + if flags == -1 { + return Err(std::io::Error::last_os_error()); + } + if libc::fcntl(descriptor, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + descriptor + } + + /// Validate and adopt a publication descriptor inherited across `exec`. + /// + /// # Safety + /// + /// `descriptor` must be a valid, uniquely owned child-process descriptor + /// produced by [`Self::inherit_into`]. The caller must not use or close it + /// after this function takes ownership. + #[cfg(unix)] + pub unsafe fn adopt_inherited( + path: &Path, + descriptor: std::os::fd::RawFd, + deadline: Instant, + ) -> std::io::Result { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; + use std::os::unix::fs::MetadataExt as _; + + if descriptor <= libc::STDERR_FILENO { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "server socket publication descriptor is invalid", + )); + } + // SAFETY: F_GETFD only validates the caller-owned descriptor. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFD) }; + if flags == -1 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: the function contract transfers unique ownership of this + // inherited child descriptor into the returned guard. + let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; + // Do not leak publication ownership into any subprocess the helper + // might launch after adoption. + if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, flags | libc::FD_CLOEXEC) } == -1 { + return Err(std::io::Error::last_os_error()); + } + let metadata = file.metadata()?; + let lock_path = socket_publication_lock_path(path)?; + let published_metadata = std::fs::symlink_metadata(&lock_path)?; + if !metadata.file_type().is_file() + || metadata.nlink() != 1 + || metadata.dev() != published_metadata.dev() + || metadata.ino() != published_metadata.ino() + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "inherited server socket publication descriptor has the wrong identity", + )); + } + let local = SocketPublicationLocalGuard::acquire( + SocketPublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, + path, + deadline, + )?; + // An inherited duplicate shares the already-locked open file + // description. A separately opened descriptor conflicts while the + // legitimate owner is alive, so this also proves lock ownership. + match FileExt::try_lock(&file) { + Ok(()) => Ok(Self { file, _local: local }), + Err(fs4::TryLockError::WouldBlock) => Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "inherited server socket publication descriptor does not own the lock", + )), + Err(fs4::TryLockError::Error(error)) => Err(error), + } } } @@ -3452,7 +3619,7 @@ impl PublishedSocket { #[cfg(unix)] fn cleanup_unix(&self) -> std::io::Result<()> { - let _publication_lock = SocketPublicationLock::acquire( + let _publication_lock = SocketPublicationGuard::acquire( &self.path, Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT + LOCAL_SOCKET_CONNECT_TIMEOUT, )?; @@ -3549,7 +3716,7 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result Result<(), String> { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if marker.exists() { + return Ok(()); + } + if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { + return Err(format!( + "publication child exited before {}: {status}", + marker.display() + )); + } + if Instant::now() >= deadline { + return Err(format!("publication child did not create {}", marker.display())); + } + std::thread::sleep(Duration::from_millis(10)); + } + } + + fn wait_for_child_exit( + child: &mut std::process::Child, + ) -> Result { + let deadline = Instant::now() + Duration::from_secs(5); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + return Err("publication child did not exit".to_string()); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error.to_string()); + } + } + } + } + + if let Some(directory) = std::env::var_os(CHILD_ENV) { + let directory = PathBuf::from(directory); + let path = directory.join("server.sock"); + let descriptor = + std::env::var(DESCRIPTOR_ENV).unwrap().parse::().unwrap(); + // SAFETY: the parent passed the unique descriptor copy prepared + // for this exec child and retains no child-side alias. + let guard = unsafe { + SocketPublicationGuard::adopt_inherited( + &path, + descriptor, + Instant::now() + Duration::from_secs(2), + ) + } + .unwrap(); + std::fs::write(directory.join("ready"), []).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !directory.join("release").exists() { + assert!(Instant::now() < deadline, "parent did not release publication child"); + std::thread::sleep(Duration::from_millis(10)); + } + drop(guard); + std::fs::write(directory.join("dropped"), []).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !directory.join("finish").exists() { + assert!(Instant::now() < deadline, "parent did not finish publication child"); + std::thread::sleep(Duration::from_millis(10)); + } + return; + } + + let sequence = SOCKET_CLEANUP_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let directory = std::env::temp_dir() + .join(format!("cmux-publication-transfer-{}-{sequence:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("server.sock"); + let guard = SocketPublicationGuard::acquire(&path, Instant::now() + Duration::from_secs(1)) + .unwrap(); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + let descriptor = guard.inherit_into(&mut command); + command + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, &directory) + .env(DESCRIPTOR_ENV, descriptor.to_string()) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + let mut child = cmux_tui_process::spawn(&mut command).unwrap(); + + let ready = wait_for_marker(&mut child, &directory.join("ready")); + drop(guard); + let competing = + SocketPublicationGuard::acquire(&path, Instant::now() + Duration::from_millis(150)); + let child_retained_lock = + matches!(&competing, Err(error) if error.kind() == std::io::ErrorKind::TimedOut); + drop(competing); + + std::fs::write(directory.join("release"), []).unwrap(); + let dropped = wait_for_marker(&mut child, &directory.join("dropped")); + let reacquired = + SocketPublicationGuard::acquire(&path, Instant::now() + Duration::from_secs(1)); + let child_released_lock = reacquired.is_ok(); + drop(reacquired); + std::fs::write(directory.join("finish"), []).unwrap(); + let status = wait_for_child_exit(&mut child); + + for marker in ["ready", "release", "dropped", "finish"] { + let _ = std::fs::remove_file(directory.join(marker)); + } + let _ = std::fs::remove_file(socket_publication_lock_path(&path).unwrap()); + std::fs::remove_dir(&directory).unwrap(); + + ready.unwrap(); + dropped.unwrap(); + let status = status.unwrap(); + assert!(status.success(), "publication child failed: {status}"); + assert!(child_retained_lock, "parent release dropped the child's publication lock"); + assert!(child_released_lock, "child guard retained publication ownership after drop"); + } + #[cfg(unix)] #[test] fn publication_lock_rejects_symlink_without_changing_target_permissions() { @@ -9535,7 +9833,7 @@ mod tests { symlink(&target, &lock_path).unwrap(); let result = - SocketPublicationLock::acquire(&socket_path, Instant::now() + Duration::from_secs(1)); + SocketPublicationGuard::acquire(&socket_path, Instant::now() + Duration::from_secs(1)); let target_mode = std::fs::metadata(&target).unwrap().permissions().mode() & 0o777; let rejected = result.is_err(); drop(result); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index de756a89a3b1..39db36333640 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -15,8 +15,8 @@ use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; use cmux_tui_core::release::{LAUNCHER_COMMAND_ENV, ReleaseIdentity}; use cmux_tui_core::server::{ - PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, SERVER_SHUTDOWN_INCOMPLETE_ERROR, - SERVER_SHUTDOWN_TIMEOUT, + LOCAL_SOCKET_CONNECT_TIMEOUT, PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, + SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, SocketPublicationGuard, }; use serde_json::{Value, json}; @@ -118,6 +118,7 @@ struct LegacySocketQuarantine { device: u64, inode: u64, armed: bool, + publication_guard: SocketPublicationGuard, } #[cfg(unix)] @@ -132,6 +133,10 @@ impl LegacySocketQuarantine { ) -> std::io::Result { use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; + let publication_guard = SocketPublicationGuard::acquire( + original, + Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT + LOCAL_SOCKET_CONNECT_TIMEOUT, + )?; let metadata = std::fs::symlink_metadata(original)?; if !metadata.file_type().is_socket() { return Err(std::io::Error::new( @@ -196,6 +201,7 @@ impl LegacySocketQuarantine { device: metadata.dev(), inode: metadata.ino(), armed: true, + publication_guard, }) } @@ -204,8 +210,10 @@ impl LegacySocketQuarantine { quarantined: PathBuf, device: u64, inode: u64, + publication_guard: SocketPublicationGuard, ) -> std::io::Result { - let quarantine = Self { original, quarantined, device, inode, armed: true }; + let quarantine = + Self { original, quarantined, device, inode, armed: true, publication_guard }; if quarantine.matches(&quarantine.quarantined)? != Some(true) { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -1061,6 +1069,7 @@ fn run_detached_legacy_stop( anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed) })?; let mut command = Command::new(executable); + let publication_descriptor = quarantine.publication_guard.inherit_into(&mut command); command .arg("__legacy-stop-helper") .arg(&quarantine.original) @@ -1069,6 +1078,7 @@ fn run_detached_legacy_stop( .arg(expected.started_at().to_string()) .arg(quarantine.device.to_string()) .arg(quarantine.inode.to_string()) + .arg(publication_descriptor.to_string()) .arg(LEGACY_SHUTDOWN_TIMEOUT.as_millis().to_string()) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -1175,6 +1185,7 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { expected_started_at, device, inode, + publication_descriptor, timeout_ms, ] = args else { @@ -1193,6 +1204,13 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { let inode = inode .parse::() .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported))?; + let publication_descriptor = publication_descriptor + .parse::() + .ok() + .filter(|descriptor| *descriptor > libc::STDERR_FILENO) + .ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.shutdown_unsupported) + })?; let timeout = timeout_ms .parse::() .ok() @@ -1203,11 +1221,22 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { })?; let deadline = Instant::now() + timeout; ensure_legacy_helper_active()?; + // SAFETY: this private exec mode receives the unique child copy created by + // `SocketPublicationGuard::inherit_into` and never aliases that raw fd. + let publication_guard = unsafe { + SocketPublicationGuard::adopt_inherited( + Path::new(original_path), + publication_descriptor, + deadline, + ) + } + .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed))?; let mut quarantine = LegacySocketQuarantine::adopt( PathBuf::from(original_path), PathBuf::from(quarantined_path), device, inode, + publication_guard, ) .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.legacy_cleanup_failed))?; let expected = ProcessIdentity::from_parts(expected_pid, expected_started_at); @@ -1752,6 +1781,15 @@ mod tests { (listener, quarantine, path) } + #[cfg(unix)] + fn remove_quarantined_test_path(path: &Path) { + let _ = std::fs::remove_file(path); + let mut lock_name = std::ffi::OsString::from("."); + lock_name.push(path.file_name().unwrap()); + lock_name.push(".publish.lock"); + let _ = std::fs::remove_file(path.with_file_name(lock_name)); + } + #[cfg(unix)] #[test] fn legacy_quarantine_blocks_replacement_publication() { @@ -2056,7 +2094,7 @@ mod tests { assert_eq!(reaped, -1, "timed-out helper lost its reaping owner"); assert_eq!(reaped_error.and_then(|error| error.raw_os_error()), Some(libc::ECHILD)); drop(listener); - let _ = std::fs::remove_file(path); + remove_quarantined_test_path(&path); } #[cfg(unix)] @@ -2082,7 +2120,7 @@ mod tests { drop(quarantine); drop(replacement); drop(original); - let _ = std::fs::remove_file(&path); + remove_quarantined_test_path(&path); assert!(rejected_replacement, "quarantine accepted a replacement socket"); assert!( @@ -2131,7 +2169,7 @@ mod tests { let restored_after_helper_exit = UnixStream::connect(&path).is_ok(); drop(listener); - let _ = std::fs::remove_file(&path); + remove_quarantined_test_path(&path); assert_eq!(error.to_string(), crate::localization::catalog().server.legacy_cleanup_failed); assert!( @@ -2180,7 +2218,7 @@ mod tests { "lost wait ownership did not release the admission fence after helper exit" ); drop(listener); - let _ = std::fs::remove_file(path); + remove_quarantined_test_path(&path); } #[cfg(unix)] From 8b1e23e1fb574f3d4d8d02829e695d53f9fd4baa Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:34:53 -0700 Subject: [PATCH 294/521] test(tui): expose current review regressions --- cmux-tui/bindings/go/resource_api_test.go | 18 +++++++ cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 29 +++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 23 +++++++++ cmux-tui/crates/cmux-tui/src/main.rs | 51 +++++++++++++++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 12 +++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 5 ++ 6 files changed, 138 insertions(+) diff --git a/cmux-tui/bindings/go/resource_api_test.go b/cmux-tui/bindings/go/resource_api_test.go index 6871ac836fab..56da634f785d 100644 --- a/cmux-tui/bindings/go/resource_api_test.go +++ b/cmux-tui/bindings/go/resource_api_test.go @@ -3586,6 +3586,24 @@ func TestSlowConsumerQueueBoundsEndOnlyThatStream(t *testing.T) { } } +func TestRejectedStreamEndStillRecordsServerTermination(t *testing.T) { + route := &streamRoute{ + messages: make(chan streamMessage, 1), + accepting: true, + } + route.messages <- streamMessage{ + envelope: streamEnvelope{Type: "stream_item"}, + size: 1, + } + + if route.deliver(streamMessage{envelope: streamEnvelope{Type: "stream_end"}, size: 1}) { + t.Fatal("stream_end unexpectedly fit in the full delivery queue") + } + if !route.hasServerEnd() { + t.Fatal("observed stream_end was lost when its delivery queue was full") + } +} + func TestOversizedUnterminatedFrameIsBounded(t *testing.T) { clientSide, serverSide := net.Pipe() go func() { diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 5d45296efe87..0ef237eb00d2 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -522,6 +522,35 @@ mod tests { assert!(chrome.child.lock().unwrap().is_none()); } + #[cfg(unix)] + #[test] + fn confirmed_kill_releases_reaper_capacity_before_chrome_drop() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let (sender, _receiver) = mpsc::channel(); + let active = Arc::new(AtomicUsize::new(0)); + let reaper = reserve_reaper_lease(sender, active.clone(), 1).unwrap(); + let chrome = Chrome { + child: Mutex::new(Some(child)), + profile_dir: make_profile_dir().unwrap(), + profile_ephemeral: true, + web_socket_url: "ws://127.0.0.1/unused".to_string(), + reaper: Some(reaper), + }; + + assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); + assert_eq!( + active.load(Ordering::Acquire), + 0, + "a confirmed child retained bounded reaper capacity until Chrome was dropped" + ); + } + #[cfg(unix)] #[test] fn drop_transfers_an_unconfirmed_child_to_a_reaper() { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index e1d3fb4b6253..c64575bfc466 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -12926,6 +12926,29 @@ mod tests { assert_eq!(outbound.try_pop(), None); } + #[test] + fn confirmed_control_precedes_initial_stream_backlog() { + let outbound = Arc::new(BoundedOutbound::default()); + let stream = OutboundStream::new(1, r#"{"event":"overflow"}"#.to_string()); + outbound.push_initial("initial".to_string(), &stream).unwrap(); + let confirmed = { + let outbound = outbound.clone(); + std::thread::spawn(move || { + outbound.push_control_confirmed("ack".to_string(), Duration::from_secs(1)) + }) + }; + let deadline = Instant::now() + Duration::from_secs(1); + while outbound.state.lock().unwrap().control_bytes == 0 && Instant::now() < deadline { + std::thread::yield_now(); + } + + let first = outbound.try_pop().unwrap(); + let second = outbound.try_pop().unwrap(); + confirmed.join().unwrap().unwrap(); + assert_eq!(first, "ack", "confirmed control waited behind initial stream state"); + assert_eq!(second, "initial"); + } + #[test] fn terminal_overflow_purges_only_its_stream_and_rejects_late_frames() { let outbound = Arc::new(BoundedOutbound::default()); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 65a74b2da4a9..ea744978f545 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1809,6 +1809,57 @@ mod tests { let _ = std::fs::remove_dir_all(socket_dir); } + #[test] + fn panicked_shutdown_retains_server_until_explicit_retry() { + let mux = Mux::new("shutdown-panic-retry-test", SurfaceOptions::default()); + let server_process = ServerProcessShutdownGuard::start(&mux).unwrap(); + let suffix = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let socket_dir = + std::env::temp_dir().join(format!("csp-{}-{suffix:x}", std::process::id())); + let socket_path = socket_dir.join("server.sock"); + let published = cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path)).unwrap(); + server_process.publish_socket(published).unwrap(); + let attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + server_process.cleanup.set_shutdown_attempt_for_test({ + let attempts = attempts.clone(); + move || { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + panic!("forced first shutdown panic"); + } + Ok(()) + } + }); + let cleanup = server_process.cleanup.clone(); + let (returned, observed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let result = finish_server_process(server_process, Ok(())); + let _ = returned.send(result); + }); + + let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { + std::thread::yield_now(); + } + assert_eq!(attempts.load(Ordering::SeqCst), 1); + assert!( + observed.recv_timeout(std::time::Duration::from_millis(75)).is_err(), + "a panicked cleanup released the server's process owners" + ); + + mux.request_shutdown(); + let retried = observed.recv_timeout(std::time::Duration::from_secs(1)); + if retried.is_err() { + cleanup.abandon(); + let _ = observed.recv_timeout(std::time::Duration::from_secs(1)); + } + worker.join().unwrap(); + assert!(retried.is_ok(), "an explicit shutdown retry did not recover from a panic"); + retried.unwrap().unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + let _ = std::fs::remove_dir_all(socket_dir); + } + #[test] fn unpublished_shutdown_failure_returns_without_waiting_for_retry() { let mux = Mux::new("unpublished-shutdown-test", SurfaceOptions::default()); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 39db36333640..c6c2affe3475 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1876,6 +1876,18 @@ mod tests { assert!(message.contains("Stopping exits pane processes.")); } + #[cfg(unix)] + #[test] + fn shell_quote_preserves_non_utf8_socket_paths() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt as _; + + let quoted = shell_quote(Path::new(OsStr::from_bytes(b"/tmp/test-\xff.sock"))); + + assert_eq!(quoted, "$'/tmp/test-\\377.sock'"); + assert!(!quoted.contains('\u{fffd}')); + } + #[test] fn replayable_launcher_override_survives_one_shot_package_shims() { let argv0 = OsStr::new("/tmp/ephemeral/bin/cmux"); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 0df37b11b690..f094cc06ad06 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1452,6 +1452,11 @@ fn server_status_and_stop_control_a_compatible_headless_session() { serde_json::from_slice::(&stop.stdout).unwrap(), serde_json::json!({"stopped": true}) ); + assert!( + String::from_utf8_lossy(&stop.stderr).contains("Stopping exits pane processes."), + "direct server stop omitted its destructive-operation warning: {}", + String::from_utf8_lossy(&stop.stderr) + ); assert!(server.child.wait().unwrap().success()); assert!(!server.socket.exists()); } From cf5ac0dd598c2ad9867947bbb06d967380c60eaa Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:44:54 -0700 Subject: [PATCH 295/521] fix(tui): close current review gaps --- cmux-tui/bindings/go/client.go | 6 +-- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 52 +++++++------------ cmux-tui/crates/cmux-tui-core/src/platform.rs | 18 ++----- .../cmux-tui-core/src/process_session.rs | 20 +++++-- cmux-tui/crates/cmux-tui-core/src/server.rs | 28 +++++++--- cmux-tui/crates/cmux-tui/src/cli.rs | 29 ++++++----- cmux-tui/crates/cmux-tui/src/main.rs | 36 +++---------- .../crates/cmux-tui/src/server_lifecycle.rs | 20 +++++++ .../src/server_lifecycle/legacy_process.rs | 24 --------- .../cmux-tui/tests/terminal_host_recovery.rs | 29 ++++++++--- cmux-tui/docs/getting-started.md | 4 +- cmux-tui/spec/commands.md | 6 ++- tests/test_tui_package_launchers.py | 1 + 13 files changed, 136 insertions(+), 137 deletions(-) diff --git a/cmux-tui/bindings/go/client.go b/cmux-tui/bindings/go/client.go index 63441e147c84..1a8c8b582a08 100644 --- a/cmux-tui/bindings/go/client.go +++ b/cmux-tui/bindings/go/client.go @@ -1383,6 +1383,9 @@ func (r *streamRoute) deliver(message streamMessage) bool { if r.terminated { return false } + if message.envelope.Type == "stream_end" { + r.serverEnded = true + } if r.cancelItem != nil { if r.cancelEnd != nil { var err error @@ -1432,9 +1435,6 @@ func (r *streamRoute) deliver(message streamMessage) bool { } select { case r.messages <- message: - if message.envelope.Type == "stream_end" { - r.serverEnded = true - } r.queuedBytes += message.size return true default: diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 0ef237eb00d2..8ba941eae7d2 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -29,7 +29,6 @@ static REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); thread_local! { static FORCE_KILL_TIMEOUT: std::cell::Cell = const { std::cell::Cell::new(false) }; static FORCE_REAPER_SPAWN_FAILURE: std::cell::Cell = const { std::cell::Cell::new(false) }; - static REAP_CHILD_CALLED_ON_THREAD: std::cell::Cell = const { std::cell::Cell::new(false) }; } #[derive(Debug, Clone)] @@ -53,7 +52,7 @@ pub struct Chrome { profile_dir: PathBuf, profile_ephemeral: bool, web_socket_url: String, - reaper: Option, + reaper: Mutex>, } impl Chrome { @@ -149,7 +148,7 @@ impl Chrome { profile_dir, profile_ephemeral, web_socket_url, - reaper: Some(reaper), + reaper: Mutex::new(Some(reaper)), }) } @@ -166,6 +165,7 @@ impl Chrome { let Some(mut child) = slot.take() else { return true }; drop(slot); if kill_child_until(&mut child, deadline) { + drop(self.reaper.lock().unwrap().take()); return true; } *self.child.lock().unwrap() = Some(child); @@ -184,7 +184,11 @@ impl Drop for Chrome { let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); if let Some(child) = child { reap_child_detached( - self.reaper.take().expect("live Chrome retains its reaper lease"), + self.reaper + .get_mut() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + .expect("live Chrome retains its reaper lease"), child, self.profile_ephemeral.then(|| self.profile_dir.clone()), ); @@ -321,8 +325,6 @@ fn run_reaper(receiver: mpsc::Receiver) { } fn poll_reap_request(request: &mut ReapRequest) -> bool { - #[cfg(test)] - REAP_CHILD_CALLED_ON_THREAD.set(true); #[cfg(test)] { REAPER_POLL_ATTEMPTS.fetch_add(1, Ordering::Relaxed); @@ -515,7 +517,7 @@ mod tests { profile_dir, profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Some(chrome_reaper_lease().unwrap()), + reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); @@ -540,7 +542,7 @@ mod tests { profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Some(reaper), + reaper: Mutex::new(Some(reaper)), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); @@ -573,7 +575,7 @@ mod tests { profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Some(chrome_reaper_lease().unwrap()), + reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), }; FORCE_KILL_TIMEOUT.set(true); @@ -603,34 +605,16 @@ mod tests { #[test] fn reaper_spawn_failure_never_waits_on_the_caller_thread() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); - let child = Command::new("sleep") - .arg("60") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - let reaper = chrome_reaper_lease().unwrap(); - let chrome = Chrome { - child: Mutex::new(Some(child)), - profile_dir: make_profile_dir().unwrap(), - profile_ephemeral: true, - web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Some(reaper), - }; - - REAP_CHILD_CALLED_ON_THREAD.set(false); - FORCE_KILL_TIMEOUT.set(true); FORCE_REAPER_SPAWN_FAILURE.set(true); - drop(chrome); + let started = Instant::now(); + let result = ChromeReaper::start(); FORCE_REAPER_SPAWN_FAILURE.set(false); - FORCE_KILL_TIMEOUT.set(false); - let reaped_inline = REAP_CHILD_CALLED_ON_THREAD.get(); - let cleanup = Command::new("true").spawn().unwrap(); - reap_child_detached(chrome_reaper_lease().unwrap(), cleanup, None); - - assert!(!reaped_inline, "reaper spawn failure fell back to an unbounded caller wait"); + assert!(result.is_err(), "the forced reaper spawn failure was not exercised"); + assert!( + started.elapsed() < Duration::from_millis(250), + "reaper spawn failure waited on caller-thread cleanup" + ); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index f6c931f28f8a..18a4441755ad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -108,13 +108,13 @@ pub mod transport { Ok(descriptor) } - #[cfg(target_os = "macos")] + #[cfg(not(target_os = "linux"))] fn create_close_on_exec_socket(deadline: Instant) -> io::Result { let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire_until(deadline)?; - // macOS has no SOCK_CLOEXEC socket flag. Every cmux-tui process - // launch shares this barrier, so no child can start until fcntl - // marks the fresh descriptor close-on-exec. + // Unix platforms without the Linux atomic socket flag share this + // barrier with every cmux-tui process launch, so no child can start + // until fcntl marks the fresh descriptor close-on-exec. // SAFETY: socket has no pointer arguments and returns a new owned // descriptor on success. let descriptor = unsafe { libc::socket(libc::AF_UNIX, libc::SOCK_STREAM, 0) }; @@ -150,14 +150,6 @@ pub mod transport { Ok(descriptor) } - #[cfg(not(any(target_os = "linux", target_os = "macos")))] - fn create_close_on_exec_socket(_deadline: Instant) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "deadline sockets require atomic close-on-exec setup", - )) - } - pub(super) fn listen(path: &Path) -> io::Result { cmux_tui_process::unix::bind_listener(path).map(|inner| Listener { inner }) } @@ -764,7 +756,7 @@ pub mod transport { ); } - #[cfg(target_os = "macos")] + #[cfg(not(target_os = "linux"))] #[test] fn deadline_connect_confines_inheritable_descriptor_to_process_barrier() { use std::os::fd::AsRawFd as _; diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index c541d082ea19..cc2750518644 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2006,9 +2006,23 @@ mod tests { } let mut leader = command.spawn().unwrap(); let session = libc::pid_t::try_from(leader.id()).unwrap(); - let mut descendant = String::new(); - io::BufReader::new(leader.stdout.take().unwrap()).read_line(&mut descendant).unwrap(); - let descendant = descendant.trim().parse::().unwrap(); + let stdout = leader.stdout.take().unwrap(); + let (descendant_sender, descendant_receiver) = mpsc::sync_channel(1); + let descendant_reader = std::thread::spawn(move || { + let mut line = String::new(); + let result = io::BufReader::new(stdout).read_line(&mut line).map(|_| line); + let _ = descendant_sender.send(result); + }); + let descendant = match descendant_receiver.recv_timeout(Duration::from_secs(1)) { + Ok(Ok(line)) => line.trim().parse::().unwrap(), + result => { + let _ = leader.kill(); + let _ = leader.wait(); + let _ = descendant_reader.join(); + panic!("session descendant PID was not published within one second: {result:?}"); + } + }; + descendant_reader.join().unwrap(); let leader_handle = StableProcessHandle::capture(session).unwrap().unwrap(); assert!(leader_handle.signal(libc::SIGSTOP).unwrap()); let mut status = 0; diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index c64575bfc466..277070347c47 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -1974,6 +1974,7 @@ struct BoundedOutbound { #[derive(Default)] struct BoundedOutboundState { initial: VecDeque, + confirmed: VecDeque, control: VecDeque, regular: VecDeque, stream_usage: HashMap, @@ -2145,7 +2146,7 @@ impl BoundedOutbound { fn push_control(&self, text: String) -> std::io::Result<()> { let mut state = self.state.lock().unwrap(); - Self::push_control_locked(&mut state, text, None)?; + Self::push_control_locked(&mut state, text, None, false)?; self.changed.notify_one(); Ok(()) } @@ -2154,7 +2155,7 @@ impl BoundedOutbound { let (completion, written) = std::sync::mpsc::sync_channel(1); { let mut state = self.state.lock().unwrap(); - Self::push_control_locked(&mut state, text, Some(completion))?; + Self::push_control_locked(&mut state, text, Some(completion), true)?; self.changed.notify_one(); } match written.recv_timeout(timeout) { @@ -2175,7 +2176,7 @@ impl BoundedOutbound { if stream.terminal_enqueued.swap(true, Ordering::AcqRel) { return Ok(()); } - Self::push_control_locked(&mut state, text, None)?; + Self::push_control_locked(&mut state, text, None, false)?; self.changed.notify_one(); Ok(()) } @@ -2190,7 +2191,7 @@ impl BoundedOutbound { return Ok(()); } let overflow_text = stream.overflow_text.lock().unwrap().clone(); - if let Err(error) = Self::push_control_locked(state, overflow_text, None) { + if let Err(error) = Self::push_control_locked(state, overflow_text, None, false) { state.closed = true; return Err(std::io::Error::new( std::io::ErrorKind::BrokenPipe, @@ -2220,12 +2221,13 @@ impl BoundedOutbound { state: &mut BoundedOutboundState, text: String, completion: Option>, + confirmed: bool, ) -> std::io::Result<()> { if state.closed { return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, "connection closed")); } let bytes = text.len(); - if state.control.len() >= OUTBOUND_CONTROL_RESERVE + if state.confirmed.len() + state.control.len() >= OUTBOUND_CONTROL_RESERVE || bytes > OUTBOUND_CONTROL_BYTE_RESERVE.saturating_sub(state.control_bytes) { return Err(std::io::Error::new( @@ -2234,7 +2236,12 @@ impl BoundedOutbound { )); } state.control_bytes += bytes; - state.control.push_back(ControlOutbound { text, completion }); + let message = ControlOutbound { text, completion }; + if confirmed { + state.confirmed.push_back(message); + } else { + state.control.push_back(message); + } Ok(()) } @@ -2258,6 +2265,10 @@ impl BoundedOutbound { } fn pop_locked(state: &mut BoundedOutboundState) -> Option { + if let Some(message) = state.confirmed.pop_front() { + state.control_bytes -= message.text.len(); + return Some(OutboundMessage { text: message.text, completion: message.completion }); + } if let Some(message) = state.initial.pop_front() { Self::record_stream_pop(state, &message); return Some(OutboundMessage { text: message.text, completion: None }); @@ -2291,6 +2302,11 @@ impl BoundedOutbound { fn close(&self) { let mut state = self.state.lock().unwrap(); state.closed = true; + for message in &mut state.confirmed { + if let Some(completion) = message.completion.take() { + let _ = completion.send(false); + } + } for message in &mut state.control { if let Some(completion) = message.completion.take() { let _ = completion.send(false); diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index d09080e06488..092c26ee2c3c 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -502,22 +502,25 @@ fn run_server_lifecycle(global: &GlobalArgs, actions: &[String]) -> i32 { }; match action { "status" => print_server_status(global.output, lifecycle.probe()), - "stop" => match lifecycle.stop() { - Ok(()) => { - match global.output { - OutputMode::Json | OutputMode::JsonLines => { - println!("{}", json!({"stopped": true})); + "stop" => { + eprintln!("cmux: {}", messages.stopping_exits_panes); + match lifecycle.stop() { + Ok(()) => { + match global.output { + OutputMode::Json | OutputMode::JsonLines => { + println!("{}", json!({"stopped": true})); + } + OutputMode::Human => println!("{}", messages.stopped), + OutputMode::Quiet => {} } - OutputMode::Human => println!("{}", messages.stopped), - OutputMode::Quiet => {} + 0 + } + Err(error) => { + eprintln!("{error}"); + 1 } - 0 - } - Err(error) => { - eprintln!("{error}"); - 1 } - }, + } _ => unreachable!("server action was validated before connecting"), } } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index ea744978f545..f173562dcdf4 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -904,7 +904,6 @@ enum ServerShutdownCleanupState { Running(u64), Complete, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, - Poisoned(String), Abandoned, } @@ -912,7 +911,6 @@ enum ServerShutdownCleanupState { enum ServerShutdownCleanupOutcome { Complete, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, - Poisoned(String), Abandoned, } @@ -971,9 +969,6 @@ impl ServerShutdownCleanup { retry: retry.clone(), }; } - ServerShutdownCleanupState::Poisoned(message) => { - return ServerShutdownCleanupOutcome::Poisoned(message.clone()); - } ServerShutdownCleanupState::Abandoned => { return ServerShutdownCleanupOutcome::Abandoned; } @@ -1006,9 +1001,11 @@ impl ServerShutdownCleanup { ), retry, }, - Err(_) => ServerShutdownCleanupOutcome::Poisoned( - localization::catalog().server.shutdown_cleanup_incomplete.to_string(), - ), + Err(_) => ServerShutdownCleanupOutcome::Degraded { + attempt, + message: localization::catalog().server.shutdown_cleanup_incomplete.to_string(), + retry, + }, }; let next_state = match &outcome { ServerShutdownCleanupOutcome::Complete => ServerShutdownCleanupState::Complete, @@ -1019,9 +1016,6 @@ impl ServerShutdownCleanup { retry: retry.clone(), } } - ServerShutdownCleanupOutcome::Poisoned(message) => { - ServerShutdownCleanupState::Poisoned(message.clone()) - } ServerShutdownCleanupOutcome::Abandoned => ServerShutdownCleanupState::Abandoned, }; state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1036,8 +1030,7 @@ impl ServerShutdownCleanup { fn run_until_complete(&self) -> anyhow::Result<()> { match self.run_once() { ServerShutdownCleanupOutcome::Complete => Ok(()), - ServerShutdownCleanupOutcome::Degraded { message, .. } - | ServerShutdownCleanupOutcome::Poisoned(message) => Err(anyhow::anyhow!(message)), + ServerShutdownCleanupOutcome::Degraded { message, .. } => Err(anyhow::anyhow!(message)), ServerShutdownCleanupOutcome::Abandoned => { Err(anyhow::anyhow!("server shutdown cleanup was abandoned")) } @@ -1069,7 +1062,6 @@ impl ServerShutdownCleanup { fn wait_for_complete(&self) { let mut reported_attempt = None; - let mut reported_poison = false; loop { match self.run_once() { ServerShutdownCleanupOutcome::Complete => return, @@ -1097,22 +1089,6 @@ impl ServerShutdownCleanup { .unwrap_or_else(std::sync::PoisonError::into_inner); } } - ServerShutdownCleanupOutcome::Poisoned(message) => { - if !reported_poison { - eprintln!( - "cmux-tui: {message}; retaining the server because cleanup panicked" - ); - reported_poison = true; - } - let state = - self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let _state = self - .changed - .wait_while(state, |state| { - matches!(state, ServerShutdownCleanupState::Poisoned(_)) - }) - .unwrap_or_else(std::sync::PoisonError::into_inner); - } ServerShutdownCleanupOutcome::Abandoned => return, } } diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index c6c2affe3475..120f2ee06b18 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1686,6 +1686,26 @@ fn wait_for_disconnect_until( } fn shell_quote(path: &Path) -> String { + #[cfg(unix)] + { + use std::fmt::Write as _; + use std::os::unix::ffi::OsStrExt as _; + + let bytes = path.as_os_str().as_bytes(); + if std::str::from_utf8(bytes).is_err() { + let mut quoted = String::from("$'"); + for byte in bytes.iter().copied() { + match byte { + b'\'' => quoted.push_str("\\'"), + b'\\' => quoted.push_str("\\\\"), + b' '..=b'~' => quoted.push(char::from(byte)), + _ => write!(quoted, "\\{byte:03o}").expect("writing to String cannot fail"), + } + } + quoted.push('\''); + return quoted; + } + } let value = path.display().to_string(); if !value.is_empty() && value.chars().all(|character| { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index f699ec3dd090..83b3365e14ff 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -11,7 +11,6 @@ const PROCESS_TREE_RETRY_TIMEOUT: Duration = Duration::from_secs(1); #[cfg(test)] thread_local! { - static RAW_PID_SIGNAL_COUNT: std::cell::Cell = const { std::cell::Cell::new(0) }; static STABLE_HANDLE_CAPTURE_BUDGET: std::cell::Cell> = const { std::cell::Cell::new(None) }; } @@ -942,29 +941,6 @@ mod tests { child.wait().unwrap(); } - #[test] - fn process_identity_never_signals_through_a_reusable_pid() { - let mut child = Command::new("sleep") - .arg("60") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - let process = - ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); - RAW_PID_SIGNAL_COUNT.set(0); - - assert_eq!(process.signal(libc::SIGCONT).unwrap(), ExactSignalResult::Signaled); - let raw_signals = RAW_PID_SIGNAL_COUNT.get(); - - child.kill().unwrap(); - child.wait().unwrap(); - assert_eq!( - raw_signals, 0, - "legacy cleanup used a reusable PID instead of a stable process identity" - ); - } - #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn frozen_tree_drop_resumes_with_its_original_process_proof() { diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 8f7b234b9ca3..944b8de1f5b1 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -2836,18 +2836,31 @@ fn wait_for_screen(path: &Path, surface: u64, marker: &str) -> String { fn wait_for_stream_disconnect(stream: &mut UnixStream, timeout: Duration) -> bool { let deadline = Instant::now() + timeout; - loop { + let previous_timeout = stream.write_timeout().ok().flatten(); + let disconnected = loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break false; + } + if stream.set_write_timeout(Some(remaining.min(Duration::from_millis(50)))).is_err() { + break true; + } // ReleaseViewer is valid and idempotent for this renderer but produces // no host-to-client frame, so it probes only the input half of the // connection while output remains deliberately unread. - if write_frame(stream, &Frame::new(MessageKind::ReleaseViewer, Vec::new())).is_err() { - return true; - } - if Instant::now() >= deadline { - return false; + match write_frame(stream, &Frame::new(MessageKind::ReleaseViewer, Vec::new())) { + Ok(()) => {} + Err(ProtocolError::Io(error)) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) => {} + Err(_) => break true, } - std::thread::sleep(Duration::from_millis(10)); - } + std::thread::sleep(remaining.min(Duration::from_millis(10))); + }; + let _ = stream.set_write_timeout(previous_timeout); + disconnected } fn wait_for_adopted_terminal( diff --git a/cmux-tui/docs/getting-started.md b/cmux-tui/docs/getting-started.md index fd6bbd6a7aac..917221ef9ee8 100644 --- a/cmux-tui/docs/getting-started.md +++ b/cmux-tui/docs/getting-started.md @@ -46,8 +46,8 @@ Detach from an attached TUI with prefix `d`. With default keys, that is `Ctrl-b Local clients require the running server to have the same release identity. Inspect a session without attaching, or stop it before starting an upgraded binary: ```bash -cmux-tui server status --session agents -cmux-tui server stop --session agents +cargo run -p cmux-tui -- server status --session agents +cargo run -p cmux-tui -- server stop --session agents ``` Stopping the server exits its pane processes. `identify`, `server status`, and `server stop` remain available when client and server releases differ. diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index b835116ff7d3..dbcba58f67b8 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -280,7 +280,11 @@ object{ running:true, compatible:boolean, mismatch_reasons:array<"distribution-version"|"source-build"|"terminal-engine"|"protocol">, - server:object{version:string,protocol:uint32}, + server:object{ + version:string, + protocol:uint32, + shutdown_cleanup:object{pending:uint64,retrying:boolean,degraded:boolean} + }, client:object{version:string,protocol:uint32} } ``` diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index 60b62eeba951..c6892f1978f6 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -205,6 +205,7 @@ def main() -> None: import tempfile test_pypi_fixture_targets_match_packager() + test_pypi_fixture_wheel_tags_match_packager() with tempfile.TemporaryDirectory() as directory: test_identity_verifier_executes_artifact_and_rejects_mismatch( Path(directory) From 95f2f9075111a7ed035c31b6cf2c960ad06bc8ab Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:03:02 -0700 Subject: [PATCH 296/521] docs(tui): explain legacy stop restoration --- .../crates/cmux-tui/src/server_lifecycle/legacy_process.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 83b3365e14ff..d7e676b37d85 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -128,6 +128,9 @@ impl ProcessFence { return Err(io::Error::new(io::ErrorKind::InvalidData, "process identity changed")); } let Some(process) = identity.stable_handle()? else { return Ok(None) }; + // POSIX coalesces repeated stop signals and does not expose their + // sender. Restore only a process observed running before this fence, + // so an aborted cleanup cannot strand a process that we stopped. let armed = !initial.stopped; if armed && !process.signal(libc::SIGSTOP)? { return Ok(None); From 911b4dfbcd66bb3e56a00b6ffbbd6b4f364a7e64 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:11:20 -0700 Subject: [PATCH 297/521] test(tui): fix merged ownership lint --- cmux-tui/crates/cmux-tui/tests/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index a51495136017..8f4dba85e526 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -944,7 +944,7 @@ fn newer_workspace_schema_failure_reports_socket_specific_recovery() { { let listener = UnixListener::bind(&socket).unwrap(); let expected_session = session.to_string(); - let expected_registry_id = registry_id; + let expected_registry_id = registry_id.clone(); let responder = std::thread::spawn(move || { let mut stream = accept_with_deadline(&listener); let mut request = String::new(); @@ -989,7 +989,7 @@ fn newer_workspace_schema_failure_reports_socket_specific_recovery() { fs::remove_file(&socket).unwrap(); let listener = UnixListener::bind(&socket).unwrap(); let expected_session = session.to_string(); - let expected_registry_id = registry_id.clone(); + let expected_registry_id = registry_id; let responder = std::thread::spawn(move || { let mut stream = accept_with_deadline(&listener); let mut request = String::new(); From bf43775a6fc75fa2ba100f8652e378f05e70d056 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:49:09 -0700 Subject: [PATCH 298/521] test(terminal): require awaitable close teardown --- ...SurfaceTeardownCallbackLifetimeTests.swift | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift index a6bac1c55886..95eafb2a3ac4 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift @@ -51,6 +51,25 @@ import Testing #expect(recorder.events == [.nativeFree, .teeLeaseRelease]) } + @Test func teardownSurfaceReturnsTicketThatFencesNativeFree() async throws { + let surface = makeSurface() + surface.installRuntimeSurfaceForTesting(fakeRuntimeSurface()) + let allowFree = DispatchSemaphore(value: 0) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in + allowFree.wait() + } + defer { + allowFree.signal() + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil + } + + let ticket = try #require(surface.teardownSurface()) + + #expect(await ticket.wait(timeout: .zero) == false) + allowFree.signal() + #expect(await ticket.wait(timeout: .seconds(1))) + } + @Test func agentHibernationSuspendKeepsTeeLeaseUntilNativeFree() async { let recorder = TeardownOrderRecorder() let registry = TerminalSurfaceRegistry() From 4067e3b40ea270c84a2e207f4d0ca7141df37f9d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:56:30 -0700 Subject: [PATCH 299/521] fix(terminal): fence explicit native teardown --- .../TerminalSurface+RuntimeLifecycle.swift | 46 ++++++++--- .../Surface/TerminalSurface.swift | 1 + .../RemoteTmuxMirrorCloseDetachTests.swift | 79 ++++++++++++++++++- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 7e66bed32266..921c7278bf62 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -229,8 +229,12 @@ extension TerminalSurface { /// Explicitly free the Ghostty runtime surface. Idempotent — safe to call /// before deinit; deinit will skip the free if already torn down. + /// + /// - Returns: A ticket that completes after the native free and callback + /// userdata releases, or the existing ticket when teardown already began. + @discardableResult @MainActor - public func teardownSurface() { + public func teardownSurface() -> TerminalSurfaceRuntimeTeardownTicket? { recordTeardownRequest(reason: "surface.teardown") markPortalLifecycleClosed(reason: "teardown") backgroundSurfaceStartSource = .normal @@ -255,7 +259,7 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() - return + return closeRuntimeTeardownTicket } #if DEBUG @@ -264,7 +268,7 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() - return + return closeRuntimeTeardownTicket } #endif @@ -273,7 +277,7 @@ extension TerminalSurface { // Transport manualIOContext and teeLease through the request too: // the coordinator releases all callback userdata only after the // native free, which is what joins ghostty's IO threads. - runtimeTeardown.enqueueRuntimeTeardown( + let ticket = runtimeTeardown.enqueueRuntimeTeardown( id: id, workspaceId: tabId, reason: "teardown", @@ -283,18 +287,36 @@ extension TerminalSurface { byteTeeLease: teeLease, freeSurface: freeSurface ) - return + closeRuntimeTeardownTicket = ticket + return ticket } #endif - Task { @MainActor in - // Keep free behavior aligned with deinit: perform the runtime teardown on - // the next main-actor turn so SIGHUP delivery is deterministic but non-reentrant. - ghostty_surface_free(surfaceToFree) - callbackContext?.release() - manualIOContext?.release() - teeLease?.release() + let ticket = runtimeTeardown.enqueueRuntimeTeardown( + id: id, + workspaceId: tabId, + reason: "teardown", + surface: surfaceToFree, + callbackContext: callbackContext, + manualIOContext: manualIOContext, + byteTeeLease: teeLease + ) + closeRuntimeTeardownTicket = ticket + return ticket + } + + /// Waits for an explicit close teardown to release its native surface. + /// + /// - Parameter timeout: Maximum wait, or `nil` for an event-driven wait. + /// - Returns: `true` when no close teardown is pending or it completed. + @MainActor + public func waitForCloseRuntimeTeardown(timeout: Duration?) async -> Bool { + guard let ticket = closeRuntimeTeardownTicket else { return true } + let completed = await ticket.wait(timeout: timeout) + if completed, closeRuntimeTeardownTicket?.id == ticket.id { + closeRuntimeTeardownTicket = nil } + return completed } /// Frees the runtime surface while keeping the model alive for an diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index b4807d1d8c8c..d3b3e03c2e3e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -276,6 +276,7 @@ public final class TerminalSurface: Identifiable, ObservableObject { (any TerminalSurfaceNativeViewing)? var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false + var closeRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? var agentHibernationRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? var agentHibernationRuntimeTeardownReservation: TerminalSurfaceRuntimeTeardownReservation? diff --git a/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift b/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift index f875fe047dcf..88fcb82823e0 100644 --- a/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift +++ b/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift @@ -33,7 +33,7 @@ import Testing /// The mark seam must NOT flag a mirror workspace's window for kill-on-close: /// the close detaches, the remote tmux session survives for resume. Before the /// fix this marked the window for kill; after, it never does. - @Test func markSeamDoesNotMarkMirrorForKill() throws { + @Test func markSeamDoesNotMarkMirrorForKill() async throws { let harness = try Harness() defer { harness.tearDown() } @@ -45,6 +45,7 @@ import Testing .windowsMarkedForKillOnClose() .contains(harness.windowId) ) + await harness.tearDownAndWait() } /// The v2 socket close path must detach a live last-workspace mirror without @@ -91,6 +92,7 @@ import Testing controller.cacheConnection(connection) #expect(try controller.mirrorSession(host: host, sessionName: "dev", into: harness.manager)) let mirrorWorkspace = try #require(harness.manager.tabs.first(where: { $0.isRemoteTmuxMirror })) + let teardownSurfaces = harness.captureTeardownSurfaces(in: [mirrorWorkspace]) let keepWorkspaceOpenKey = "closeWorkspaceOnLastSurfaceShortcut" let previousKeepWorkspaceOpen = UserDefaults.standard.object(forKey: keepWorkspaceOpenKey) UserDefaults.standard.set(false, forKey: keepWorkspaceOpenKey) @@ -130,6 +132,8 @@ import Testing #expect(!log.contains("kill-session"), Comment(rawValue: log)) #expect(controller.sessionMirror(host: host, sessionName: "dev") == nil) #expect(connection.exited) + await harness.waitForTeardown(of: teardownSurfaces) + await harness.tearDownAndWait() } /// Explicit detach of a mirror opened in its own window must close that @@ -171,6 +175,7 @@ import Testing controller.cacheConnection(connection) #expect(try controller.mirrorSession(host: host, sessionName: "dev", into: harness.manager)) let mirrorWorkspace = try #require(harness.manager.tabs.first(where: { $0.isRemoteTmuxMirror })) + let teardownSurfaces = harness.captureTeardownSurfaces(in: [mirrorWorkspace]) harness.manager.closeWorkspace(harness.workspace, recordHistory: false) #expect(harness.manager.tabs.map(\.id) == [mirrorWorkspace.id]) let owningWindow = try #require(harness.appDelegate.mainWindow(for: harness.windowId)) @@ -195,12 +200,14 @@ import Testing $0.windowId == harness.windowId }) #expect(harness.appDelegate.recoverableMainWindowRoute(windowId: harness.windowId) == nil) + await harness.waitForTeardown(of: teardownSurfaces) + await harness.tearDownAndWait() } /// A remote session ending removes its dead mirror but preserves the owning /// window with a fresh local workspace. Only explicit detach closes a /// dedicated final-mirror window. - @Test func remoteSessionEndOfDedicatedLastMirrorKeepsOwningWindowUsable() throws { + @Test func remoteSessionEndOfDedicatedLastMirrorKeepsOwningWindowUsable() async throws { let harness = try Harness() defer { harness.tearDown() } let host = RemoteTmuxHost(destination: "remote-end-\(UUID().uuidString)@example.test") @@ -209,6 +216,7 @@ import Testing controller.cacheConnection(connection) #expect(try controller.mirrorSession(host: host, sessionName: "dev", into: harness.manager)) let mirrorWorkspace = try #require(harness.manager.tabs.first(where: { $0.isRemoteTmuxMirror })) + let teardownSurfaces = harness.captureTeardownSurfaces(in: [mirrorWorkspace]) harness.manager.closeWorkspace(harness.workspace, recordHistory: false) let owningWindow = try #require(harness.appDelegate.mainWindow(for: harness.windowId)) @@ -221,6 +229,8 @@ import Testing #expect(harness.manager.tabs.count == 1) #expect(harness.manager.tabs.allSatisfy { !$0.isRemoteTmuxMirror }) #expect(!harness.manager.tabs.contains { $0.id == mirrorWorkspace.id }) + await harness.waitForTeardown(of: teardownSurfaces) + await harness.tearDownAndWait() } /// The ordinary non-last tab-close route shares the same detach contract as @@ -265,6 +275,7 @@ import Testing controller.cacheConnection(connection) #expect(try controller.mirrorSession(host: host, sessionName: "dev", into: harness.manager)) let mirrorWorkspace = try #require(harness.manager.tabs.first(where: { $0.isRemoteTmuxMirror })) + let teardownSurfaces = harness.captureTeardownSurfaces(in: [mirrorWorkspace]) #expect(harness.manager.tabs.count == 2) harness.manager.closeWorkspace(mirrorWorkspace, recordHistory: false) @@ -274,6 +285,8 @@ import Testing #expect(harness.manager.tabs.map(\.id) == [harness.workspace.id]) #expect(controller.sessionMirror(host: host, sessionName: "dev") == nil) #expect(connection.exited) + await harness.waitForTeardown(of: teardownSurfaces) + await harness.tearDownAndWait() } @Test func windowCreationFailureUsesLocalErrorMessage() { @@ -351,6 +364,13 @@ import Testing #expect(targetManager.tabs.filter(\.isRemoteTmuxMirror).count == 2) #expect(harness.manager.tabs.allSatisfy { !$0.isRemoteTmuxMirror }) #expect(secondManager.tabs.allSatisfy { !$0.isRemoteTmuxMirror }) + for windowID in extraWindowIDs.reversed() { + await harness.closeWindowAndWait(windowID) + } + extraWindowIDs.removeAll() + harness.controller.detach(host: host, sessionName: "one") + harness.controller.detach(host: host, sessionName: "two") + await harness.tearDownAndWait() } /// A direct socket caller must opt into focus. The CLI supplies an explicit @@ -429,6 +449,12 @@ import Testing #expect(focusedAfter["workspace_id"] as? String == workspaceIDBefore) #expect(focusedAfter["pane_id"] as? String == paneIDBefore) #expect(focusedAfter["surface_id"] as? String == surfaceIDBefore) + if let targetWindowID { + await harness.closeWindowAndWait(targetWindowID) + } + targetWindowID = nil + harness.controller.detach(host: host, sessionName: "one") + await harness.tearDownAndWait() } private func writeExecutable(at url: URL, contents: String) throws { @@ -467,6 +493,7 @@ import Testing let windowId: UUID let manager: TabManager let workspace: Workspace + let initialSurfaces: [TerminalSurface] var controller: RemoteTmuxController { appDelegate.remoteTmuxController } init() throws { @@ -474,6 +501,9 @@ import Testing windowId = appDelegate.createMainWindow() manager = try #require(appDelegate.tabManagerFor(windowId: windowId)) workspace = try #require(manager.selectedWorkspace) + initialSurfaces = workspace.panels.values.compactMap { + ($0 as? TerminalPanel)?.surface + } } func tearDown() { @@ -483,6 +513,13 @@ import Testing closeWindow(windowId) } + func tearDownAndWait() async { + workspace.isRemoteTmuxMirror = false + // Clear any marker so it can't leak into another serialized test. + controller.consumeKillSessionsOnWindowClose(windowId: windowId) + await closeWindowAndWait(windowId) + } + func cacheConnection(host: RemoteTmuxHost, session: String) { controller.cacheConnection(RemoteTmuxControlConnection(host: host, sessionName: session)) } @@ -499,5 +536,43 @@ import Testing appDelegate.forgetRecoverableMainWindowRoute(windowId: id) RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) } + + func closeWindowAndWait(_ id: UUID) async { + let surfaces = terminalSurfaces(in: id) + closeWindow(id) + await waitForTeardown(of: surfaces) + } + + func waitForTeardown(of workspaces: [Workspace]) async { + await waitForTeardown(of: captureTeardownSurfaces(in: workspaces)) + } + + private func terminalSurfaces(in id: UUID) -> [TerminalSurface] { + var workspaces = appDelegate.tabManagerFor(windowId: id)?.tabs ?? [] + if id == windowId, !workspaces.contains(where: { $0 === workspace }) { + workspaces.append(workspace) + } + return captureTeardownSurfaces(in: workspaces) + } + + func captureTeardownSurfaces(in workspaces: [Workspace]) -> [TerminalSurface] { + var seen: Set = [] + return (initialSurfaces + workspaces.flatMap { workspace in + workspace.panels.values.compactMap { panel -> TerminalSurface? in + (panel as? TerminalPanel)?.surface + } + }).filter { surface in + seen.insert(ObjectIdentifier(surface)).inserted + } + } + + func waitForTeardown(of surfaces: [TerminalSurface]) async { + for surface in surfaces { + #expect( + await surface.waitForCloseRuntimeTeardown(timeout: .seconds(10)), + "Timed out waiting for a terminal surface to finish native teardown" + ) + } + } } } From 0c9006e415ef3fcbef45b06f0703c26ba26ef4c9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:15:19 -0700 Subject: [PATCH 300/521] test(terminal): import teardown surface type --- cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift b/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift index 88fcb82823e0..7a8b3d434ef3 100644 --- a/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift +++ b/cmuxTests/RemoteTmuxMirrorCloseDetachTests.swift @@ -1,6 +1,7 @@ import AppKit import CmuxControlSocket import CmuxSettings +import CmuxTerminal import Testing #if canImport(cmux_DEV) From b50825362138098e1814cbd0c5f96331fb652d25 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:56:10 -0700 Subject: [PATCH 301/521] test(tui): preserve restartable server stop --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bad2d46150c7..fe33c59fe591 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -21785,6 +21785,43 @@ mod tests { assert!(mux.surface(surface).is_none()); } + #[cfg(unix)] + #[test] + fn server_stop_persists_a_restartable_empty_topology() { + const TERMINAL: &str = "00000000000040008000000000000022"; + const INCARNATION: &str = "10000000000040008000000000000022"; + let root = std::env::temp_dir() + .join(format!("cmux-server-stop-restart-{}", crate::workspace_registry::new_uuid_v4())); + let session = "server-stop-restart"; + { + let registry = WorkspaceRegistry::open(&root, session).unwrap(); + let mux = Mux::from_workspace_registry( + session.into(), + SurfaceOptions::default(), + registry, + ProviderWorkspaceState::default(), + true, + ) + .unwrap(); + let workspace = mux.create_empty_workspace(Some("durable".into()), None, None).unwrap(); + mux.seed_running_terminal_for_test(TERMINAL, INCARNATION, &workspace.key).unwrap(); + + assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + } + + let reopened = Mux::open_persistent(session, SurfaceOptions::default(), &root) + .expect("server stop must leave a restartable durable session"); + reopened.with_state(|state| { + assert_eq!(state.workspaces.len(), 1); + assert!(state.workspaces[0].screens.is_empty()); + assert!(state.panes.is_empty()); + assert!(state.surfaces.is_empty()); + }); + reopened.shutdown().unwrap(); + drop(reopened); + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn closing_active_pane_focuses_most_recent_remaining_pane() { let mux = test_mux(); From 03bf6f099fc1fbe2f4bbcdf1627caa1fe1eab96c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:07:37 -0700 Subject: [PATCH 302/521] fix(tui): close durable topology on server stop --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 89 ++++++++++++++++++------ 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fe33c59fe591..7dd627dad521 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -7816,45 +7816,87 @@ impl Mux { records }; - let (surfaces, retained_count, tree_changed) = { + let (surfaces, retained_count, tree_changed, closed_public_ids) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); + let operation = "server.stop"; + let fingerprint = serde_json::json!({"operation":operation}); let mut registry = self.workspace_registry.lock().unwrap(); - let terminals = registry - .terminal_snapshot()? + let terminal_snapshot = registry.terminal_snapshot()?; + let terminals = terminal_snapshot .terminals .into_iter() .map(|terminal| (terminal.terminal_id, terminal.incarnation)) .collect::>(); - let batch = registry.close_terminals_atomically(&mutation, &terminals)?; + let closed_public_ids = Self::terminal_public_ids_for_hosted(®istry, &terminals)?; + let preparation = registry.prepare_resource_effect( + &mutation.id, + operation, + &fingerprint, + &serde_json::json!({"scope":"session"}), + None, + None, + )?; + anyhow::ensure!( + matches!(preparation, ResourceEffectPreparation::Execute { .. }), + "server stop could not reserve its durable close transaction" + ); + registry.mark_resource_effect_executing(&mutation.id, operation, &fingerprint)?; - let (surfaces, retained_count, tree_changed) = { + let result = (|| -> anyhow::Result<_> { let mut state = self.state.lock().unwrap(); let retained_count = self.shutdown_owners.len(); let tree_changed = !state.surfaces.is_empty() || !state.panes.is_empty() || state.workspaces.iter().any(|workspace| !workspace.screens.is_empty()); - let surfaces = - state.surfaces.drain().map(|(_, surface)| surface).collect::>(); - for surface in &surfaces { - let _ = self.shutdown_owners.stage_surface(surface); - } - for workspace in &mut state.workspaces { + let surfaces = state.surfaces.values().cloned().collect::>(); + let mut projected = state.clone(); + projected.surfaces.clear(); + for workspace in &mut projected.workspaces { workspace.screens.clear(); workspace.active_screen = 0; } - if !state.panes.is_empty() { - state.pane_revision = state.pane_revision.saturating_add(1); + if !projected.panes.is_empty() { + projected.pane_revision = projected.pane_revision.saturating_add(1); + } + projected.panes.clear(); + projected.split_screens.clear(); + let projection = self.resource_effect_projection_locked( + ®istry, + &mut projected, + serde_json::json!({"stopped":true}), + )?; + let close = registry.commit_resource_close_patch( + &mutation.id, + operation, + &fingerprint, + &projection.patch, + &projection.result, + &projection.changes, + &terminals, + None, + )?; + projected.resource_revision = close.resource.revision; + for surface in &surfaces { + let _ = self.shutdown_owners.stage_surface(surface); + } + *state = projected; + Ok((surfaces, retained_count, tree_changed, close.terminal_batch)) + })(); + let (surfaces, retained_count, tree_changed, terminal_batch) = match result { + Ok(result) => result, + Err(error) => { + let _ = registry.mark_resource_effect_indeterminate(&mutation.id); + return Err(error); } - state.panes.clear(); - state.split_screens.clear(); - (surfaces, retained_count, tree_changed) }; - if batch.closed != 0 { - self.emit_terminal_registry_changed(®istry, batch.revision); + if terminal_batch.closed != 0 { + self.emit_terminal_registry_changed(®istry, terminal_batch.revision); } - (surfaces, retained_count, tree_changed) + (surfaces, retained_count, tree_changed, closed_public_ids) }; + self.notify_terminal_exit_waiters(closed_public_ids); + self.publish_resource_event(); self.pending_workspace_surfaces.lock().unwrap().clear(); self.agent_records.lock().unwrap().clear(); self.surface_notifications.lock().unwrap().clear(); @@ -19921,7 +19963,14 @@ mod tests { drop(sizing); let events = events.try_iter().collect::>(); assert_eq!(events.iter().filter(|event| matches!(event, MuxEvent::TreeChanged)).count(), 1); - assert_eq!(events.len(), 1); + assert_eq!( + events + .iter() + .filter(|event| matches!(event, MuxEvent::TerminalRegistryChanged { .. })) + .count(), + 1 + ); + assert_eq!(events.len(), 2); let error = mux.new_workspace(None, None).unwrap_err(); assert_eq!(error.to_string(), "server is shutting down"); } From 3e56fd32a5b1c035d9fe246e40f84f05fead77c4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:28:46 -0700 Subject: [PATCH 303/521] fix(tui): fence terminal creation through publication --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7dd627dad521..98417a73d8a3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9002,6 +9002,7 @@ impl Mux { name: Option, size: Option<(u16, u16)>, ) -> anyhow::Result<(Arc, RunPlacement)> { + let _creation = self.begin_surface_creation()?; let _creation_handoff = self.resource_creation_handoff.lock().unwrap(); let selectors = self .ordinary_workspace_selectors(workspace) From 1c824fe724ca74bf81a9aa916c9aab66ae073c2f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:08:56 -0700 Subject: [PATCH 304/521] test: expose cleanup ownership failures --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 53 +++++++++++++++++ cmux-tui/crates/cmux-tui/src/main.rs | 58 ++++++++++++++++++- 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 25973959cc17..b46d1651193b 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -120,6 +120,59 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await Set(recorder.freed) == Set(surfaces.map { UInt(bitPattern: $0) })) } + @Test func oneStuckCloseDoesNotStrandLaterCloses() async { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let surfaces = (0..<3).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + defer { for surface in surfaces { surface.deallocate() } } + let firstFreeStarted = AsyncStream.makeStream() + let releaseFirstFree = DispatchSemaphore(value: 0) + defer { + releaseFirstFree.signal() + firstFreeStarted.continuation.finish() + } + + let firstTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.stuckClose", + surface: surfaces[0], + callbackContext: nil, + freeSurface: { _ in + firstFreeStarted.continuation.yield() + releaseFirstFree.wait() + } + ) + var firstFreeIterator = firstFreeStarted.stream.makeAsyncIterator() + _ = await firstFreeIterator.next() + + let secondTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.closeAfterStuckClose", + surface: surfaces[1], + callbackContext: nil, + freeSurface: { _ in } + ) + let thirdTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondCloseAfterStuckClose", + surface: surfaces[2], + callbackContext: nil, + freeSurface: { _ in } + ) + + let secondCompleted = await secondTicket.wait(timeout: .seconds(1)) + let thirdCompleted = await thirdTicket.wait(timeout: .seconds(1)) + releaseFirstFree.signal() + + #expect(secondCompleted, "one stuck native free stranded the next explicit close") + #expect(thirdCompleted, "one stuck native free stranded the serialized close lane") + #expect(await firstTicket.wait(timeout: .seconds(1))) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index f4d4c5ce19dc..3b7b1bc72c07 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1325,6 +1325,19 @@ enum ServerShutdownCleanupOutcome { #[cfg(test)] type ServerShutdownAttempt = Arc anyhow::Result<()> + Send + Sync + 'static>; +#[cfg(test)] +thread_local! { + static SERVER_SHUTDOWN_ATTEMPT_FOR_RUN: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(test)] +fn set_server_shutdown_attempt_for_run(attempt: Option) { + SERVER_SHUTDOWN_ATTEMPT_FOR_RUN.with(|slot| { + *slot.borrow_mut() = attempt; + }); +} + struct ServerShutdownCleanup { mux: Arc, state: std::sync::Mutex, @@ -1340,7 +1353,9 @@ impl ServerShutdownCleanup { state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending(0)), changed: std::sync::Condvar::new(), #[cfg(test)] - shutdown_attempt: std::sync::Mutex::new(None), + shutdown_attempt: std::sync::Mutex::new( + SERVER_SHUTDOWN_ATTEMPT_FOR_RUN.with(|slot| slot.borrow().clone()), + ), } } @@ -2451,6 +2466,47 @@ mod tests { assert_eq!(attempts.load(Ordering::SeqCst), 1); } + #[cfg(unix)] + #[test] + fn remote_runtime_start_failure_cleans_published_server() { + use std::time::{SystemTime, UNIX_EPOCH}; + + let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + let directory = + PathBuf::from("/tmp").join(format!("crs-{}-{suffix:x}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let socket_path = directory.join("server.sock"); + let remote_state_file = directory.join("remote-state-file"); + std::fs::write(&remote_state_file, b"not a directory").unwrap(); + let mut server_args = args(&["--ephemeral", "--headless", "--remote"]); + server_args.session = format!("remote-start-cleanup-{suffix:x}"); + server_args.socket = Some(socket_path.clone()); + server_args.remote_state_dir = Some(remote_state_file); + + let shutdown_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + set_server_shutdown_attempt_for_run(Some({ + let shutdown_attempts = shutdown_attempts.clone(); + Arc::new(move || { + shutdown_attempts.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + })); + + let result = run_server(server_args, None); + set_server_shutdown_attempt_for_run(None); + let shutdown_attempt_count = shutdown_attempts.load(Ordering::SeqCst); + let socket_remained = socket_path.exists(); + cmux_tui_core::server::cleanup(&socket_path); + std::fs::remove_dir_all(directory).unwrap(); + + assert!(result.is_err(), "invalid remote state unexpectedly started the server"); + assert_eq!( + shutdown_attempt_count, 1, + "remote startup failure bypassed server process cleanup" + ); + assert!(!socket_remained, "remote startup failure retained the published server socket"); + } + fn args(values: &[&str]) -> Args { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } From 143b05298ddc92f86d0904c25731359ba09927b9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:20:08 -0700 Subject: [PATCH 305/521] fix: preserve cleanup ownership under failure --- ...nalSurfaceRuntimeTeardownCoordinator.swift | 66 ++++++++++++------- ...lSurfaceRuntimeTeardownExecutionLane.swift | 4 +- ...TerminalSurfaceRuntimeTeardownTicket.swift | 2 +- ...rfaceRuntimeTeardownCoordinatorTests.swift | 22 +++---- cmux-tui/crates/cmux-tui/src/main.rs | 59 +++++++++++++++-- cmux-tui/crates/cmux-tui/tests/cli.rs | 2 +- 6 files changed, 112 insertions(+), 43 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index af2ba25d4dbf..ec4847448c11 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -8,13 +8,17 @@ internal import CMUXDebugLog /// Coordinates native `ghostty_surface_free` calls off the close/deinit paths. /// -/// Close/deinit frees run one at a time on a utility worker so re-entrant -/// teardown loops cannot form. Each admitted hibernation owns one independently -/// startable utility slot, so one stuck native join cannot strand another pane. +/// Close/deinit frees use two independently startable utility slots. Each slot +/// remains serial, so teardown cannot recursively reuse one worker, while one +/// stuck native join leaves the other slot available to drain later closes. +/// Each admitted hibernation also owns one independently startable utility slot. /// Deadline observers report, but never block on, stuck frees. The app constructs /// exactly one instance and injects it through /// ``TerminalSurfaceRuntimeDependencies``. public actor TerminalSurfaceRuntimeTeardownCoordinator { + /// Maximum native close frees that can run concurrently. + public static let maximumConcurrentCloseTeardownCount = 2 + /// Largest batch that can own independently startable native-free slots. public static let maximumIsolatedHibernationTeardownCount = 2 @@ -27,14 +31,26 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { #else private var pendingReasonsById: [UUID: String] = [:] #endif - private var queuedRequests: [TerminalSurfaceRuntimeTeardownRequest] = [] - private var isWorkerRunning = false + private var queuedCloseRequests: [TerminalSurfaceRuntimeTeardownRequest] = [] + private var availableCloseExecutionSlots: Set + private let closeTeardownQueues: [DispatchQueue] private let isolatedHibernationQueues: [DispatchQueue] private nonisolated let isolatedHibernationAdmission = TerminalSurfaceRuntimeTeardownAdmission() /// Creates the process's teardown coordinator. public init() { + availableCloseExecutionSlots = Set( + 0..?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, - executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .boundedClose, isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? = nil, freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in @@ -172,7 +188,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { func enqueue( _ request: TerminalSurfaceRuntimeTeardownRequest, - executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .serializedClose, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .boundedClose, isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? = nil ) async { @@ -209,31 +225,35 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { isolatedHibernationReservation ) } - case .serializedClose: + case .boundedClose: break } - queuedRequests.append(request) - if !isWorkerRunning { - isWorkerRunning = true - Task.detached(priority: .utility) { - while let request = await self.nextRequestForWorker() { - Task { - await self.observeTimeout(id: request.id) - } - self.freeNativeSurface(request) + queuedCloseRequests.append(request) + scheduleCloseTeardowns() + } + + private func scheduleCloseTeardowns() { + while !queuedCloseRequests.isEmpty, + let executionSlot = availableCloseExecutionSlots.min() { + availableCloseExecutionSlots.remove(executionSlot) + let request = queuedCloseRequests.removeFirst() + Task { + await self.observeTimeout(id: request.id) + } + closeTeardownQueues[executionSlot].async { + self.freeNativeSurface(request) + Task { await self.finishFree(request) await self.complete(id: request.id) + await self.closeTeardownFinished(executionSlot: executionSlot) } } } } - private func nextRequestForWorker() -> TerminalSurfaceRuntimeTeardownRequest? { - guard !queuedRequests.isEmpty else { - isWorkerRunning = false - return nil - } - return queuedRequests.removeFirst() + private func closeTeardownFinished(executionSlot: Int) { + availableCloseExecutionSlots.insert(executionSlot) + scheduleCloseTeardowns() } private nonisolated func freeNativeSurface( diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift index 0a2a800691d5..418dce3fced7 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -1,7 +1,7 @@ /// Selects the ownership boundary for a native surface free. enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { - /// Preserves ordering for close/deinit flows that can re-enter teardown. - case serializedClose + /// Uses the bounded, failure-isolated close worker pool. + case boundedClose /// Gives an explicitly owned hibernation join an independent bounded slot. case isolatedHibernation diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift index 1ed5df6ac631..a3feec39c2b2 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownTicket.swift @@ -1,6 +1,6 @@ public import Foundation -/// An awaitable handle for one serialized native surface teardown. +/// An awaitable handle for one native surface teardown. public struct TerminalSurfaceRuntimeTeardownTicket: Sendable { /// Stable identity used by the surface to clear only its current teardown. public let id: UUID diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index b46d1651193b..2395834b6549 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -169,7 +169,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec releaseFirstFree.signal() #expect(secondCompleted, "one stuck native free stranded the next explicit close") - #expect(thirdCompleted, "one stuck native free stranded the serialized close lane") + #expect(thirdCompleted, "one stuck native free stranded the remaining close worker") #expect(await firstTicket.wait(timeout: .seconds(1))) } @@ -180,16 +180,16 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec byteCount: 8, alignment: 8 ) - let serializedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + let closeSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { isolatedSurface.deallocate() queuedIsolatedSurface.deallocate() - serializedSurface.deallocate() + closeSurface.deallocate() } let isolatedFreeStarted = AsyncStream.makeStream() let releaseIsolatedFree = DispatchSemaphore(value: 0) let secondIsolatedFreeCount = OSAllocatedUnfairLock(initialState: 0) - let serializedFreeCount = OSAllocatedUnfairLock(initialState: 0) + let closeFreeCount = OSAllocatedUnfairLock(initialState: 0) defer { releaseIsolatedFree.signal() isolatedFreeStarted.continuation.finish() @@ -234,20 +234,20 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec secondIsolatedFreeCount.withLock { $0 += 1 } } ) - let serializedTicket = coordinator.enqueueRuntimeTeardown( + let closeTicket = coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), - reason: "test.serializedClose", - surface: serializedSurface, + reason: "test.boundedClose", + surface: closeSurface, callbackContext: nil, freeSurface: { _ in - serializedFreeCount.withLock { $0 += 1 } + closeFreeCount.withLock { $0 += 1 } } ) - #expect(await serializedTicket.wait(timeout: .seconds(1))) + #expect(await closeTicket.wait(timeout: .seconds(1))) #expect(await secondIsolatedTicket.wait(timeout: .seconds(1))) - #expect(serializedFreeCount.withLock { $0 } == 1) + #expect(closeFreeCount.withLock { $0 } == 1) #expect(await isolatedTicket.wait(timeout: .zero) == false) #expect(secondIsolatedFreeCount.withLock { $0 } == 1) @@ -264,7 +264,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec await coordinator.cancelIsolatedHibernationTeardown(nextReservation) } - @Test func staleIsolatedReservationFallsBackToSerializedFree() async throws { + @Test func staleIsolatedReservationFallsBackToBoundedClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { surface.deallocate() } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 3b7b1bc72c07..d34560bb7e5f 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1675,6 +1675,20 @@ fn finish_server_process( result } +#[cfg(unix)] +fn combine_server_and_remote_shutdown_results( + server_result: anyhow::Result<()>, + remote_shutdown_result: anyhow::Result<()>, +) -> anyhow::Result<()> { + match (server_result, remote_shutdown_result) { + (result, Ok(())) => result, + (Ok(()), Err(shutdown_error)) => Err(shutdown_error), + (Err(error), Err(shutdown_error)) => { + Err(error.context(format!("remote daemon shutdown also failed: {shutdown_error:#}"))) + } + } +} + fn server_shutdown_exit_grace() -> std::time::Duration { #[cfg(debug_assertions)] if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS") @@ -1877,7 +1891,7 @@ fn run_server( #[cfg(unix)] let remote_runtime = if args.remote { - let runtime = remote_runtime::start_daemon_runtime( + let runtime = match remote_runtime::start_daemon_runtime( socket_path.clone(), remote_runtime::DaemonRuntimeOptions { session: args.session.clone(), @@ -1893,7 +1907,15 @@ fn run_server( resume_lease: std::time::Duration::from_secs(args.remote_resume_lease_seconds), replaceable_sidecar: false, }, - )?; + ) { + Ok(runtime) => runtime, + Err(error) => { + drop(websocket_server); + #[cfg(target_os = "linux")] + drop(_provider_management); + return finish_server_process(server_process, Err(error)); + } + }; eprintln!( "cmux-tui: remote daemon {}, link {}, admin {}", runtime.info().daemon_fingerprint, @@ -1929,9 +1951,10 @@ fn run_server( run_tui(Session::Local(mux), args.session, None) }; #[cfg(unix)] - if let Some(runtime) = remote_runtime { - runtime.shutdown()?; - } + let result = match remote_runtime { + Some(runtime) => combine_server_and_remote_shutdown_results(result, runtime.shutdown()), + None => result, + }; drop(websocket_server); #[cfg(target_os = "linux")] drop(_provider_management); @@ -2507,6 +2530,32 @@ mod tests { assert!(!socket_remained, "remote startup failure retained the published server socket"); } + #[cfg(unix)] + #[test] + fn remote_shutdown_failure_preserves_an_earlier_server_error() { + let error = combine_server_and_remote_shutdown_results( + Err(anyhow::anyhow!("frontend failed")), + Err(anyhow::anyhow!("remote shutdown failed")), + ) + .unwrap_err(); + let message = format!("{error:#}"); + + assert!(message.contains("frontend failed")); + assert!(message.contains("remote shutdown failed")); + } + + #[cfg(unix)] + #[test] + fn remote_shutdown_failure_is_returned_after_successful_server_execution() { + let error = combine_server_and_remote_shutdown_results( + Ok(()), + Err(anyhow::anyhow!("remote shutdown failed")), + ) + .unwrap_err(); + + assert_eq!(error.to_string(), "remote shutdown failed"); + } + fn args(values: &[&str]) -> Args { parse_args_result(values.iter().map(|value| value.to_string())).unwrap() } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 8f4dba85e526..b68b5a793688 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -184,7 +184,7 @@ fn server_establishes_shutdown_ownership_before_publishing_its_listener() { .spawn() .unwrap(); - let marker_deadline = Instant::now() + Duration::from_secs(5); + let marker_deadline = Instant::now() + Duration::from_secs(15); while !marker.exists() && Instant::now() < marker_deadline { assert!( child.try_wait().unwrap().is_none(), From f7b70aab2f26ebf9ff4e28385ac2621128498f98 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 07:30:10 -0700 Subject: [PATCH 306/521] test(tui): call stream termination accessor --- cmux-tui/bindings/go/resource_api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/bindings/go/resource_api_test.go b/cmux-tui/bindings/go/resource_api_test.go index 2c4125645321..273afd74da4a 100644 --- a/cmux-tui/bindings/go/resource_api_test.go +++ b/cmux-tui/bindings/go/resource_api_test.go @@ -3757,7 +3757,7 @@ func TestRejectedStreamEndStillRecordsServerTermination(t *testing.T) { if route.deliver(streamMessage{envelope: streamEnvelope{Type: "stream_end"}, size: 1}) { t.Fatal("stream_end unexpectedly fit in the full delivery queue") } - if !route.hasServerEnd() { + if !route.endedByServer() { t.Fatal("observed stream_end was lost when its delivery queue was full") } } From 96fad712e81b19f8ad578473d256120260dfce83 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:11:39 -0700 Subject: [PATCH 307/521] test: expose remaining lifecycle fences --- ...SurfaceTeardownCallbackLifetimeTests.swift | 31 ++++++++++++ cmux-tui/Cargo.lock | 2 + cmux-tui/crates/cmux-pty/Cargo.toml | 3 ++ cmux-tui/crates/cmux-pty/src/lib.rs | 32 +++++++++++++ cmux-tui/crates/cmux-remote/Cargo.toml | 1 + .../cmux-remote/src/workspace/process.rs | 47 +++++++++++++++++++ 6 files changed, 116 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift index 95eafb2a3ac4..148e2664a3b1 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceTeardownCallbackLifetimeTests.swift @@ -126,6 +126,37 @@ import Testing freeStarted.continuation.finish() } + @Test func closeDuringAgentHibernationWaitsForNativeFreeCompletion() async throws { + let registry = TerminalSurfaceRegistry() + let surface = makeSurface(registry: registry) + let runtimeSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + registry.registerRuntimeSurface(runtimeSurface, ownerId: surface.id) + surface.installRuntimeSurfaceForTesting(runtimeSurface) + defer { runtimeSurface.deallocate() } + let freeStarted = AsyncStream.makeStream() + let allowFree = DispatchSemaphore(value: 0) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in + freeStarted.continuation.yield() + allowFree.wait() + } + defer { + allowFree.signal() + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil + } + + #expect(surface.suspendRuntimeSurfaceForAgentHibernation(reason: "test.hibernate")) + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + + let closeTicket = try #require(surface.teardownSurface()) + #expect(await closeTicket.wait(timeout: .zero) == false) + #expect(await surface.waitForCloseRuntimeTeardown(timeout: .zero) == false) + + allowFree.signal() + #expect(await surface.waitForCloseRuntimeTeardown(timeout: .seconds(1))) + freeStarted.continuation.finish() + } + @Test func hibernationAdmissionFailurePreservesLiveRuntimeSurface() throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let firstReservation = try #require( diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index aa14b53438db..76b799fb0937 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -548,6 +548,7 @@ name = "cmux-pty" version = "0.1.0" dependencies = [ "anyhow", + "cmux-tui-process", "libc", "portable-pty", ] @@ -584,6 +585,7 @@ dependencies = [ "cmux-pty", "cmux-relay", "cmux-remote-protocol", + "cmux-tui-process", "diffy", "fs4", "futures-util", diff --git a/cmux-tui/crates/cmux-pty/Cargo.toml b/cmux-tui/crates/cmux-pty/Cargo.toml index 557f55fbd29f..635ce3147f7f 100644 --- a/cmux-tui/crates/cmux-pty/Cargo.toml +++ b/cmux-tui/crates/cmux-pty/Cargo.toml @@ -14,3 +14,6 @@ workspace = true anyhow.workspace = true libc.workspace = true portable-pty.workspace = true + +[dev-dependencies] +cmux-tui-process.workspace = true diff --git a/cmux-tui/crates/cmux-pty/src/lib.rs b/cmux-tui/crates/cmux-pty/src/lib.rs index 21ff09bdfcdd..8ee966c370b9 100644 --- a/cmux-tui/crates/cmux-pty/src/lib.rs +++ b/cmux-tui/crates/cmux-pty/src/lib.rs @@ -161,4 +161,36 @@ mod tests { "child cleanup changed the parent descriptor" ); } + + #[test] + fn pty_spawn_waits_for_process_creation_barrier() { + let pair = open(PtySize { rows: 24, cols: 80, pixel_width: 0, pixel_height: 0 }).unwrap(); + let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (sender, receiver) = std::sync::mpsc::channel(); + let publisher = std::thread::spawn(move || { + sender.send(pair.spawn(PtyCommand::new("/usr/bin/true"))).unwrap(); + }); + + let early = match receiver.recv_timeout(std::time::Duration::from_millis(250)) { + Ok(result) => Some(result), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => None, + Err(error) => panic!("PTY spawn channel failed: {error}"), + }; + drop(barrier); + + let spawned_while_barrier_held = early.is_some(); + let result = match early { + Some(result) => result, + None => receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("PTY spawn did not resume after the process barrier was released"), + }; + let mut spawned = result.unwrap(); + assert!(spawned.child.wait().unwrap().success()); + publisher.join().unwrap(); + assert!( + !spawned_while_barrier_held, + "PTY child started while the process barrier was held" + ); + } } diff --git a/cmux-tui/crates/cmux-remote/Cargo.toml b/cmux-tui/crates/cmux-remote/Cargo.toml index 0fce33b39272..4842e903015a 100644 --- a/cmux-tui/crates/cmux-remote/Cargo.toml +++ b/cmux-tui/crates/cmux-remote/Cargo.toml @@ -46,6 +46,7 @@ zeroize.workspace = true [dev-dependencies] cmux-relay = { path = "../cmux-relay" } +cmux-tui-process.workspace = true rcgen = "0.14" tempfile = "3" tower = { version = "0.5", features = ["util"] } diff --git a/cmux-tui/crates/cmux-remote/src/workspace/process.rs b/cmux-tui/crates/cmux-remote/src/workspace/process.rs index 3a0a2ee9026e..d21409489ba0 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/process.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/process.rs @@ -2567,6 +2567,53 @@ mod tests { } } + #[cfg(target_os = "macos")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn pipe_spawn_waits_for_process_creation_barrier() { + let (directory, root) = root().await; + let manager = Arc::new(ProcessManager::default()); + let pid_file = directory.path().join("barrier-pipe.pid"); + let (barrier_held_sender, barrier_held_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let barrier_holder = std::thread::spawn(move || { + let _barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + barrier_held_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + }); + barrier_held_receiver + .recv_timeout(std::time::Duration::from_secs(1)) + .expect("process creation barrier was not acquired"); + + let mut options = spawn_options( + vec![ + "/bin/sh".into(), + "-c".into(), + "printf '%s' \"$$\" > \"$PIDFILE\"; exec sleep 30".into(), + ], + ProcessIo::Pipes { stdin: false }, + ProcessLifetime::Workspace, + ); + options.env.insert("PIDFILE".into(), pid_file.to_string_lossy().into_owned()); + let spawn_manager = manager.clone(); + let spawn = tokio::spawn(async move { spawn_manager.spawn(root, options).await }); + + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + let spawned_while_barrier_held = pid_file.exists(); + release_sender.send(()).unwrap(); + barrier_holder.join().unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(5), spawn) + .await + .expect("pipe spawn did not resume after the process barrier was released") + .unwrap() + .unwrap(); + manager.shutdown().await; + assert!( + !spawned_while_barrier_held, + "remote pipe child started while the process barrier was held" + ); + } + #[cfg(unix)] #[test] fn pty_identity_uses_the_direct_child_instead_of_the_foreground_job() { From 511bc460160149442d5faf67257a9abeec86a246 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:18:27 -0700 Subject: [PATCH 308/521] fix: close remaining lifecycle fence gaps --- .../Surface/TerminalSurface+RuntimeLifecycle.swift | 7 +++++++ cmux-tui/crates/cmux-pty/Cargo.toml | 4 +--- cmux-tui/crates/cmux-pty/src/macos.rs | 2 ++ cmux-tui/crates/cmux-remote/Cargo.toml | 2 +- cmux-tui/crates/cmux-remote/src/lib.rs | 1 + cmux-tui/crates/cmux-remote/src/process_creation.rs | 9 +++++++++ cmux-tui/crates/cmux-remote/src/provider/relay.rs | 4 ++-- cmux-tui/crates/cmux-remote/src/provider/ssh.rs | 3 +-- cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs | 7 +++---- cmux-tui/crates/cmux-remote/src/workspace/git.rs | 3 +-- cmux-tui/crates/cmux-remote/src/workspace/process.rs | 3 +-- cmux-tui/crates/cmux-tui/src/remote_cli.rs | 6 ++++-- 12 files changed, 33 insertions(+), 18 deletions(-) create mode 100644 cmux-tui/crates/cmux-remote/src/process_creation.rs diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 921c7278bf62..1332629345ec 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -259,6 +259,10 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if closeRuntimeTeardownTicket == nil { + closeRuntimeTeardownTicket = + agentHibernationRuntimeTeardownTicket + } return closeRuntimeTeardownTicket } @@ -316,6 +320,9 @@ extension TerminalSurface { if completed, closeRuntimeTeardownTicket?.id == ticket.id { closeRuntimeTeardownTicket = nil } + if completed, agentHibernationRuntimeTeardownTicket?.id == ticket.id { + agentHibernationRuntimeTeardownTicket = nil + } return completed } diff --git a/cmux-tui/crates/cmux-pty/Cargo.toml b/cmux-tui/crates/cmux-pty/Cargo.toml index 635ce3147f7f..290bb3b26411 100644 --- a/cmux-tui/crates/cmux-pty/Cargo.toml +++ b/cmux-tui/crates/cmux-pty/Cargo.toml @@ -12,8 +12,6 @@ workspace = true [dependencies] anyhow.workspace = true +cmux-tui-process.workspace = true libc.workspace = true portable-pty.workspace = true - -[dev-dependencies] -cmux-tui-process.workspace = true diff --git a/cmux-tui/crates/cmux-pty/src/macos.rs b/cmux-tui/crates/cmux-pty/src/macos.rs index ddcbb49ac8bc..66043f5f53f0 100644 --- a/cmux-tui/crates/cmux-pty/src/macos.rs +++ b/cmux-tui/crates/cmux-pty/src/macos.rs @@ -15,6 +15,7 @@ use super::{Child, MasterPty, PtyCommand, PtySize}; pub(crate) struct Slave(File); pub(crate) fn open(size: PtySize) -> anyhow::Result<(Box, Slave)> { + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); let mut master_fd = -1; let mut slave_fd = -1; let mut window_size = libc::winsize { @@ -48,6 +49,7 @@ pub(crate) fn spawn( slave: &Slave, command: PtyCommand, ) -> anyhow::Result> { + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); let shell = resolved_shell(&command); let mut process = Command::new(&command.program); process.args(&command.args); diff --git a/cmux-tui/crates/cmux-remote/Cargo.toml b/cmux-tui/crates/cmux-remote/Cargo.toml index 4842e903015a..e176b963a6b3 100644 --- a/cmux-tui/crates/cmux-remote/Cargo.toml +++ b/cmux-tui/crates/cmux-remote/Cargo.toml @@ -22,6 +22,7 @@ base64.workspace = true bytes.workspace = true cmux-pty.workspace = true cmux-remote-protocol.workspace = true +cmux-tui-process.workspace = true diffy.workspace = true futures-util.workspace = true fs4.workspace = true @@ -46,7 +47,6 @@ zeroize.workspace = true [dev-dependencies] cmux-relay = { path = "../cmux-relay" } -cmux-tui-process.workspace = true rcgen = "0.14" tempfile = "3" tower = { version = "0.5", features = ["util"] } diff --git a/cmux-tui/crates/cmux-remote/src/lib.rs b/cmux-tui/crates/cmux-remote/src/lib.rs index 346399b1985c..3723e805592d 100644 --- a/cmux-tui/crates/cmux-remote/src/lib.rs +++ b/cmux-tui/crates/cmux-remote/src/lib.rs @@ -14,6 +14,7 @@ mod mux_codec; mod mux_input; mod mux_lanes; pub mod observability; +mod process_creation; pub mod provider; pub mod service; pub mod services; diff --git a/cmux-tui/crates/cmux-remote/src/process_creation.rs b/cmux-tui/crates/cmux-remote/src/process_creation.rs new file mode 100644 index 000000000000..f4c77f2dc0fb --- /dev/null +++ b/cmux-tui/crates/cmux-remote/src/process_creation.rs @@ -0,0 +1,9 @@ +//! Process creation coordinated with non-atomic descriptor setup on macOS. + +use std::io; + +/// Starts a Tokio child while holding cmux-tui's process-wide creation guard. +pub(crate) fn spawn(command: &mut tokio::process::Command) -> io::Result { + let _process_creation = cmux_tui_process::ProcessCreationGuard::acquire(); + command.spawn() +} diff --git a/cmux-tui/crates/cmux-remote/src/provider/relay.rs b/cmux-tui/crates/cmux-remote/src/provider/relay.rs index 7b6cde701cc0..9f7d6948e502 100644 --- a/cmux-tui/crates/cmux-remote/src/provider/relay.rs +++ b/cmux-tui/crates/cmux-remote/src/provider/relay.rs @@ -219,8 +219,8 @@ async fn read_credential_command( .stdout(Stdio::piped()) .stderr(Stdio::null()) .kill_on_drop(true); - let mut child = - command.spawn().map_err(|_| credential_source_error("command could not be started"))?; + let mut child = crate::process_creation::spawn(&mut command) + .map_err(|_| credential_source_error("command could not be started"))?; let stdout = child .stdout .take() diff --git a/cmux-tui/crates/cmux-remote/src/provider/ssh.rs b/cmux-tui/crates/cmux-remote/src/provider/ssh.rs index 937296cca8e8..f61bbb66078b 100644 --- a/cmux-tui/crates/cmux-remote/src/provider/ssh.rs +++ b/cmux-tui/crates/cmux-remote/src/provider/ssh.rs @@ -180,8 +180,7 @@ impl LinkGroup for SshLinkGroup { .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .kill_on_drop(true); - let mut child = command - .spawn() + let mut child = crate::process_creation::spawn(&mut command) .map_err(|error| ProviderError::Transport(format!("could not start ssh: {error}")))?; let stdin = child .stdin diff --git a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs index b880f6c630da..a4c675958d36 100644 --- a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs +++ b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs @@ -262,13 +262,12 @@ impl SshBootstrapper { for argument in remote_arguments { command.arg(argument); } - let mut child = command + command .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .map_err(BootstrapError::Io)?; + .kill_on_drop(true); + let mut child = crate::process_creation::spawn(&mut command).map_err(BootstrapError::Io)?; let stdout = match child.stdout.take() { Some(stdout) => stdout, None => { diff --git a/cmux-tui/crates/cmux-remote/src/workspace/git.rs b/cmux-tui/crates/cmux-remote/src/workspace/git.rs index c977e765e595..e2a92f0f8f7d 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/git.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/git.rs @@ -207,8 +207,7 @@ async fn run_git( .env("GIT_PAGER", "cat") .env("LC_ALL", "C") .kill_on_drop(true); - let mut child = command - .spawn() + let mut child = crate::process_creation::spawn(&mut command) .map_err(|error| RpcError::new("git-unavailable", format!("start git: {error}")))?; let mut stdout = child.stdout.take().ok_or_else(|| RpcError::new("internal", "git stdout was not piped"))?; diff --git a/cmux-tui/crates/cmux-remote/src/workspace/process.rs b/cmux-tui/crates/cmux-remote/src/workspace/process.rs index d21409489ba0..f0c54ec0f901 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/process.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/process.rs @@ -1165,8 +1165,7 @@ impl ProcessManager { let mut child_events = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::child()) .map_err(|error| RpcError::new("process-spawn-failed", error.to_string()))?; - let mut child = command - .spawn() + let mut child = crate::process_creation::spawn(&mut command) .map_err(|error| RpcError::new("process-spawn-failed", error.to_string()))?; let pid = child.id(); #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/src/remote_cli.rs b/cmux-tui/crates/cmux-tui/src/remote_cli.rs index 216416205cec..30866fc6b262 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_cli.rs @@ -1843,7 +1843,8 @@ fn ensure_daemon( .stdout(Stdio::from(log.try_clone()?)) .stderr(Stdio::from(log)); configure_detached_process(&mut mux_owner); - let mut child = mux_owner.spawn().context("could not start remote mux owner")?; + let mut child = + cmux_tui_process::spawn(&mut mux_owner).context("could not start remote mux owner")?; let deadline = Instant::now() + Duration::from_secs(20); while Instant::now() < deadline { if UnixStream::connect(&mux_socket).is_ok() { @@ -1875,7 +1876,8 @@ fn ensure_daemon( command.stdin(Stdio::null()); command.stdout(Stdio::from(log.try_clone()?)).stderr(Stdio::from(log)); configure_detached_process(&mut command); - let mut child = command.spawn().context("could not start remote daemon")?; + let mut child = + cmux_tui_process::spawn(&mut command).context("could not start remote daemon")?; let deadline = Instant::now() + Duration::from_secs(20); while Instant::now() < deadline { if UnixStream::connect(link).is_ok() { From 74fe3587d691cb897a9da28e5b2252df893f71aa Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 08:45:11 -0700 Subject: [PATCH 309/521] fix(tui): clean up published startup failures --- cmux-tui/crates/cmux-tui/src/main.rs | 14 ++++++++++++-- cmux-tui/crates/cmux-tui/tests/cli.rs | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index d34560bb7e5f..da770e828cf0 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1661,9 +1661,19 @@ fn finish_server_process( server_process: ServerProcessShutdownGuard, result: anyhow::Result<()>, ) -> anyhow::Result<()> { - if !server_process.has_published_socket() { + let has_published_socket = server_process.has_published_socket(); + if !has_published_socket || result.is_err() { let shutdown_result = server_process.shutdown(); - server_process.abandon_unpublished(); + if has_published_socket { + if shutdown_result.is_err() { + // A failed cleanup retains the published socket and exact + // process owners until an explicit stop retries it. + server_process.wait_for_shutdown(); + } + server_process.complete(); + } else { + server_process.abandon_unpublished(); + } return match (result, shutdown_result) { (result, Ok(())) => result, (Ok(()), Err(shutdown_error)) => Err(shutdown_error), diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b68b5a793688..7fac5ad3d8a7 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1793,10 +1793,15 @@ fn invalid_websocket_startup_returns_after_publishing_the_local_socket() { let dir = unique_temp_dir("invalid-websocket-startup"); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); + let ghostty = dir.join("ghostty"); + fs::write(&ghostty, "#!/bin/sh\nprintf 'background = #272822\\nforeground = #fdfff1\\n'\n") + .unwrap(); + fs::set_permissions(&ghostty, fs::Permissions::from_mode(0o700)).unwrap(); let mut server = Command::new(bin()) .args(["--headless", "--ephemeral", "--socket"]) .arg(&socket) .args(["--ws", "not-an-address"]) + .env("GHOSTTY_BIN", &ghostty) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() From f2d8ae46487f19ad4d78f3cd26732fd6f9ab3055 Mon Sep 17 00:00:00 2001 From: Abdulaziz Albahar <67667005+azooz2003-bit@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:14:34 -0700 Subject: [PATCH 310/521] test: fix workspace drop fixture initializer --- .../WorkspaceListTableCoordinatorDropTests.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceListTableCoordinatorDropTests.swift b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceListTableCoordinatorDropTests.swift index 94b554c24e34..278ebc1b09f9 100644 --- a/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceListTableCoordinatorDropTests.swift +++ b/Packages/iOS/CmuxMobileShellUI/Tests/CmuxMobileShellUITests/WorkspaceListTableCoordinatorDropTests.swift @@ -58,8 +58,6 @@ import UIKit workspaceChangeChipsByWorkspaceID: [:], openWorkspaceChanges: nil, connectionRequiresReauth: false, - connectionRecoveryFailed: false, - isRecoveringConnection: false, connectionError: nil, host: "Test Mac", isInitialConnectionLoading: false, @@ -90,7 +88,6 @@ import UIKit deleteWorkspaceGroup: nil, toggleGroupCollapsed: nil, showAll: {}, - retryConnectionRecovery: nil, signOut: nil, retryInitialConnection: nil, showAddDevice: nil, From bf837991f0cc305c9b1bf9aee0eed8cb60675de3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 11:06:52 -0700 Subject: [PATCH 311/521] test(tui): update generated command inventory counts --- cmux-tui/bindings/cpp/tests/test_generated.cpp | 2 +- cmux-tui/bindings/go/raw/client_test.go | 4 ++-- cmux-tui/bindings/python/tests/test_protocol.py | 2 +- cmux-tui/bindings/zig/examples/watch.zig | 2 +- cmux-tui/bindings/zig/src/raw.zig | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmux-tui/bindings/cpp/tests/test_generated.cpp b/cmux-tui/bindings/cpp/tests/test_generated.cpp index 10e540918e17..a457bc71a6de 100644 --- a/cmux-tui/bindings/cpp/tests/test_generated.cpp +++ b/cmux-tui/bindings/cpp/tests/test_generated.cpp @@ -84,7 +84,7 @@ static_assert(std::is_same_v< static_assert(!std::is_copy_constructible_v); static_assert(std::is_move_constructible_v); -constexpr std::size_t kExpectedRawCommandCount = 92U; +constexpr std::size_t kExpectedRawCommandCount = 93U; constexpr std::array kViewportHistoryCommandNames{ "clear-history", "new-pane-right", diff --git a/cmux-tui/bindings/go/raw/client_test.go b/cmux-tui/bindings/go/raw/client_test.go index 6ce472b68fa4..8faa57a5839d 100644 --- a/cmux-tui/bindings/go/raw/client_test.go +++ b/cmux-tui/bindings/go/raw/client_test.go @@ -20,8 +20,8 @@ import ( func TestGeneratedInventoryHasTypedMethodForEveryCommand(t *testing.T) { commands := AllCommandMetadata() - if len(commands) != 92 { - t.Fatalf("generated commands = %d, want 92", len(commands)) + if len(commands) != 93 { + t.Fatalf("generated commands = %d, want 93", len(commands)) } clientType := reflect.TypeOf((*Client)(nil)) commandNames := make(map[string]struct{}, len(commands)) diff --git a/cmux-tui/bindings/python/tests/test_protocol.py b/cmux-tui/bindings/python/tests/test_protocol.py index 3b745a12a514..edc224246a5f 100644 --- a/cmux-tui/bindings/python/tests/test_protocol.py +++ b/cmux-tui/bindings/python/tests/test_protocol.py @@ -22,7 +22,7 @@ class GeneratedProtocolTests(unittest.TestCase): def test_protocol_ten_inventory_is_exhaustive(self) -> None: self.assertEqual(MUX_PROTOCOL, 10) - self.assertEqual(len(COMMANDS), 92) + self.assertEqual(len(COMMANDS), 93) self.assertEqual(set(COMMANDS), set(SCHEMA["commands"])) self.assertEqual(set(EVENTS), set(SCHEMA["events"])) self.assertEqual(len(IR_SHA256), 64) diff --git a/cmux-tui/bindings/zig/examples/watch.zig b/cmux-tui/bindings/zig/examples/watch.zig index fa3d606074a9..74d48e64cdbe 100644 --- a/cmux-tui/bindings/zig/examples/watch.zig +++ b/cmux-tui/bindings/zig/examples/watch.zig @@ -126,7 +126,7 @@ test "package consumer imports handwritten root and generated raw module" { wheel.pointer_frame_seq, ); try std.testing.expectEqual( - @as(usize, 92), + @as(usize, 93), cmux.raw.protocol.command_count, ); try std.testing.expectEqual( diff --git a/cmux-tui/bindings/zig/src/raw.zig b/cmux-tui/bindings/zig/src/raw.zig index 779ceb0b2c8c..eae0c805008d 100644 --- a/cmux-tui/bindings/zig/src/raw.zig +++ b/cmux-tui/bindings/zig/src/raw.zig @@ -44,7 +44,7 @@ test { _ = @import("raw/wire_presence_test.zig"); _ = @import("raw/generated/presence_test.zig"); std.testing.refAllDecls(protocol); - try std.testing.expectEqual(@as(usize, 92), protocol.command_count); + try std.testing.expectEqual(@as(usize, 93), protocol.command_count); for ([_][]const u8{ "browser-frame-presented", "browser-key-press", From 55f3318cbfef83616535b0e7da5b4d571ed6466e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 11:40:58 -0700 Subject: [PATCH 312/521] test(ci): follow safe TUI package env contract --- tests/test_tui_npm_package_artifact.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_tui_npm_package_artifact.py b/tests/test_tui_npm_package_artifact.py index dc893373f9c4..c1782106b357 100644 --- a/tests/test_tui_npm_package_artifact.py +++ b/tests/test_tui_npm_package_artifact.py @@ -86,13 +86,16 @@ def test_publish_workflows_restore_the_mode_preserving_archive() -> None: def test_build_workflow_executes_stamped_raw_and_packaged_artifacts() -> None: build = (ROOT / ".github/workflows/cmux-tui-build-package.yml").read_text() - conditional_stamp = ( - "CMUX_TUI_DISTRIBUTION_VERSION: " - "${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }}" + safe_build_env = ( + "PACKAGE_NPM: ${{ inputs.package_npm }}\n" + " VERSION: ${{ inputs.version }}" ) - assert "CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }}" not in build - assert build.count(conditional_stamp) == 3 + assert build.count(safe_build_env) == 3 + assert build.count('CMUX_TUI_DISTRIBUTION_VERSION="$VERSION"') == 3 + assert build.count('CMUX_TUI_NPM_BOOTSTRAP_VERSION="$VERSION"') == 3 + assert 'CMUX_TUI_DISTRIBUTION_VERSION="${{ inputs.version }}"' not in build + assert 'if [[ "${{ inputs.package_npm }}" == "true" ]]; then' not in build assert build.count("verify_artifact_identity.py") == 4 assert "dist/npm-smoke/node_modules/cmux/bin/cmux.js" in build assert "/tmp/cmux-tui-wheel-smoke/bin/cmux" in build From c3f47885ab40d09a26a0f39fec6786d3887e24c3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:07:58 -0700 Subject: [PATCH 313/521] test(tui): expose remote socket inheritance race --- cmux-tui/crates/cmux-remote/src/daemon.rs | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/cmux-tui/crates/cmux-remote/src/daemon.rs b/cmux-tui/crates/cmux-remote/src/daemon.rs index 693abd1f4434..e36898334bb6 100644 --- a/cmux-tui/crates/cmux-remote/src/daemon.rs +++ b/cmux-tui/crates/cmux-remote/src/daemon.rs @@ -1495,6 +1495,83 @@ mod tests { CarrierEvidence, LinkGroup, LinkRequest, ProviderCapabilities, ProviderError, }; + #[cfg(target_os = "macos")] + fn hold_process_creation_barrier() + -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) { + let (held_sender, held_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let _barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + held_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + }); + held_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("process creation barrier was not acquired"); + (release_sender, holder) + } + + #[cfg(target_os = "macos")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn direct_socket_bind_waits_for_process_creation_barrier() { + let directory = tempdir().unwrap(); + let auth = AuthDatabase::load_or_create(directory.path(), "guarded-bind", true).unwrap(); + let (daemon, _accepted) = RemoteDaemon::new(auth, SessionLimits::default()); + let (release, holder) = hold_process_creation_barrier(); + + let bind = tokio::spawn(serve_direct_websocket( + daemon, + "127.0.0.1:0".parse().unwrap(), + 128 * 1024, + false, + )); + tokio::time::sleep(Duration::from_millis(250)).await; + let completed_while_barrier_held = bind.is_finished(); + release.send(()).unwrap(); + holder.join().unwrap(); + + let server = tokio::time::timeout(Duration::from_secs(5), bind) + .await + .expect("direct socket bind did not resume after the process barrier was released") + .unwrap() + .unwrap(); + server.shutdown().await.unwrap(); + assert!( + !completed_while_barrier_held, + "direct socket bound while a concurrent process could inherit its descriptor" + ); + } + + #[cfg(target_os = "macos")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn direct_socket_accept_waits_for_process_creation_barrier() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let client = tokio::net::TcpStream::connect(address).await.unwrap(); + let mut listener = LimitedTcpListener { + inner: listener, + permits: Arc::new(Semaphore::new(1)), + }; + let (release, holder) = hold_process_creation_barrier(); + + let accept = tokio::spawn(async move { listener.accept().await }); + tokio::time::sleep(Duration::from_millis(250)).await; + let completed_while_barrier_held = accept.is_finished(); + release.send(()).unwrap(); + holder.join().unwrap(); + + let (server, _) = tokio::time::timeout(Duration::from_secs(5), accept) + .await + .expect("direct socket accept did not resume after the process barrier was released") + .unwrap(); + drop(server); + drop(client); + assert!( + !completed_while_barrier_held, + "direct socket accepted while a concurrent process could inherit its descriptor" + ); + } + struct PreludeProbeLink { reads: Arc, } From 9e3f556836ebb2965858de9c25d9774cf2340988 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:28:47 -0700 Subject: [PATCH 314/521] fix(tui): fence remote socket creation --- cmux-tui/Cargo.lock | 1 + cmux-tui/crates/cmux-remote/src/admin.rs | 12 +- cmux-tui/crates/cmux-remote/src/bridge.rs | 6 +- cmux-tui/crates/cmux-remote/src/daemon.rs | 23 +-- cmux-tui/crates/cmux-remote/src/http.rs | 2 +- .../crates/cmux-remote/src/provider/unix.rs | 6 +- .../cmux-remote/src/provider/websocket.rs | 18 +- cmux-tui/crates/cmux-remote/src/services.rs | 2 +- .../crates/cmux-remote/src/workspace/route.rs | 11 +- cmux-tui/crates/cmux-tui-process/Cargo.toml | 3 + cmux-tui/crates/cmux-tui-process/src/lib.rs | 171 ++++++++++++++++++ cmux-tui/crates/cmux-tui/src/remote_cli.rs | 24 +-- .../crates/cmux-tui/src/remote_runtime.rs | 4 +- 13 files changed, 234 insertions(+), 49 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 6ea1fde28e57..803cd62702c3 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -742,6 +742,7 @@ version = "0.1.0" dependencies = [ "libc", "socket2 0.5.10", + "tokio", ] [[package]] diff --git a/cmux-tui/crates/cmux-remote/src/admin.rs b/cmux-tui/crates/cmux-remote/src/admin.rs index 94dea3db3ea7..e73f586513dc 100644 --- a/cmux-tui/crates/cmux-remote/src/admin.rs +++ b/cmux-tui/crates/cmux-remote/src/admin.rs @@ -14,7 +14,7 @@ use cmux_remote_protocol::SessionId; use serde::{Deserialize, Serialize}; use serde_json::Value; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::net::{UnixListener, UnixStream}; +use tokio::net::UnixStream; use tokio::sync::{oneshot, watch}; use crate::daemon::RemoteDaemon; @@ -178,7 +178,7 @@ pub async fn serve_admin_with_shutdown( ) -> Result { let path = path.into(); prepare_socket_path(&path).await?; - let listener = UnixListener::bind(&path)?; + let listener = cmux_tui_process::tokio_net::bind_unix_listener(&path)?; std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?; let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); let task_path = path.clone(); @@ -186,7 +186,7 @@ pub async fn serve_admin_with_shutdown( loop { tokio::select! { _ = &mut shutdown_rx => break, - accepted = listener.accept() => { + accepted = cmux_tui_process::tokio_net::accept_unix_stream(&listener) => { let Ok((stream, _)) = accepted else { break }; let daemon = daemon.clone(); let default_route_hints = default_route_hints.clone(); @@ -214,7 +214,7 @@ pub async fn call_admin( path: impl AsRef, request: &AdminRequest, ) -> Result { - let stream = UnixStream::connect(path).await?; + let stream = cmux_tui_process::tokio_net::connect_unix_stream(path).await?; verify_unix_peer_owner(&stream)?; call_admin_over_stream(stream, request).await } @@ -225,7 +225,7 @@ async fn call_admin_with_expected_uid( request: &AdminRequest, expected_uid: u32, ) -> Result { - let stream = UnixStream::connect(path).await?; + let stream = cmux_tui_process::tokio_net::connect_unix_stream(path).await?; verify_unix_peer_uid(&stream, expected_uid)?; call_admin_over_stream(stream, request).await } @@ -390,7 +390,7 @@ async fn prepare_socket_path(path: &Path) -> Result<(), AdminError> { path.display() ))); } - if UnixStream::connect(path).await.is_ok() { + if cmux_tui_process::tokio_net::connect_unix_stream(path).await.is_ok() { return Err(AdminError::Protocol(format!( "another daemon owns admin socket {}", path.display() diff --git a/cmux-tui/crates/cmux-remote/src/bridge.rs b/cmux-tui/crates/cmux-remote/src/bridge.rs index c54480bfa131..aefa45e2637a 100644 --- a/cmux-tui/crates/cmux-remote/src/bridge.rs +++ b/cmux-tui/crates/cmux-remote/src/bridge.rs @@ -41,7 +41,7 @@ impl LocalPortForward { if maximum_connections == 0 { return Err(BridgeError::InvalidConnectionLimit); } - let listener = tokio::net::TcpListener::bind(address).await?; + let listener = cmux_tui_process::tokio_net::bind_tcp_listener(address)?; let local_addr = listener.local_addr()?; let permits = Arc::new(Semaphore::new(maximum_connections)); let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); @@ -58,7 +58,7 @@ impl LocalPortForward { }; let accepted = tokio::select! { _ = &mut shutdown_rx => break, - accepted = listener.accept() => accepted, + accepted = cmux_tui_process::tokio_net::accept_tcp_stream(&listener) => accepted, }; let Ok((socket, _)) = accepted else { break }; if configure_forward_socket(&socket).is_err() { @@ -132,7 +132,7 @@ pub async fn serve_mux_bridge( completed = connections.join_next(), if !connections.is_empty() => { let _ = completed; } - accepted = listener.accept() => { + accepted = cmux_tui_process::tokio_net::accept_unix_stream(&listener) => { let Ok((socket, _)) = accepted else { break }; let multiplexer = multiplexer.clone(); connections.spawn(async move { diff --git a/cmux-tui/crates/cmux-remote/src/daemon.rs b/cmux-tui/crates/cmux-remote/src/daemon.rs index e36898334bb6..4440bfacc1c1 100644 --- a/cmux-tui/crates/cmux-remote/src/daemon.rs +++ b/cmux-tui/crates/cmux-remote/src/daemon.rs @@ -21,8 +21,6 @@ use axum::serve::{IncomingStream, Listener}; use bytes::Bytes; use cmux_remote_protocol::{FrameFlags, Lane, SessionId}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -#[cfg(unix)] -use tokio::net::UnixListener; use tokio::sync::{Mutex, Notify, OwnedSemaphorePermit, RwLock, Semaphore, mpsc, oneshot, watch}; use crate::connection::{ConnectionError, LinkRejection, send_link_ready, send_link_rejection}; @@ -1137,7 +1135,7 @@ impl Listener for LimitedTcpListener { .acquire_owned() .await .expect("direct WebSocket admission semaphore is never closed"); - match self.inner.accept().await { + match cmux_tui_process::tokio_net::accept_tcp_stream(&self.inner).await { Ok((inner, address)) => { let _ = inner.set_nodelay(true); return ( @@ -1249,8 +1247,7 @@ pub async fn serve_direct_websocket( "refusing plaintext remote WebSocket bind {address}; use TLS or explicitly allow it" ))); } - let listener = tokio::net::TcpListener::bind(address) - .await + let listener = cmux_tui_process::tokio_net::bind_tcp_listener(address) .map_err(|error| DaemonError::Protocol(format!("could not bind WebSocket: {error}")))?; let local_addr = listener.local_addr().map_err(|error| { DaemonError::Protocol(format!("could not read WebSocket address: {error}")) @@ -1362,7 +1359,7 @@ pub async fn serve_unix( path.display() ))); } - if tokio::net::UnixStream::connect(&path).await.is_ok() { + if cmux_tui_process::tokio_net::connect_unix_stream(&path).await.is_ok() { return Err(DaemonError::Protocol(format!( "another daemon is listening at {}", path.display() @@ -1372,7 +1369,7 @@ pub async fn serve_unix( DaemonError::Protocol(format!("could not remove stale socket: {error}")) })?; } - let listener = UnixListener::bind(&path) + let listener = cmux_tui_process::tokio_net::bind_unix_listener(&path) .map_err(|error| DaemonError::Protocol(format!("could not bind Unix socket: {error}")))?; std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)) .map_err(|error| DaemonError::Protocol(format!("could not secure Unix socket: {error}")))?; @@ -1382,7 +1379,7 @@ pub async fn serve_unix( loop { tokio::select! { _ = &mut shutdown_rx => break, - accepted = listener.accept() => { + accepted = cmux_tui_process::tokio_net::accept_unix_stream(&listener) => { let Ok((stream, _)) = accepted else { break }; let Ok(peer) = stream.peer_cred() else { continue }; let owner = unsafe { libc::geteuid() }; @@ -1496,8 +1493,8 @@ mod tests { }; #[cfg(target_os = "macos")] - fn hold_process_creation_barrier() - -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) { + fn hold_process_creation_barrier() -> (std::sync::mpsc::Sender<()>, std::thread::JoinHandle<()>) + { let (held_sender, held_receiver) = std::sync::mpsc::channel(); let (release_sender, release_receiver) = std::sync::mpsc::channel(); let holder = std::thread::spawn(move || { @@ -1548,10 +1545,8 @@ mod tests { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let client = tokio::net::TcpStream::connect(address).await.unwrap(); - let mut listener = LimitedTcpListener { - inner: listener, - permits: Arc::new(Semaphore::new(1)), - }; + let mut listener = + LimitedTcpListener { inner: listener, permits: Arc::new(Semaphore::new(1)) }; let (release, holder) = hold_process_creation_barrier(); let accept = tokio::spawn(async move { listener.accept().await }); diff --git a/cmux-tui/crates/cmux-remote/src/http.rs b/cmux-tui/crates/cmux-remote/src/http.rs index 06bcdc168fda..9170ddcde228 100644 --- a/cmux-tui/crates/cmux-remote/src/http.rs +++ b/cmux-tui/crates/cmux-remote/src/http.rs @@ -236,7 +236,7 @@ pub async fn serve_workspace_http( } let token_file = token_file.into(); let token = load_or_create_workspace_http_token(&token_file)?; - let listener = tokio::net::TcpListener::bind(address).await?; + let listener = cmux_tui_process::tokio_net::bind_tcp_listener(address)?; let local_addr = listener.local_addr()?; let router = workspace_http_router(workspace, token); let (shutdown_tx, shutdown_rx) = oneshot::channel(); diff --git a/cmux-tui/crates/cmux-remote/src/provider/unix.rs b/cmux-tui/crates/cmux-remote/src/provider/unix.rs index cd6b9308464e..3e286c630bf3 100644 --- a/cmux-tui/crates/cmux-remote/src/provider/unix.rs +++ b/cmux-tui/crates/cmux-remote/src/provider/unix.rs @@ -2,9 +2,6 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use async_trait::async_trait; -use tokio::net::UnixStream; - use crate::admin::verify_unix_peer_owner; #[cfg(test)] use crate::admin::verify_unix_peer_uid; @@ -14,6 +11,7 @@ use crate::provider::{ CarrierEvidence, ConnectRequest, LengthDelimitedLink, LinkGroup, LinkRequest, ProviderCapabilities, ProviderError, SupportedClientAuthModes, TransportProvider, }; +use async_trait::async_trait; #[derive(Debug, Clone)] pub struct UnixProvider { @@ -107,7 +105,7 @@ impl LinkGroup for UnixLinkGroup { if self.closed.load(Ordering::Acquire) { return Err(ProviderError::Link(LinkError::Closed)); } - let stream = UnixStream::connect(&self.path) + let stream = cmux_tui_process::tokio_net::connect_unix_stream(&self.path) .await .map_err(|error| ProviderError::Link(LinkError::Transport(error.to_string())))?; #[cfg(test)] diff --git a/cmux-tui/crates/cmux-remote/src/provider/websocket.rs b/cmux-tui/crates/cmux-remote/src/provider/websocket.rs index 1d82c266b14c..7549e25c9177 100644 --- a/cmux-tui/crates/cmux-remote/src/provider/websocket.rs +++ b/cmux-tui/crates/cmux-remote/src/provider/websocket.rs @@ -11,7 +11,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tokio::sync::Mutex; use tokio_tungstenite::tungstenite::Message as TungsteniteMessage; use tokio_tungstenite::tungstenite::protocol::WebSocketConfig; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async_with_config}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, client_async_tls_with_config}; use url::Url; use crate::link::{FrameLink, LinkError}; @@ -121,7 +121,21 @@ pub async fn connect_websocket( let description = sanitized_route(endpoint); let config = WebSocketConfig::default().max_message_size(Some(maximum)).max_frame_size(Some(maximum)); - let (socket, _) = connect_async_with_config(endpoint.as_str(), Some(config), true) + let host = endpoint + .host_str() + .ok_or_else(|| LinkError::Transport("WebSocket endpoint has no host".into()))?; + let port = endpoint + .port_or_known_default() + .ok_or_else(|| LinkError::Transport("WebSocket endpoint has no port".into()))?; + let addresses = tokio::net::lookup_host((host, port)) + .await + .map_err(|error| LinkError::Transport(error.to_string()))? + .collect::>(); + let stream = cmux_tui_process::tokio_net::connect_tcp_stream(&addresses) + .await + .map_err(|error| LinkError::Transport(error.to_string()))?; + stream.set_nodelay(true).map_err(|error| LinkError::Transport(error.to_string()))?; + let (socket, _) = client_async_tls_with_config(endpoint.as_str(), stream, Some(config), None) .await .map_err(|error| LinkError::Transport(error.to_string()))?; Ok(TungsteniteWebSocketLink::new(description, maximum, socket)) diff --git a/cmux-tui/crates/cmux-remote/src/services.rs b/cmux-tui/crates/cmux-remote/src/services.rs index 06334da49f58..afbc3cf5f497 100644 --- a/cmux-tui/crates/cmux-remote/src/services.rs +++ b/cmux-tui/crates/cmux-remote/src/services.rs @@ -601,7 +601,7 @@ impl DaemonServices { let path = mux_socket.as_ref().ok_or_else(|| { ServicesError::Unavailable("mux control socket is not configured".into()) })?; - let socket = tokio::net::UnixStream::connect(path).await?; + let socket = cmux_tui_process::tokio_net::connect_unix_stream(path).await?; let stream = Arc::new(stream); send_opened(&stream, Lane::Interactive).await?; let (reader, writer) = socket.into_split(); diff --git a/cmux-tui/crates/cmux-remote/src/workspace/route.rs b/cmux-tui/crates/cmux-remote/src/workspace/route.rs index 26e5ffdf5d68..6483182e68da 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/route.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/route.rs @@ -92,10 +92,13 @@ impl RouteManager { .cloned() .ok_or_else(|| RpcError::new("unknown-route", format!("unknown route {}", id.0)))?; let addresses = resolve_allowed(&route.host, route.port, route.policy).await?; - let stream = tokio::time::timeout(DIAL_TIMEOUT, TcpStream::connect(addresses.as_slice())) - .await - .map_err(|_| RpcError::new("route-unavailable", "connection timed out"))? - .map_err(|error| RpcError::new("route-unavailable", error.to_string()))?; + let stream = tokio::time::timeout( + DIAL_TIMEOUT, + cmux_tui_process::tokio_net::connect_tcp_stream(addresses.as_slice()), + ) + .await + .map_err(|_| RpcError::new("route-unavailable", "connection timed out"))? + .map_err(|error| RpcError::new("route-unavailable", error.to_string()))?; stream .set_nodelay(true) .map_err(|error| RpcError::new("route-error", error.to_string()))?; diff --git a/cmux-tui/crates/cmux-tui-process/Cargo.toml b/cmux-tui/crates/cmux-tui-process/Cargo.toml index 56f3269b8f6d..a6662ba7bc3e 100644 --- a/cmux-tui/crates/cmux-tui-process/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-process/Cargo.toml @@ -10,6 +10,9 @@ description = "Process creation coordination for cmux-tui" [lints] workspace = true +[dependencies] +tokio.workspace = true + [target.'cfg(target_os = "macos")'.dependencies] libc.workspace = true socket2 = { workspace = true, features = ["all"] } diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index 7bd043723681..5a0529714434 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -320,3 +320,174 @@ pub mod tcp { ) } } + +/// Tokio socket operations coordinated with process launch on macOS. +/// +/// Tokio and Mio cannot create or accept every macOS socket with +/// close-on-exec set atomically. These helpers hold the same process-wide +/// guard used by child launch only around descriptor creation and registration. +pub mod tokio_net { + use std::io; + use std::net::SocketAddr; + + use tokio::net::{TcpListener, TcpSocket, TcpStream}; + + use super::ProcessCreationGuard; + + /// Bind and register a Tokio TCP listener without an inheritance window. + pub fn bind_tcp_listener(address: SocketAddr) -> io::Result { + let listener = super::tcp::bind_listener(address)?; + listener.set_nonblocking(true)?; + TcpListener::from_std(listener) + } + + /// Connect to the first reachable TCP address without an inheritance window. + pub async fn connect_tcp_stream(addresses: &[SocketAddr]) -> io::Result { + let mut last_error = None; + for &address in addresses { + let socket = { + let _guard = ProcessCreationGuard::acquire(); + if address.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() } + }?; + match socket.connect(address).await { + Ok(stream) => return Ok(stream), + Err(error) => last_error = Some(error), + } + } + Err(last_error.unwrap_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "TCP address list is empty") + })) + } + + /// Accept a Tokio TCP stream without an inheritance window. + pub async fn accept_tcp_stream(listener: &TcpListener) -> io::Result<(TcpStream, SocketAddr)> { + #[cfg(target_os = "macos")] + { + std::future::poll_fn(|context| { + let _guard = ProcessCreationGuard::acquire(); + listener.poll_accept(context) + }) + .await + } + #[cfg(not(target_os = "macos"))] + { + listener.accept().await + } + } + + #[cfg(unix)] + use std::path::Path; + #[cfg(unix)] + use tokio::net::{UnixListener, UnixStream}; + + /// Bind and register a Tokio Unix listener without an inheritance window. + #[cfg(unix)] + pub fn bind_unix_listener(path: impl AsRef) -> io::Result { + let listener = super::unix::bind_listener(path)?; + listener.set_nonblocking(true)?; + UnixListener::from_std(listener) + } + + /// Connect a Tokio Unix stream without an inheritance window. + #[cfg(unix)] + pub async fn connect_unix_stream(path: impl AsRef) -> io::Result { + #[cfg(target_os = "macos")] + { + let address = socket2::SockAddr::unix(path)?; + let (stream, pending) = { + let _guard = ProcessCreationGuard::acquire(); + let socket = + socket2::Socket::new(socket2::Domain::UNIX, socket2::Type::STREAM, None)?; + socket.set_nonblocking(true)?; + let pending = match socket.connect(&address) { + Ok(()) => false, + Err(error) if connect_is_pending(&error) => true, + Err(error) => return Err(error), + }; + let stream: std::os::unix::net::UnixStream = socket.into(); + (stream, pending) + }; + let stream = UnixStream::from_std(stream)?; + if pending { + stream.writable().await?; + if let Some(error) = stream.take_error()? { + return Err(error); + } + } + Ok(stream) + } + #[cfg(not(target_os = "macos"))] + { + UnixStream::connect(path).await + } + } + + /// Accept a Tokio Unix stream without an inheritance window. + #[cfg(unix)] + pub async fn accept_unix_stream( + listener: &UnixListener, + ) -> io::Result<(UnixStream, tokio::net::unix::SocketAddr)> { + #[cfg(target_os = "macos")] + { + std::future::poll_fn(|context| { + let _guard = ProcessCreationGuard::acquire(); + listener.poll_accept(context) + }) + .await + } + #[cfg(not(target_os = "macos"))] + { + listener.accept().await + } + } + + #[cfg(target_os = "macos")] + fn connect_is_pending(error: &io::Error) -> bool { + error.kind() == io::ErrorKind::WouldBlock + || matches!( + error.raw_os_error(), + Some(libc::EINPROGRESS | libc::EALREADY | libc::EWOULDBLOCK) + ) + } +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use std::time::Duration; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn tokio_tcp_connect_waits_for_process_creation_barrier() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let (held_sender, held_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let _barrier = super::ProcessCreationGuard::acquire(); + held_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + }); + held_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("process creation barrier was not acquired"); + + let connect = + tokio::spawn(async move { super::tokio_net::connect_tcp_stream(&[address]).await }); + tokio::time::sleep(Duration::from_millis(250)).await; + let completed_while_barrier_held = connect.is_finished(); + release_sender.send(()).unwrap(); + holder.join().unwrap(); + + let stream = tokio::time::timeout(Duration::from_secs(5), connect) + .await + .expect("TCP connect did not resume after the process barrier was released") + .unwrap() + .unwrap(); + let (peer, _) = listener.accept().unwrap(); + drop(peer); + drop(stream); + assert!( + !completed_while_barrier_held, + "TCP connect created a socket while a concurrent process could inherit it" + ); + } +} diff --git a/cmux-tui/crates/cmux-tui/src/remote_cli.rs b/cmux-tui/crates/cmux-tui/src/remote_cli.rs index 30866fc6b262..3021d162173c 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_cli.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_cli.rs @@ -1711,8 +1711,8 @@ fn run_remote_stop(args: &[String]) -> anyhow::Result<()> { let runtime = match load_runtime_info(&parsed.session, parsed.state_dir.as_deref()) { Ok(runtime) => runtime, Err(_) - if UnixStream::connect(&default_link).is_err() - && UnixStream::connect(&default_admin).is_err() => + if cmux_tui_process::unix::connect_stream(&default_link).is_err() + && cmux_tui_process::unix::connect_stream(&default_admin).is_err() => { return Ok(()); } @@ -1736,7 +1736,7 @@ fn run_remote_stop(args: &[String]) -> anyhow::Result<()> { } let deadline = Instant::now() + Duration::from_secs(20); while Instant::now() < deadline { - if UnixStream::connect(&link).is_err() && !runtime_file.exists() { + if cmux_tui_process::unix::connect_stream(&link).is_err() && !runtime_file.exists() { return Ok(()); } thread::sleep(Duration::from_millis(50)); @@ -1823,7 +1823,7 @@ fn ensure_daemon( mux_socket_override: Option<&Path>, ) -> anyhow::Result<()> { let _lock = lock_daemon_start(session_state)?; - if UnixStream::connect(link).is_ok() { + if cmux_tui_process::unix::connect_stream(link).is_ok() { return Ok(()); } @@ -1833,7 +1833,7 @@ fn ensure_daemon( .map(Path::to_path_buf) .or_else(|| std::env::var_os("CMUX_MUX_SOCKET").map(PathBuf::from)) .unwrap_or_else(|| cmux_tui_core::server::default_socket_path(session)); - if UnixStream::connect(&mux_socket).is_err() { + if cmux_tui_process::unix::connect_stream(&mux_socket).is_err() { let log = OpenOptions::new().create(true).append(true).open(&log_path)?; let mut mux_owner = Command::new(&executable); mux_owner @@ -1847,7 +1847,7 @@ fn ensure_daemon( cmux_tui_process::spawn(&mut mux_owner).context("could not start remote mux owner")?; let deadline = Instant::now() + Duration::from_secs(20); while Instant::now() < deadline { - if UnixStream::connect(&mux_socket).is_ok() { + if cmux_tui_process::unix::connect_stream(&mux_socket).is_ok() { break; } if let Some(status) = child.try_wait()? { @@ -1858,7 +1858,7 @@ fn ensure_daemon( } thread::sleep(Duration::from_millis(50)); } - if UnixStream::connect(&mux_socket).is_err() { + if cmux_tui_process::unix::connect_stream(&mux_socket).is_err() { return Err(anyhow!("remote mux owner did not create {}", mux_socket.display())); } } @@ -1880,7 +1880,7 @@ fn ensure_daemon( cmux_tui_process::spawn(&mut command).context("could not start remote daemon")?; let deadline = Instant::now() + Duration::from_secs(20); while Instant::now() < deadline { - if UnixStream::connect(link).is_ok() { + if cmux_tui_process::unix::connect_stream(link).is_ok() { return Ok(()); } if let Some(status) = child.try_wait()? { @@ -1905,7 +1905,7 @@ fn configure_detached_process(command: &mut Command) { } fn open_mux_monitor(path: &Path) -> anyhow::Result { - let stream = UnixStream::connect(path).with_context(|| { + let stream = cmux_tui_process::unix::connect_stream(path).with_context(|| { format!("cannot attach remote sidecar to mux socket {}", path.display()) })?; stream.set_read_timeout(Some(Duration::from_millis(250)))?; @@ -1968,7 +1968,7 @@ fn cleanup_stale_sidecar_artifacts( ) -> anyhow::Result<()> { // A replacement started outside the bootstrap lock owns these paths. Do // not unlink its sockets or metadata. - if UnixStream::connect(link_socket).is_ok() { + if cmux_tui_process::unix::connect_stream(link_socket).is_ok() { return Ok(()); } remove_regular_file_if_present(&state_dir.join("runtime.json"))?; @@ -1991,7 +1991,7 @@ fn remove_stale_socket_if_present(path: &Path) -> anyhow::Result<()> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_socket() => { - if UnixStream::connect(path).is_err() { + if cmux_tui_process::unix::connect_stream(path).is_err() { fs::remove_file(path)?; } } @@ -2032,7 +2032,7 @@ where { use tokio::io::{AsyncWriteExt, copy}; - let stream = tokio::net::UnixStream::connect(link).await?; + let stream = cmux_tui_process::tokio_net::connect_unix_stream(link).await?; verify_peer(&stream)?; let (mut socket_read, mut socket_write) = stream.into_split(); let upload = async { diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index 74bb2a9c817a..d5739377327c 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -486,7 +486,7 @@ async fn run_client( prepare_client_socket(&local_socket).await?; let daemon_public_key = connection.daemon_public_key(); let multiplexer = ServiceMultiplexer::new(connection.clone(), EndpointRole::Client); - let listener = tokio::net::UnixListener::bind(&local_socket)?; + let listener = cmux_tui_process::tokio_net::bind_unix_listener(&local_socket)?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -1188,7 +1188,7 @@ async fn prepare_client_socket(path: &Path) -> anyhow::Result<()> { path.display() )); } - if tokio::net::UnixStream::connect(path).await.is_ok() { + if cmux_tui_process::tokio_net::connect_unix_stream(path).await.is_ok() { return Err(anyhow!("another client owns {}", path.display())); } fs::remove_file(path)?; From 98c93947ae83ad69b043019604ccdea2bf948993 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 12:52:34 -0700 Subject: [PATCH 315/521] test(tui): expose fail-open host cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 31bd732661d1..271d40de3c43 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -17356,6 +17356,52 @@ mod tests { let _ = mux.shutdown(); } + #[cfg(unix)] + #[test] + fn terminal_close_reports_exact_host_record_read_failure() { + let mux = test_mux(); + let surface = mux.new_workspace(Some("exact-host-close".into()), Some((80, 24))).unwrap(); + let identity = mux.resource_terminal_host_identity(&surface).unwrap(); + let root = std::env::temp_dir().join(format!( + "cmux-terminal-close-host-record-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join(format!("{}.json", identity.terminal_id)), b"{invalid-json") + .unwrap(); + mux.surface_options.lock().unwrap().terminal_host_root = Some(root.clone()); + + let result = mux.close_terminal(&identity.terminal_id, &identity.incarnation); + let _ = std::fs::remove_dir_all(root); + + let error = result.expect_err("close acknowledged an unreadable exact host record"); + assert!(format!("{error:#}").contains("terminal-host")); + } + + #[cfg(unix)] + #[test] + fn workspace_close_reports_exact_host_record_read_failure() { + let mux = test_mux(); + let surface = + mux.new_workspace(Some("exact-workspace-close".into()), Some((80, 24))).unwrap(); + let identity = mux.resource_terminal_host_identity(&surface).unwrap(); + let workspace = mux.with_state(|state| state.workspaces[0].id); + let root = std::env::temp_dir().join(format!( + "cmux-workspace-close-host-record-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(root.join(format!("{}.json", identity.terminal_id)), b"{invalid-json") + .unwrap(); + mux.surface_options.lock().unwrap().terminal_host_root = Some(root.clone()); + + let result = mux.close_workspace_at_revision(workspace, None); + let _ = std::fs::remove_dir_all(root); + + let error = result.expect_err("workspace close acknowledged an unreadable host record"); + assert!(format!("{error:#}").contains("terminal-host")); + } + #[test] fn ordinary_topology_projection_failure_keeps_memory_and_public_state_unchanged() { let mux = test_mux(); From abc6bf25c90a751b0a8ea2b5d307656ea5c6abae Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:04:12 -0700 Subject: [PATCH 316/521] fix(tui): target terminal host cleanup exactly --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 105 +++++++++--------- .../src/mux/resource_topology.rs | 17 ++- .../src/terminal_host_runtime.rs | 43 +++++++ .../cmux-tui-core/src/workspace_registry.rs | 18 +++ 4 files changed, 126 insertions(+), 57 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 271d40de3c43..cb0ae324a912 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -7540,8 +7540,10 @@ impl Mux { self.emit(MuxEvent::LayoutChanged(screen)); } } - self.terminate_discovered_terminal_host(terminal_id, terminal_incarnation.as_deref()); + let termination = + self.terminate_discovered_terminal_host(terminal_id, terminal_incarnation.as_deref()); self.emit_empty_if_current(empty_revision); + termination?; Ok(TerminalCloseResult { surface: target, terminal_id: terminal_id.to_string(), @@ -7615,16 +7617,23 @@ impl Mux { Ok(()) } - fn terminate_discovered_terminal_host(&self, terminal_id: &str, incarnation: Option<&str>) { + fn terminate_discovered_terminal_host( + &self, + terminal_id: &str, + incarnation: Option<&str>, + ) -> anyhow::Result<()> { self.terminate_discovered_terminal_hosts(&[( terminal_id.to_string(), incarnation.map(str::to_string), - )]); + )]) } - fn terminate_discovered_terminal_hosts(&self, terminals: &[(String, Option)]) { + fn terminate_discovered_terminal_hosts( + &self, + terminals: &[(String, Option)], + ) -> anyhow::Result<()> { if terminals.is_empty() { - return; + return Ok(()); } #[cfg(test)] self.discovered_terminal_termination_requests @@ -7634,29 +7643,34 @@ impl Mux { #[cfg(unix)] { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - let Some(root) = root else { return }; - let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(&root) - else { - return; - }; - let targets = terminals - .iter() - .map(|(terminal_id, incarnation)| (terminal_id.as_str(), incarnation.as_deref())) - .collect::>(); - for (path, record) in records { - let Some(expected_incarnation) = targets.get(record.terminal_id.as_str()) else { + let Some(root) = root else { return Ok(()) }; + for (terminal_id, expected_incarnation) in terminals { + let Some((path, record)) = + crate::terminal_host_runtime::load_terminal_host_record(&root, terminal_id) + .with_context(|| { + format!("load exact terminal-host record for {terminal_id}") + })? + else { continue; }; - if match *expected_incarnation { - Some(expected) => record.incarnation == expected, - None => true, - } { - terminate_host_record(record, path); - } + anyhow::ensure!( + expected_incarnation + .as_deref() + .is_none_or(|expected| record.incarnation == expected), + "terminal-host incarnation changed while closing {terminal_id}" + ); + anyhow::ensure!( + cleanup_terminal_host_record(&record, &path), + "could not clean up terminal host {terminal_id}" + ); } + Ok(()) } #[cfg(not(unix))] - let _ = terminals; + { + let _ = terminals; + Ok(()) + } } #[cfg(test)] @@ -7666,37 +7680,21 @@ impl Mux { std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) } - fn terminate_tombstoned_workspace_hosts(&self, workspace_key: &str) { + fn terminate_tombstoned_workspace_hosts(&self, workspace_key: &str) -> anyhow::Result<()> { #[cfg(unix)] { - let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - let Some(root) = root else { return }; - let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(&root) - else { - return; - }; - for (path, record) in records { - let terminal = self - .workspace_registry - .lock() - .unwrap() - .terminal_record(&record.terminal_id) - .ok() - .flatten(); - if terminal.as_ref().is_some_and(|terminal| { - terminal.workspace_key == workspace_key - && terminal.lifecycle == TerminalLifecycle::Tombstoned - && terminal - .incarnation - .as_deref() - .is_none_or(|expected| expected == record.incarnation) - }) { - terminate_host_record(record, path); - } - } + let terminals = self + .workspace_registry + .lock() + .unwrap() + .tombstoned_terminal_host_identities_in_workspace(workspace_key)?; + self.terminate_discovered_terminal_hosts(&terminals) } #[cfg(not(unix))] - let _ = workspace_key; + { + let _ = workspace_key; + Ok(()) + } } /// Run `f` with the session state. @@ -12148,7 +12146,7 @@ impl Mux { let result = workspace_mutation_result(&commit)?; let key = result.key.clone(); drop(registry); - self.terminate_tombstoned_workspace_hosts(&key); + self.terminate_tombstoned_workspace_hosts(&key)?; return Ok(result); } } @@ -12210,7 +12208,7 @@ impl Mux { let result = workspace_mutation_result(&commit)?; let key = result.key.clone(); drop(registry); - self.terminate_tombstoned_workspace_hosts(&key); + self.terminate_tombstoned_workspace_hosts(&key)?; return Ok(result); } let terminal_revision_before = registry.terminal_snapshot()?.revision; @@ -12324,9 +12322,10 @@ impl Mux { self.purge_surface_side_tables(surface.id); } self.retire_surface_runtimes(removed); - self.terminate_tombstoned_workspace_hosts(&closed_workspace_key); + let host_termination = self.terminate_tombstoned_workspace_hosts(&closed_workspace_key); self.emit_tree_delta(delta, selection_resync); self.emit_empty_if_current(empty_revision); + host_termination?; Ok(result) } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 7bed5f469156..14cde5ad24e7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2087,21 +2087,30 @@ impl Mux { drop(registry); drop(workspace_lifecycle); - self.terminate_discovered_terminal_hosts(&plan.unmaterialized_terminals); + let unmaterialized_termination = if plan.closed_workspace_key.is_none() { + self.terminate_discovered_terminal_hosts(&plan.unmaterialized_terminals) + } else { + Ok(()) + }; self.notify_terminal_exit_waiters(closed_public_ids); self.publish_resource_event(); for surface in &plan.removed { self.purge_surface_side_tables(surface.id); } self.retire_surface_runtimes(plan.removed); - if let Some(workspace_key) = plan.closed_workspace_key { - self.terminate_tombstoned_workspace_hosts(&workspace_key); - } + let workspace_termination = + if let Some(workspace_key) = plan.closed_workspace_key.as_deref() { + self.terminate_tombstoned_workspace_hosts(workspace_key) + } else { + Ok(()) + }; self.emit_tree_delta(plan.delta, plan.selection_resync); for screen in plan.changed_screens { self.emit(MuxEvent::LayoutChanged(screen)); } self.emit_empty_if_current(empty_revision); + unmaterialized_termination?; + workspace_termination?; Ok(close.resource) } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 34c23f60c073..aa164d83e59d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2408,6 +2408,34 @@ mod unix { ) } + /// Load the one canonical discovery record for `terminal_id` without + /// scanning unrelated hosts. A missing record means there is no discovery + /// owner to terminate; malformed or unreadable records remain errors so a + /// close cannot silently abandon a live host. + pub(crate) fn load_terminal_host_record( + root: &Path, + terminal_id: &str, + ) -> anyhow::Result> { + let terminal = TerminalId::from_hex(terminal_id) + .ok_or_else(|| anyhow::anyhow!("terminal-host id is not a canonical UUIDv4"))?; + anyhow::ensure!(terminal.to_hex() == terminal_id, "terminal-host id is not canonical"); + let path = root.join(format!("{terminal_id}.json")); + match read_terminal_host_record_until( + &path, + Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT, + ) { + Ok(record) => Ok(Some((path, record))), + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.kind() == std::io::ErrorKind::NotFound) => + { + Ok(None) + } + Err(error) => Err(error), + } + } + /// Load every discovery record without omission. Adoption tolerates /// transient or stale files, but an atomic server shutdown must account /// for every record before it discards topology. @@ -6973,6 +7001,20 @@ mod unix { let _ = fs::remove_dir_all(root); } + #[test] + fn exact_record_loader_does_not_read_unrelated_host_files() { + let (record_path, record, lease) = record_fixture("exact-record"); + let root = record_path.parent().unwrap(); + fs::write(root.join("unrelated.json"), b"{invalid-json").unwrap(); + + let loaded = load_terminal_host_record(root, &record.terminal_id).unwrap(); + + assert_eq!(loaded, Some((record_path.clone(), record.clone()))); + drop(lease); + assert!(remove_stale_terminal_host_record(&record_path, &record).unwrap()); + let _ = fs::remove_dir_all(root); + } + #[test] fn private_directory_setup_rejects_symlink_without_changing_target_permissions() { let root = std::env::temp_dir().join(format!( @@ -8457,6 +8499,7 @@ mod unix { pub(crate) use unix::{ ControlResponses, DecodedHostResize, DeferredCellPixelResolution, adopt_terminal_host_with_kitty_limits, decode_host_resize_payload_for_version, + load_terminal_host_record, }; #[cfg(unix)] pub use unix::{ diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index c65b29b54f1c..1c2391b6b7af 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -864,6 +864,24 @@ impl WorkspaceRegistry { read_terminal(&self.connection, terminal_id) } + #[cfg(unix)] + pub(crate) fn tombstoned_terminal_host_identities_in_workspace( + &self, + workspace_key: &str, + ) -> anyhow::Result)>> { + validate_workspace_key(workspace_key)?; + let mut statement = self.connection.prepare( + "SELECT terminal_id, incarnation + FROM terminal_placements + WHERE workspace_key = ?1 AND lifecycle = 'tombstoned' + ORDER BY created_revision ASC, terminal_id ASC", + )?; + statement + .query_map([workspace_key], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::, _>>() + .map_err(Into::into) + } + #[cfg(test)] pub(crate) fn set_terminal_record_read_failure_for_test(&self, enabled: bool) { self.fail_terminal_record_reads.store(enabled, Ordering::Relaxed); From 5d251fdcf1637df9e397d3279d57e1a0a18832d5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:58:38 -0700 Subject: [PATCH 317/521] test(tui): expose historical host cleanup scan --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index cb0ae324a912..424379781ab8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -17401,6 +17401,40 @@ mod tests { assert!(format!("{error:#}").contains("terminal-host")); } + #[cfg(unix)] + #[test] + fn workspace_close_cleanup_is_driven_by_live_host_records() { + let mux = test_mux(); + let surface = + mux.new_workspace(Some("bounded-workspace-close".into()), Some((80, 24))).unwrap(); + let workspace = mux.with_state(|state| state.workspaces[0].id); + let pane = mux.with_state(|state| state.pane_of(surface.id).unwrap()); + for _ in 0..32 { + let historical = mux.new_tab(Some(pane), None, Some((80, 24))).unwrap(); + let identity = mux.resource_terminal_host_identity(&historical).unwrap(); + mux.close_terminal(&identity.terminal_id, &identity.incarnation).unwrap(); + } + let _ = mux.take_discovered_terminal_termination_requests_for_test(); + + let root = std::env::temp_dir().join(format!( + "cmux-workspace-close-live-records-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + mux.surface_options.lock().unwrap().terminal_host_root = Some(root.clone()); + + let result = mux.close_workspace_at_revision(workspace, None); + let requested = mux.take_discovered_terminal_termination_requests_for_test(); + let _ = std::fs::remove_dir_all(root); + + result.unwrap(); + assert!( + requested.is_empty(), + "workspace close probed {} historical tombstone(s) without live host records", + requested.len() + ); + } + #[test] fn ordinary_topology_projection_failure_keeps_memory_and_public_state_unchanged() { let mux = test_mux(); From 3211656a66147510f67f745434cad42926357371 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:05:58 -0700 Subject: [PATCH 318/521] fix(tui): bound workspace host cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 53 ++++++++++++++++--- .../src/terminal_host_runtime.rs | 16 +++++- .../cmux-tui-core/src/workspace_registry.rs | 18 ------- 3 files changed, 62 insertions(+), 25 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 424379781ab8..5a8e99f290dc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -7683,12 +7683,53 @@ impl Mux { fn terminate_tombstoned_workspace_hosts(&self, workspace_key: &str) -> anyhow::Result<()> { #[cfg(unix)] { - let terminals = self - .workspace_registry - .lock() - .unwrap() - .tombstoned_terminal_host_identities_in_workspace(workspace_key)?; - self.terminate_discovered_terminal_hosts(&terminals) + let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + let Some(root) = root else { return Ok(()) }; + let records = + crate::terminal_host_runtime::load_terminal_host_records_for_cleanup(&root) + .context("load bounded terminal-host records for workspace close")?; + let records = { + let registry = self.workspace_registry.lock().unwrap(); + let mut matching = Vec::new(); + for (path, record) in records { + let Some(terminal) = + registry.terminal_record(&record.terminal_id).with_context(|| { + format!("load terminal lifecycle for {}", record.terminal_id) + })? + else { + continue; + }; + if terminal.workspace_key != workspace_key + || terminal.lifecycle != TerminalLifecycle::Tombstoned + { + continue; + } + anyhow::ensure!( + terminal + .incarnation + .as_deref() + .is_some_and(|expected| expected == record.incarnation), + "terminal-host incarnation changed while closing {}", + record.terminal_id + ); + matching.push((path, record)); + } + matching + }; + #[cfg(test)] + self.discovered_terminal_termination_requests.lock().unwrap().extend( + records.iter().map(|(_, record)| { + (record.terminal_id.clone(), Some(record.incarnation.clone())) + }), + ); + for (path, record) in records { + anyhow::ensure!( + cleanup_terminal_host_record(&record, &path), + "could not clean up terminal host {}", + record.terminal_id + ); + } + Ok(()) } #[cfg(not(unix))] { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index aa164d83e59d..60823edf4649 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2436,6 +2436,20 @@ mod unix { } } + /// Strictly load the bounded active discovery set for a targeted cleanup. + /// Historical registry rows never expand this work set; malformed or + /// unsafe live records remain errors so cleanup cannot omit an owner. + pub(crate) fn load_terminal_host_records_for_cleanup( + root: &Path, + ) -> anyhow::Result> { + load_terminal_host_records_until( + root, + MAX_TERMINAL_HOST_RECORDS, + Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT, + TerminalHostRecordLoadMode::Strict, + ) + } + /// Load every discovery record without omission. Adoption tolerates /// transient or stale files, but an atomic server shutdown must account /// for every record before it discards topology. @@ -8499,7 +8513,7 @@ mod unix { pub(crate) use unix::{ ControlResponses, DecodedHostResize, DeferredCellPixelResolution, adopt_terminal_host_with_kitty_limits, decode_host_resize_payload_for_version, - load_terminal_host_record, + load_terminal_host_record, load_terminal_host_records_for_cleanup, }; #[cfg(unix)] pub use unix::{ diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 1c2391b6b7af..c65b29b54f1c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -864,24 +864,6 @@ impl WorkspaceRegistry { read_terminal(&self.connection, terminal_id) } - #[cfg(unix)] - pub(crate) fn tombstoned_terminal_host_identities_in_workspace( - &self, - workspace_key: &str, - ) -> anyhow::Result)>> { - validate_workspace_key(workspace_key)?; - let mut statement = self.connection.prepare( - "SELECT terminal_id, incarnation - FROM terminal_placements - WHERE workspace_key = ?1 AND lifecycle = 'tombstoned' - ORDER BY created_revision ASC, terminal_id ASC", - )?; - statement - .query_map([workspace_key], |row| Ok((row.get(0)?, row.get(1)?)))? - .collect::, _>>() - .map_err(Into::into) - } - #[cfg(test)] pub(crate) fn set_terminal_record_read_failure_for_test(&self, enabled: bool) { self.fail_terminal_record_reads.store(enabled, Ordering::Relaxed); From 452d79599ffc1733c31df3c9a0885754e2baa2ba Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:56:15 -0700 Subject: [PATCH 319/521] test(tui): expose incomplete cleanup fences --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 54 +++++++++++++++ cmux-tui/crates/cmux-tui/src/main.rs | 18 +++-- .../crates/cmux-tui/src/remote_runtime.rs | 20 +++++- cmux-tui/crates/cmux-tui/tests/cli.rs | 66 +++++++++++++++++++ 4 files changed, 150 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5a8e99f290dc..14bd8c9e2190 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -17436,10 +17436,64 @@ mod tests { mux.surface_options.lock().unwrap().terminal_host_root = Some(root.clone()); let result = mux.close_workspace_at_revision(workspace, None); + let workspace_remained = + mux.with_state(|state| state.workspaces.iter().any(|entry| entry.id == workspace)); + let terminal_lifecycle = mux + .workspace_registry + .lock() + .unwrap() + .terminal_record(&identity.terminal_id) + .unwrap() + .unwrap() + .lifecycle; let _ = std::fs::remove_dir_all(root); let error = result.expect_err("workspace close acknowledged an unreadable host record"); assert!(format!("{error:#}").contains("terminal-host")); + assert!(workspace_remained, "failed host validation committed the workspace close"); + assert_ne!( + terminal_lifecycle, + TerminalLifecycle::Tombstoned, + "failed host validation tombstoned the target terminal" + ); + } + + #[cfg(unix)] + #[test] + fn workspace_close_ignores_unrelated_malformed_host_record() { + let mux = test_mux(); + let target = mux.new_workspace(Some("target-workspace".into()), Some((80, 24))).unwrap(); + let unrelated = + mux.new_workspace(Some("unrelated-workspace".into()), Some((80, 24))).unwrap(); + let target_workspace = mux.with_state(|state| { + let pane = state.pane_of(target.id).unwrap(); + let (workspace, _) = state.screen_of(pane).unwrap(); + state.workspaces[workspace].id + }); + let unrelated_identity = mux.resource_terminal_host_identity(&unrelated).unwrap(); + let root = std::env::temp_dir().join(format!( + "cmux-workspace-close-unrelated-record-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let unrelated_path = root.join(format!("{}.json", unrelated_identity.terminal_id)); + std::fs::write(&unrelated_path, b"{invalid-json").unwrap(); + mux.surface_options.lock().unwrap().terminal_host_root = Some(root.clone()); + + let result = mux.close_workspace_at_revision(target_workspace, None); + let target_remained = mux + .with_state(|state| state.workspaces.iter().any(|entry| entry.id == target_workspace)); + let unrelated_remained = mux.with_state(|state| state.surfaces.contains_key(&unrelated.id)); + let unrelated_record_remained = unrelated_path.exists(); + let _ = std::fs::remove_dir_all(root); + + result.expect("an unrelated malformed host record blocked workspace close"); + assert!(!target_remained, "successful close retained its target workspace"); + assert!(unrelated_remained, "targeted close removed an unrelated terminal"); + assert!( + unrelated_record_remained, + "targeted close removed an unrelated malformed host record" + ); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 587282dc9349..1d2fc919810d 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1898,13 +1898,6 @@ fn run_server( eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } - #[cfg(debug_assertions)] - if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { - loop { - std::thread::park(); - } - } - #[cfg(unix)] let remote_runtime = if args.remote { let runtime = match remote_runtime::start_daemon_runtime( @@ -1941,11 +1934,22 @@ fn run_server( for route in &runtime.info().routes { eprintln!("cmux-tui: remote route {route}"); } + #[cfg(debug_assertions)] + if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER") { + std::fs::write(marker, b"started")?; + } Some(runtime) } else { None }; + #[cfg(debug_assertions)] + if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { + loop { + std::thread::park(); + } + } + let machine_runtime = (config.machine_sidebar.enabled || !config.machines.is_empty()) .then(|| MachineRuntime::new(socket_path.clone(), config.machines.clone())); let result = if args.headless { diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index d5739377327c..a7cf7b797dc9 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -172,11 +172,29 @@ impl DaemonRuntimeHandle { } pub fn shutdown(mut self) -> anyhow::Result<()> { + #[cfg(debug_assertions)] + let shutdown_marker = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER"); + #[cfg(debug_assertions)] + if let Some(marker) = shutdown_marker.as_ref() { + fs::write(marker, b"started")?; + } + #[cfg(debug_assertions)] + if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_REMOTE_SHUTDOWN_DELAY_MS") + .ok() + .and_then(|value| value.parse::().ok()) + { + thread::sleep(Duration::from_millis(milliseconds)); + } let _ = self.shutdown.send(true); - match self.thread.take().expect("daemon runtime thread is present").join() { + let result = match self.thread.take().expect("daemon runtime thread is present").join() { Ok(result) => result, Err(_) => Err(anyhow!("remote daemon runtime thread panicked")), + }; + #[cfg(debug_assertions)] + if let Some(marker) = shutdown_marker { + fs::write(marker, b"complete")?; } + result } } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 9b6cb29489da..57c822570118 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1871,6 +1871,72 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { fs::remove_dir_all(dir).unwrap(); } +#[cfg(unix)] +#[test] +fn forced_exit_waits_for_remote_runtime_cleanup() { + let dir = unique_temp_dir("remote-cleanup-fence"); + fs::create_dir_all(&dir).unwrap(); + let socket = dir.join("mux.sock"); + let state = dir.join("state"); + let remote_state = dir.join("remote-state"); + let remote_started = dir.join("remote-started"); + let remote_shutdown = dir.join("remote-shutdown"); + let mut server = PtyChild::start_with_env( + &[ + "--socket", + socket.to_str().unwrap(), + "--state", + state.to_str().unwrap(), + "--remote", + "--remote-state-dir", + remote_state.to_str().unwrap(), + ], + &[ + ("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER", std::ffi::OsStr::new("1")), + ("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS", std::ffi::OsStr::new("100")), + ("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER", remote_started.as_os_str()), + ("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER", remote_shutdown.as_os_str()), + ("CMUX_TUI_TEST_REMOTE_SHUTDOWN_DELAY_MS", std::ffi::OsStr::new("500")), + ], + ); + wait_for_socket_path(&socket); + let startup_deadline = Instant::now() + Duration::from_secs(10); + while !remote_started.exists() { + assert!( + server.child.try_wait().unwrap().is_none(), + "server exited before starting its remote runtime" + ); + assert!(Instant::now() < startup_deadline, "remote runtime did not start"); + std::thread::sleep(Duration::from_millis(10)); + } + + let response = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "shutdown"})); + assert_eq!(response, serde_json::json!({})); + std::thread::sleep(Duration::from_millis(250)); + assert_eq!( + fs::read(&remote_shutdown).ok().as_deref(), + Some(b"started".as_slice()), + "remote runtime cleanup did not enter the process completion fence" + ); + assert!( + server.child.try_wait().unwrap().is_none(), + "server forced exit before remote runtime cleanup completed" + ); + + let deadline = Instant::now() + Duration::from_secs(5); + let status = loop { + if let Some(status) = server.child.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "server remained alive after remote cleanup"); + std::thread::sleep(Duration::from_millis(10)); + }; + assert_eq!(fs::read(&remote_shutdown).unwrap(), b"complete"); + assert!(status.success(), "server exited with {status}"); + drop(server); + fs::remove_dir_all(dir).unwrap(); +} + #[cfg(unix)] #[test] fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_to_exit() { From e92ee8ba9d70990ec87aa69438350b0939878718 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:30:41 -0700 Subject: [PATCH 320/521] fix(tui): fence server and workspace cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 191 +++++++--- .../src/mux/resource_topology.rs | 28 +- .../cmux-tui-core/src/provider_management.rs | 117 ++++-- .../src/terminal_host_runtime.rs | 16 +- .../src/workspace_registry/resource_store.rs | 18 + cmux-tui/crates/cmux-tui/src/main.rs | 341 ++++++++++++++---- .../crates/cmux-tui/src/remote_runtime.rs | 4 +- 7 files changed, 553 insertions(+), 162 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 14bd8c9e2190..d57da8437293 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -72,6 +72,10 @@ type ShutdownAttemptHook = Arc; #[cfg(unix)] type TerminalHostRecords = Vec<(std::path::PathBuf, crate::terminal_host_runtime::TerminalHostRecord)>; +#[cfg(not(unix))] +type TerminalHostRecords = (); +const TERMINAL_HOST_CLEANUP_RECEIPT_FIELD: &str = "terminal_host_cleanup"; +const MAX_TERMINAL_HOST_CLEANUP_IDENTITIES: usize = 4_096; #[cfg(all(test, unix))] type TerminalAdoptionSurfaceFactory = Arc anyhow::Result> + Send + Sync>; @@ -7680,48 +7684,77 @@ impl Mux { std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) } - fn terminate_tombstoned_workspace_hosts(&self, workspace_key: &str) -> anyhow::Result<()> { + fn prepare_terminal_host_cleanup( + &self, + terminals: &[(String, Option)], + ) -> anyhow::Result { + let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + self.prepare_terminal_host_cleanup_at_root(root.as_deref(), terminals) + } + + fn prepare_terminal_host_cleanup_at_root( + &self, + root: Option<&Path>, + terminals: &[(String, Option)], + ) -> anyhow::Result { + anyhow::ensure!( + terminals.len() <= MAX_TERMINAL_HOST_CLEANUP_IDENTITIES, + "terminal-host cleanup target exceeds capacity" + ); + let mut seen = HashSet::with_capacity(terminals.len()); + for (terminal_id, incarnation) in terminals { + validate_terminal_hex(terminal_id, "terminal-host cleanup id is invalid")?; + anyhow::ensure!( + seen.insert(terminal_id.as_str()), + "terminal-host cleanup target contains a duplicate id" + ); + if let Some(incarnation) = incarnation { + validate_terminal_hex(incarnation, "terminal-host cleanup incarnation is invalid")?; + } + } #[cfg(unix)] { - let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - let Some(root) = root else { return Ok(()) }; - let records = - crate::terminal_host_runtime::load_terminal_host_records_for_cleanup(&root) - .context("load bounded terminal-host records for workspace close")?; - let records = { - let registry = self.workspace_registry.lock().unwrap(); - let mut matching = Vec::new(); - for (path, record) in records { - let Some(terminal) = - registry.terminal_record(&record.terminal_id).with_context(|| { - format!("load terminal lifecycle for {}", record.terminal_id) + let Some(root) = root else { return Ok(Vec::new()) }; + let mut records = Vec::new(); + for (terminal_id, expected_incarnation) in terminals { + let Some((path, record)) = + crate::terminal_host_runtime::load_terminal_host_record(root, terminal_id) + .with_context(|| { + format!("load exact terminal-host record for {terminal_id}") })? - else { - continue; - }; - if terminal.workspace_key != workspace_key - || terminal.lifecycle != TerminalLifecycle::Tombstoned - { - continue; - } - anyhow::ensure!( - terminal - .incarnation - .as_deref() - .is_some_and(|expected| expected == record.incarnation), - "terminal-host incarnation changed while closing {}", - record.terminal_id - ); - matching.push((path, record)); - } - matching - }; + else { + continue; + }; + anyhow::ensure!( + expected_incarnation + .as_deref() + .is_none_or(|expected| record.incarnation == expected), + "terminal-host incarnation changed while closing {terminal_id}" + ); + records.push((path, record)); + } #[cfg(test)] self.discovered_terminal_termination_requests.lock().unwrap().extend( records.iter().map(|(_, record)| { (record.terminal_id.clone(), Some(record.incarnation.clone())) }), ); + Ok(records) + } + #[cfg(not(unix))] + { + let _ = root; + let _ = terminals; + Ok(()) + } + } + + fn terminate_prepared_terminal_hosts( + &self, + records: TerminalHostRecords, + ) -> anyhow::Result<()> { + #[cfg(unix)] + { for (path, record) in records { anyhow::ensure!( cleanup_terminal_host_record(&record, &path), @@ -7733,11 +7766,17 @@ impl Mux { } #[cfg(not(unix))] { - let _ = workspace_key; + let _ = records; Ok(()) } } + fn terminate_workspace_hosts_from_receipt(&self, result: &Value) -> anyhow::Result<()> { + let identities = terminal_host_cleanup_identities(result)?; + let records = self.prepare_terminal_host_cleanup(&identities)?; + self.terminate_prepared_terminal_hosts(records) + } + /// Run `f` with the session state. /// /// The state lock is held for the duration of `f`; do not call back @@ -12185,9 +12224,8 @@ impl Mux { let registry = self.workspace_registry.lock().unwrap(); if let Some(commit) = registry.replay(mutation, &fingerprint)? { let result = workspace_mutation_result(&commit)?; - let key = result.key.clone(); drop(registry); - self.terminate_tombstoned_workspace_hosts(&key)?; + self.terminate_workspace_hosts_from_receipt(&commit.result)?; return Ok(result); } } @@ -12244,12 +12282,12 @@ impl Mux { project_resource: bool, ) -> anyhow::Result { let notifications = self.surface_notifications(); + let terminal_host_root = self.surface_options.lock().unwrap().terminal_host_root.clone(); let mut registry = self.workspace_registry.lock().unwrap(); if let Some(commit) = registry.replay(mutation, fingerprint)? { let result = workspace_mutation_result(&commit)?; - let key = result.key.clone(); drop(registry); - self.terminate_tombstoned_workspace_hosts(&key)?; + self.terminate_workspace_hosts_from_receipt(&commit.result)?; return Ok(result); } let terminal_revision_before = registry.terminal_snapshot()?.revision; @@ -12259,8 +12297,8 @@ impl Mux { empty_revision, selection_resync, result, - closed_workspace_key, closed_public_ids, + host_cleanup, ) = { let mut state = self.state.lock().unwrap(); Self::require_workspace_revision(&state, expected_revision)?; @@ -12272,6 +12310,11 @@ impl Mux { let previous_active = state.active_pane(); let key = state.workspaces[index].key.clone(); let closed_public_ids = registry.terminal_resource_ids_in_workspace(&key)?; + let host_identities = registry.terminal_host_identities_in_workspace(&key)?; + let host_cleanup = self.prepare_terminal_host_cleanup_at_root( + terminal_host_root.as_deref(), + &host_identities, + )?; let mut desired = self.registry_projection(&state); desired.remove(index); let desired_active_workspace = if state.active_workspace == index { @@ -12284,6 +12327,8 @@ impl Mux { "key": key, "index": index, "changed": true, + (TERMINAL_HOST_CLEANUP_RECEIPT_FIELD): + terminal_host_cleanup_receipt(&host_identities), }); let commit = if project_resource { registry.commit_with_active_workspace( @@ -12348,7 +12393,15 @@ impl Mux { let empty_revision = state.workspaces.is_empty().then_some(state.workspace_revision); let selection_resync = was_active && empty_revision.is_none(); let result = workspace_mutation_result(&commit)?; - (removed, delta, empty_revision, selection_resync, result, key, closed_public_ids) + ( + removed, + delta, + empty_revision, + selection_resync, + result, + closed_public_ids, + host_cleanup, + ) }; let terminal_revision_after = registry.terminal_snapshot()?.revision; if terminal_revision_after != terminal_revision_before { @@ -12363,7 +12416,7 @@ impl Mux { self.purge_surface_side_tables(surface.id); } self.retire_surface_runtimes(removed); - let host_termination = self.terminate_tombstoned_workspace_hosts(&closed_workspace_key); + let host_termination = self.terminate_prepared_terminal_hosts(host_cleanup); self.emit_tree_delta(delta, selection_resync); self.emit_empty_if_current(empty_revision); host_termination?; @@ -15829,6 +15882,62 @@ fn workspace_mutation_result(commit: &RegistryCommit) -> anyhow::Result)]) -> Value { + Value::Array( + terminals + .iter() + .map(|(terminal_id, incarnation)| { + serde_json::json!({ + "terminal_id": terminal_id, + "incarnation": incarnation, + }) + }) + .collect(), + ) +} + +fn terminal_host_cleanup_identities( + result: &Value, +) -> anyhow::Result)>> { + let Some(receipt) = result.get(TERMINAL_HOST_CLEANUP_RECEIPT_FIELD) else { + return Ok(Vec::new()); + }; + let entries = receipt + .as_array() + .ok_or_else(|| anyhow::anyhow!("stored terminal-host cleanup receipt is invalid"))?; + anyhow::ensure!( + entries.len() <= MAX_TERMINAL_HOST_CLEANUP_IDENTITIES, + "stored terminal-host cleanup receipt exceeds capacity" + ); + let mut seen = HashSet::with_capacity(entries.len()); + let mut identities = Vec::with_capacity(entries.len()); + for entry in entries { + let terminal_id = entry["terminal_id"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("stored terminal-host cleanup id is missing"))?; + validate_terminal_hex(terminal_id, "stored terminal-host cleanup id is invalid")?; + anyhow::ensure!( + seen.insert(terminal_id.to_string()), + "stored terminal-host cleanup receipt contains a duplicate id" + ); + let incarnation = match entry.get("incarnation") { + None | Some(Value::Null) => None, + Some(value) => { + let incarnation = value.as_str().ok_or_else(|| { + anyhow::anyhow!("stored terminal-host cleanup incarnation is invalid") + })?; + validate_terminal_hex( + incarnation, + "stored terminal-host cleanup incarnation is invalid", + )?; + Some(incarnation.to_string()) + } + }; + identities.push((terminal_id.to_string(), incarnation)); + } + Ok(identities) +} + fn close_surface_delta( state: &State, notifications: &HashMap, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 14cde5ad24e7..945230b57778 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2035,6 +2035,7 @@ impl Mux { fingerprint: &Value, resolve_slots: impl FnOnce(&State) -> anyhow::Result, ) -> anyhow::Result { + let terminal_host_root = self.surface_options.lock().unwrap().terminal_host_root.clone(); let lifecycle = self.workspace_lifecycle(workspace); let workspace_lifecycle = lifecycle.lock().unwrap(); let notifications = self.surface_notifications(); @@ -2047,6 +2048,22 @@ impl Mux { ); let mut plan = self.resource_close_plan_locked(operation, slots, ®istry, &state, ¬ifications)?; + let workspace_host_identities = + if let Some(workspace_key) = plan.closed_workspace_key.as_deref() { + registry.terminal_host_identities_in_workspace(workspace_key)? + } else { + Vec::new() + }; + if let Some(workspace_close) = plan.workspace_close.as_mut() { + workspace_close + .legacy_result + .as_object_mut() + .expect("workspace close result is an object") + .insert( + TERMINAL_HOST_CLEANUP_RECEIPT_FIELD.to_string(), + terminal_host_cleanup_receipt(&workspace_host_identities), + ); + } let projection = self.resource_effect_projection_locked(®istry, &mut plan.state, json!({}))?; let closed_public_ids = if let Some(workspace_key) = plan.closed_workspace_key.as_deref() { @@ -2054,6 +2071,10 @@ impl Mux { } else { Self::terminal_public_ids_for_hosted(®istry, &plan.terminals)? }; + let workspace_host_cleanup = self.prepare_terminal_host_cleanup_at_root( + terminal_host_root.as_deref(), + &workspace_host_identities, + )?; #[cfg(test)] if let Some(hook) = self.resource_projection_before_commit.lock().unwrap().clone() { hook(); @@ -2098,12 +2119,7 @@ impl Mux { self.purge_surface_side_tables(surface.id); } self.retire_surface_runtimes(plan.removed); - let workspace_termination = - if let Some(workspace_key) = plan.closed_workspace_key.as_deref() { - self.terminate_tombstoned_workspace_hosts(workspace_key) - } else { - Ok(()) - }; + let workspace_termination = self.terminate_prepared_terminal_hosts(workspace_host_cleanup); self.emit_tree_delta(plan.delta, plan.selection_resync); for screen in plan.changed_screens { self.emit(MuxEvent::LayoutChanged(screen)); diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index 522e53a7ed7d..2696126f2738 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -8,7 +8,9 @@ use std::io::{self, Read, Write}; #[cfg(target_os = "linux")] use std::path::Path; #[cfg(target_os = "linux")] -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +#[cfg(target_os = "linux")] +use std::sync::{Arc, Mutex}; #[cfg(target_os = "linux")] use std::time::Duration; @@ -216,40 +218,103 @@ fn peer_uid(stream: &std::os::unix::net::UnixStream) -> io::Result { Ok(credentials.uid) } -/// Serves the systemd-provided listener in a detached thread. Each peer is -/// credential-checked before any request bytes are read. +/// Owns the systemd-provided listener and its credential-checked peers. +#[cfg(target_os = "linux")] +pub struct ProviderManagementServer { + shutdown: Arc, + thread: Option>, +} + +#[cfg(target_os = "linux")] +impl Drop for ProviderManagementServer { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// Serve the systemd-provided listener until the returned owner is dropped. +/// Dropping the owner closes every active peer and joins all management +/// threads before returning. #[cfg(target_os = "linux")] pub fn serve( listener: std::os::unix::net::UnixListener, mux: Arc, -) -> io::Result> { +) -> io::Result { let result = unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) }; if result != 0 { return Err(io::Error::last_os_error()); } - std::thread::Builder::new().name("provider-management".into()).spawn(move || { - for connection in listener.incoming() { - let Ok(stream) = connection else { continue }; - let mux = mux.clone(); - let _ = std::thread::Builder::new().name("provider-management-peer".into()).spawn( - move || { - let Ok(uid) = peer_uid(&stream) else { return }; - if uid != 0 { - let _ = write_response( - &stream, - &Response::error("access_denied", "provider management requires root"), - ); - return; + listener.set_nonblocking(true)?; + let shutdown = Arc::new(AtomicBool::new(false)); + let thread_shutdown = shutdown.clone(); + let thread = + std::thread::Builder::new().name("provider-management".into()).spawn(move || { + let active = Arc::new(Mutex::new(std::collections::HashMap::new())); + let next_peer = AtomicU64::new(1); + let mut peers = Vec::new(); + while !thread_shutdown.load(Ordering::Acquire) { + let stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + continue; } - let _ = stream.set_read_timeout(Some(IO_TIMEOUT)); - let _ = stream.set_write_timeout(Some(IO_TIMEOUT)); - let Ok(bytes) = read_message(&stream) else { return }; - let response = handle_request(&mux, uid, &bytes.0); - let _ = write_response(&stream, &response); - }, - ); - } - }) + Err(_) => { + if thread_shutdown.load(Ordering::Acquire) { + break; + } + std::thread::sleep(Duration::from_millis(10)); + continue; + } + }; + let peer = next_peer.fetch_add(1, Ordering::Relaxed); + let tracked = match stream.try_clone() { + Ok(tracked) => tracked, + Err(_) => continue, + }; + active.lock().unwrap().insert(peer, tracked); + let mux = mux.clone(); + let peer_active = active.clone(); + match std::thread::Builder::new().name("provider-management-peer".into()).spawn( + move || { + let result = (|| { + let uid = peer_uid(&stream)?; + if uid != 0 { + return write_response( + &stream, + &Response::error( + "access_denied", + "provider management requires root", + ), + ); + } + stream.set_read_timeout(Some(IO_TIMEOUT))?; + stream.set_write_timeout(Some(IO_TIMEOUT))?; + let bytes = read_message(&stream)?; + let response = handle_request(&mux, uid, &bytes.0); + write_response(&stream, &response) + })(); + peer_active.lock().unwrap().remove(&peer); + let _ = result; + }, + ) { + Ok(handle) => peers.push(handle), + Err(_) => { + active.lock().unwrap().remove(&peer); + } + } + } + for stream in active.lock().unwrap().values() { + let _ = stream.shutdown(std::net::Shutdown::Both); + } + for peer in peers { + let _ = peer.join(); + } + })?; + Ok(ProviderManagementServer { shutdown, thread: Some(thread) }) } #[derive(Debug)] diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 60823edf4649..aa164d83e59d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2436,20 +2436,6 @@ mod unix { } } - /// Strictly load the bounded active discovery set for a targeted cleanup. - /// Historical registry rows never expand this work set; malformed or - /// unsafe live records remain errors so cleanup cannot omit an owner. - pub(crate) fn load_terminal_host_records_for_cleanup( - root: &Path, - ) -> anyhow::Result> { - load_terminal_host_records_until( - root, - MAX_TERMINAL_HOST_RECORDS, - Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT, - TerminalHostRecordLoadMode::Strict, - ) - } - /// Load every discovery record without omission. Adoption tolerates /// transient or stale files, but an atomic server shutdown must account /// for every record before it discards topology. @@ -8513,7 +8499,7 @@ mod unix { pub(crate) use unix::{ ControlResponses, DecodedHostResize, DeferredCellPixelResolution, adopt_terminal_host_with_kitty_limits, decode_host_resize_payload_for_version, - load_terminal_host_record, load_terminal_host_records_for_cleanup, + load_terminal_host_record, }; #[cfg(unix)] pub use unix::{ diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index b020b0258a64..de2b427cdb32 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -438,6 +438,24 @@ impl WorkspaceRegistry { .collect() } + pub(crate) fn terminal_host_identities_in_workspace( + &self, + workspace_key: &str, + ) -> anyhow::Result)>> { + validate_workspace_key(workspace_key)?; + let mut statement = self.connection.prepare( + "SELECT terminal_id, incarnation + FROM terminal_placements + WHERE workspace_key = ?1 + AND lifecycle != 'tombstoned' + ORDER BY created_revision ASC, terminal_id ASC", + )?; + statement + .query_map([workspace_key], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::, _>>() + .map_err(Into::into) + } + pub fn terminal_host_id(&self, public_id: &TerminalPublicId) -> anyhow::Result> { self.connection .query_row( diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 1d2fc919810d..d96ee57e38da 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1307,17 +1307,154 @@ fn socket_option(fd: std::os::fd::RawFd, option: libc::c_int) -> io::Result, + #[cfg(unix)] + remote: Option, + #[cfg(target_os = "linux")] + provider_management: Option, +} + +impl ServerProcessOwners { + fn shutdown(mut self) -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + drop(self.provider_management.take()); + drop(self.websocket.take()); + #[cfg(unix)] + if let Some(remote) = self.remote.take() { + remote.shutdown()?; + } + Ok(()) + } +} + +enum ServerProcessOwnersState { + Registering(ServerProcessOwners), + Ready(ServerProcessOwners), + Running, + Complete(Option), +} + +struct ServerProcessOwnersCleanup { + state: std::sync::Mutex, + changed: std::sync::Condvar, +} + +impl ServerProcessOwnersCleanup { + fn new() -> Self { + Self { + state: std::sync::Mutex::new(ServerProcessOwnersState::Registering( + ServerProcessOwners::default(), + )), + changed: std::sync::Condvar::new(), + } + } + + fn register_websocket(&self, server: cmux_tui_core::server::WebSocketServer) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let ServerProcessOwnersState::Registering(owners) = &mut *state else { + panic!("server process owners were sealed before WebSocket registration"); + }; + assert!(owners.websocket.replace(server).is_none(), "WebSocket owner registered twice"); + } + + #[cfg(unix)] + fn register_remote(&self, runtime: remote_runtime::DaemonRuntimeHandle) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let ServerProcessOwnersState::Registering(owners) = &mut *state else { + panic!("server process owners were sealed before remote registration"); + }; + assert!(owners.remote.replace(runtime).is_none(), "remote owner registered twice"); + } + + #[cfg(target_os = "linux")] + fn register_provider_management( + &self, + server: cmux_tui_core::provider_management::ProviderManagementServer, + ) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let ServerProcessOwnersState::Registering(owners) = &mut *state else { + panic!("server process owners were sealed before provider registration"); + }; + assert!( + owners.provider_management.replace(server).is_none(), + "provider owner registered twice" + ); + } + + fn seal(&self) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if matches!(&*state, ServerProcessOwnersState::Registering(_)) { + let ServerProcessOwnersState::Registering(owners) = + std::mem::replace(&mut *state, ServerProcessOwnersState::Running) + else { + unreachable!() + }; + *state = ServerProcessOwnersState::Ready(owners); + self.changed.notify_all(); + } + } + + #[cfg(unix)] + fn remote_is_finished(&self) -> bool { + let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + match &*state { + ServerProcessOwnersState::Registering(owners) + | ServerProcessOwnersState::Ready(owners) => { + owners.remote.as_ref().is_some_and(remote_runtime::DaemonRuntimeHandle::is_finished) + } + ServerProcessOwnersState::Running | ServerProcessOwnersState::Complete(_) => true, + } + } + + fn run_once(&self) -> anyhow::Result<()> { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let owners = loop { + match &*state { + ServerProcessOwnersState::Registering(_) | ServerProcessOwnersState::Running => { + state = + self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + } + ServerProcessOwnersState::Ready(_) => { + let ServerProcessOwnersState::Ready(owners) = + std::mem::replace(&mut *state, ServerProcessOwnersState::Running) + else { + unreachable!() + }; + break owners; + } + ServerProcessOwnersState::Complete(error) => { + return error + .as_ref() + .map(|message| Err(anyhow::anyhow!(message.clone()))) + .unwrap_or(Ok(())); + } + } + }; + drop(state); + + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| owners.shutdown())) + .unwrap_or_else(|_| Err(anyhow::anyhow!("server process owner cleanup panicked"))); + let error = result.as_ref().err().map(|error| format!("{error:#}")); + state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + *state = ServerProcessOwnersState::Complete(error); + self.changed.notify_all(); + result + } +} + enum ServerShutdownCleanupState { Pending(u64), Running(u64), - Complete, + Complete { error: Option }, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, Abandoned, } #[derive(Clone)] enum ServerShutdownCleanupOutcome { - Complete, + Complete { error: Option }, Degraded { attempt: u64, message: String, retry: cmux_tui_core::ShutdownRequestWatch }, Abandoned, } @@ -1340,6 +1477,7 @@ fn set_server_shutdown_attempt_for_run(attempt: Option) { struct ServerShutdownCleanup { mux: Arc, + process_owners: Arc, state: std::sync::Mutex, changed: std::sync::Condvar, #[cfg(test)] @@ -1347,9 +1485,20 @@ struct ServerShutdownCleanup { } impl ServerShutdownCleanup { + #[cfg(test)] fn new(mux: Arc) -> Self { + let process_owners = Arc::new(ServerProcessOwnersCleanup::new()); + process_owners.seal(); + Self::new_with_process_owners(mux, process_owners) + } + + fn new_with_process_owners( + mux: Arc, + process_owners: Arc, + ) -> Self { Self { mux, + process_owners, state: std::sync::Mutex::new(ServerShutdownCleanupState::Pending(0)), changed: std::sync::Condvar::new(), #[cfg(test)] @@ -1359,7 +1508,7 @@ impl ServerShutdownCleanup { } } - fn attempt_shutdown(&self) -> anyhow::Result<()> { + fn attempt_mux_shutdown(&self) -> anyhow::Result<()> { #[cfg(test)] if let Some(attempt) = self.shutdown_attempt.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() @@ -1382,8 +1531,8 @@ impl ServerShutdownCleanup { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let (attempt, retry) = loop { match &*state { - ServerShutdownCleanupState::Complete => { - return ServerShutdownCleanupOutcome::Complete; + ServerShutdownCleanupState::Complete { error } => { + return ServerShutdownCleanupOutcome::Complete { error: error.clone() }; } ServerShutdownCleanupState::Degraded { attempt, message, retry } => { return ServerShutdownCleanupOutcome::Degraded { @@ -1409,29 +1558,44 @@ impl ServerShutdownCleanup { }; drop(state); - // Mux::shutdown owns one bounded attempt. A failure retains the mux, - // socket, and exact process owners until another explicit request - // authorizes the next attempt. + // Runtime and listener owners join exactly once before Mux::shutdown. + // A mux failure retains the socket until another explicit request, + // while a joined runtime error is terminal and remains reportable. + let process_owner_error = + self.process_owners.run_once().err().map(|error| format!("{error:#}")); let result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_shutdown())); + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_mux_shutdown())); let outcome = match result { - Ok(Ok(())) => ServerShutdownCleanupOutcome::Complete, + Ok(Ok(())) => ServerShutdownCleanupOutcome::Complete { error: process_owner_error }, Ok(Err(error)) => ServerShutdownCleanupOutcome::Degraded { attempt, message: format!( - "{}: {error:#}", - localization::catalog().server.shutdown_cleanup_incomplete + "{}: {error:#}{}", + localization::catalog().server.shutdown_cleanup_incomplete, + process_owner_error + .as_deref() + .map(|owner| format!("; process owner cleanup: {owner}")) + .unwrap_or_default() ), retry, }, Err(_) => ServerShutdownCleanupOutcome::Degraded { attempt, - message: localization::catalog().server.shutdown_cleanup_incomplete.to_string(), + message: format!( + "{}{}", + localization::catalog().server.shutdown_cleanup_incomplete, + process_owner_error + .as_deref() + .map(|owner| format!("; process owner cleanup: {owner}")) + .unwrap_or_default() + ), retry, }, }; let next_state = match &outcome { - ServerShutdownCleanupOutcome::Complete => ServerShutdownCleanupState::Complete, + ServerShutdownCleanupOutcome::Complete { error } => { + ServerShutdownCleanupState::Complete { error: error.clone() } + } ServerShutdownCleanupOutcome::Degraded { attempt, message, retry } => { ServerShutdownCleanupState::Degraded { attempt: *attempt, @@ -1452,7 +1616,10 @@ impl ServerShutdownCleanup { fn run_until_complete(&self) -> anyhow::Result<()> { match self.run_once() { - ServerShutdownCleanupOutcome::Complete => Ok(()), + ServerShutdownCleanupOutcome::Complete { error: Some(error) } => { + Err(anyhow::anyhow!(error)) + } + ServerShutdownCleanupOutcome::Complete { error: None } => Ok(()), ServerShutdownCleanupOutcome::Degraded { message, .. } => Err(anyhow::anyhow!(message)), ServerShutdownCleanupOutcome::Abandoned => { Err(anyhow::anyhow!("server shutdown cleanup was abandoned")) @@ -1483,11 +1650,14 @@ impl ServerShutdownCleanup { } } - fn wait_for_complete(&self) { + fn wait_for_complete(&self) -> anyhow::Result<()> { let mut reported_attempt = None; loop { match self.run_once() { - ServerShutdownCleanupOutcome::Complete => return, + ServerShutdownCleanupOutcome::Complete { error: Some(error) } => { + return Err(anyhow::anyhow!(error)); + } + ServerShutdownCleanupOutcome::Complete { error: None } => return Ok(()), ServerShutdownCleanupOutcome::Degraded { attempt, message, retry } => { if reported_attempt != Some(attempt) { eprintln!("cmux-tui: {message}; retry with `cmux-tui server stop`"); @@ -1512,7 +1682,9 @@ impl ServerShutdownCleanup { .unwrap_or_else(std::sync::PoisonError::into_inner); } } - ServerShutdownCleanupOutcome::Abandoned => return, + ServerShutdownCleanupOutcome::Abandoned => { + return Err(anyhow::anyhow!("server shutdown cleanup was abandoned")); + } } } } @@ -1520,7 +1692,7 @@ impl ServerShutdownCleanup { fn is_complete(&self) -> bool { matches!( *self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner), - ServerShutdownCleanupState::Complete + ServerShutdownCleanupState::Complete { .. } ) } } @@ -1528,6 +1700,7 @@ impl ServerShutdownCleanup { struct ServerProcessShutdownGuard { published_socket: Arc>>, shutdown_watch: cmux_tui_core::ShutdownRequestWatch, + process_owners: Arc, cleanup: Arc, completion: Option>, worker: Option>, @@ -1545,7 +1718,11 @@ impl ServerProcessShutdownGuard { let worker_watch = shutdown_watch.clone(); let published_socket = Arc::new(std::sync::Mutex::new(None)); let worker_published_socket = published_socket.clone(); - let cleanup = Arc::new(ServerShutdownCleanup::new(mux.clone())); + let process_owners = Arc::new(ServerProcessOwnersCleanup::new()); + let cleanup = Arc::new(ServerShutdownCleanup::new_with_process_owners( + mux.clone(), + process_owners.clone(), + )); let worker_cleanup = cleanup.clone(); let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::Builder::new().name("cmux-server-exit-watchdog".into()).spawn( @@ -1558,7 +1735,7 @@ impl ServerProcessShutdownGuard { // same bounded cleanup path that run_server joins below. // Failed attempts keep serving the control socket and require // another explicit stop request before retrying. - worker_cleanup.wait_for_complete(); + let cleanup_result = worker_cleanup.wait_for_complete(); if matches!( completed.recv_timeout(server_shutdown_exit_grace()), Err(std::sync::mpsc::RecvTimeoutError::Timeout) @@ -1567,6 +1744,10 @@ impl ServerProcessShutdownGuard { // remains. The frontend alone is stuck, so the old // process can now release its control socket. cleanup_published_socket(&worker_published_socket); + if let Err(error) = cleanup_result { + eprintln!("cmux-tui: server shutdown cleanup failed: {error:#}"); + std::process::exit(1); + } std::process::exit(0); } }, @@ -1574,6 +1755,7 @@ impl ServerProcessShutdownGuard { Ok(Self { published_socket, shutdown_watch, + process_owners, cleanup, completion: Some(completion), worker: Some(worker), @@ -1598,16 +1780,42 @@ impl ServerProcessShutdownGuard { self.published_socket.lock().unwrap_or_else(std::sync::PoisonError::into_inner).is_some() } + fn register_websocket(&self, server: cmux_tui_core::server::WebSocketServer) { + self.process_owners.register_websocket(server); + } + + #[cfg(unix)] + fn register_remote(&self, runtime: remote_runtime::DaemonRuntimeHandle) { + self.process_owners.register_remote(runtime); + } + + #[cfg(target_os = "linux")] + fn register_provider_management( + &self, + server: cmux_tui_core::provider_management::ProviderManagementServer, + ) { + self.process_owners.register_provider_management(server); + } + + fn seal_process_owners(&self) { + self.process_owners.seal(); + } + + #[cfg(unix)] + fn remote_is_finished(&self) -> bool { + self.process_owners.remote_is_finished() + } + fn shutdown(&self) -> anyhow::Result<()> { self.cleanup.run_until_complete() } - fn wait_for_shutdown(&self) { + fn wait_for_shutdown(&self) -> anyhow::Result<()> { // `wait_for_complete` runs the first bounded cleanup attempt // synchronously. It waits for another mux request only after that // attempt fails, while the published socket remains available for an // explicit `server stop` retry. - self.cleanup.wait_for_complete(); + self.cleanup.wait_for_complete() } fn complete(mut self) { @@ -1661,28 +1869,24 @@ fn finish_server_process( server_process: ServerProcessShutdownGuard, result: anyhow::Result<()>, ) -> anyhow::Result<()> { + server_process.seal_process_owners(); let has_published_socket = server_process.has_published_socket(); - if !has_published_socket || result.is_err() { - let shutdown_result = server_process.shutdown(); - if has_published_socket { - if shutdown_result.is_err() { - // A failed cleanup retains the published socket and exact - // process owners until an explicit stop retries it. - server_process.wait_for_shutdown(); - } - server_process.complete(); - } else { - server_process.abandon_unpublished(); - } - return match (result, shutdown_result) { - (result, Ok(())) => result, - (Ok(()), Err(shutdown_error)) => Err(shutdown_error), - (Err(error), Err(shutdown_error)) => Err(error.context(shutdown_error.to_string())), - }; + let mut shutdown_result = if !has_published_socket || result.is_err() { + server_process.shutdown() + } else { + server_process.wait_for_shutdown() + }; + if has_published_socket && shutdown_result.is_err() && !server_process.cleanup.is_complete() { + // A retryable mux failure retains the published socket until another + // explicit stop request authorizes the next bounded attempt. + shutdown_result = server_process.wait_for_shutdown(); + } + if has_published_socket || server_process.cleanup.is_complete() { + server_process.complete(); + } else { + server_process.abandon_unpublished(); } - server_process.wait_for_shutdown(); - server_process.complete(); - result + combine_server_and_remote_shutdown_results(result, shutdown_result) } #[cfg(unix)] @@ -1856,7 +2060,7 @@ fn run_server( } #[cfg(target_os = "linux")] - let _provider_management = match provider_management_listener + let provider_management = match provider_management_listener .map(|listener| cmux_tui_core::provider_management::serve(listener, mux.clone())) .transpose() { @@ -1865,13 +2069,15 @@ fn run_server( return finish_server_process(server_process, Err(error.into())); } }; + #[cfg(target_os = "linux")] + if let Some(provider_management) = provider_management { + server_process.register_provider_management(provider_management); + } let websocket_server = match ws_addr { Some(addr) => { let addr = match addr.parse() { Ok(addr) => addr, Err(error) => { - #[cfg(target_os = "linux")] - drop(_provider_management); return finish_server_process( server_process, Err(anyhow::anyhow!("invalid WebSocket address {addr:?}: {error}")), @@ -1886,8 +2092,6 @@ fn run_server( ) { Ok(server) => Some(server), Err(error) => { - #[cfg(target_os = "linux")] - drop(_provider_management); return finish_server_process(server_process, Err(error)); } } @@ -1897,9 +2101,12 @@ fn run_server( if let Some(server) = &websocket_server { eprintln!("cmux-tui: WebSocket control at ws://{}", server.local_addr()); } + if let Some(websocket_server) = websocket_server { + server_process.register_websocket(websocket_server); + } #[cfg(unix)] - let remote_runtime = if args.remote { + if args.remote { let runtime = match remote_runtime::start_daemon_runtime( socket_path.clone(), remote_runtime::DaemonRuntimeOptions { @@ -1919,9 +2126,6 @@ fn run_server( ) { Ok(runtime) => runtime, Err(error) => { - drop(websocket_server); - #[cfg(target_os = "linux")] - drop(_provider_management); return finish_server_process(server_process, Err(error)); } }; @@ -1934,14 +2138,15 @@ fn run_server( for route in &runtime.info().routes { eprintln!("cmux-tui: remote route {route}"); } + server_process.register_remote(runtime); #[cfg(debug_assertions)] - if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER") { - std::fs::write(marker, b"started")?; + if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER") + && let Err(error) = std::fs::write(marker, b"started") + { + return finish_server_process(server_process, Err(error.into())); } - Some(runtime) - } else { - None - }; + } + server_process.seal_process_owners(); #[cfg(debug_assertions)] if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { @@ -1955,11 +2160,7 @@ fn run_server( let result = if args.headless { #[cfg(unix)] { - run_headless(&mux, &socket_path, || { - remote_runtime - .as_ref() - .is_some_and(remote_runtime::DaemonRuntimeHandle::is_finished) - }) + run_headless(&mux, &socket_path, || server_process.remote_is_finished()) } #[cfg(not(unix))] { @@ -1970,14 +2171,6 @@ fn run_server( } else { run_tui(Session::Local(mux), args.session, None) }; - #[cfg(unix)] - let result = match remote_runtime { - Some(runtime) => combine_server_and_remote_shutdown_results(result, runtime.shutdown()), - None => result, - }; - drop(websocket_server); - #[cfg(target_os = "linux")] - drop(_provider_management); finish_server_process(server_process, result) } @@ -2621,12 +2814,14 @@ mod tests { let mux = Mux::new("server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); - *cleanup.state.lock().unwrap() = ServerShutdownCleanupState::Complete; + *cleanup.state.lock().unwrap() = ServerShutdownCleanupState::Complete { error: None }; + let process_owners = cleanup.process_owners.clone(); let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || completed.recv().unwrap()); let guard = ServerProcessShutdownGuard { published_socket: Arc::new(std::sync::Mutex::new(Some(published))), shutdown_watch, + process_owners, cleanup, completion: Some(completion), worker: Some(worker), @@ -2658,6 +2853,7 @@ mod tests { let mux = Mux::new("incomplete-server-guard-test", SurfaceOptions::default()); let shutdown_watch = mux.watch_shutdown_request(); let cleanup = Arc::new(ServerShutdownCleanup::new(mux)); + let process_owners = cleanup.process_owners.clone(); let (completion, completed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || { let _ = completed.recv(); @@ -2665,6 +2861,7 @@ mod tests { let guard = ServerProcessShutdownGuard { published_socket: Arc::new(std::sync::Mutex::new(Some(published))), shutdown_watch, + process_owners, cleanup, completion: Some(completion), worker: Some(worker), diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index a7cf7b797dc9..49cba597b14e 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -176,7 +176,7 @@ impl DaemonRuntimeHandle { let shutdown_marker = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER"); #[cfg(debug_assertions)] if let Some(marker) = shutdown_marker.as_ref() { - fs::write(marker, b"started")?; + let _ = fs::write(marker, b"started"); } #[cfg(debug_assertions)] if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_REMOTE_SHUTDOWN_DELAY_MS") @@ -192,7 +192,7 @@ impl DaemonRuntimeHandle { }; #[cfg(debug_assertions)] if let Some(marker) = shutdown_marker { - fs::write(marker, b"complete")?; + let _ = fs::write(marker, b"complete"); } result } From 77c7a8627f59266e23c4e9fa2e3be2453a965321 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:03:51 -0700 Subject: [PATCH 321/521] test(tui): expose committed cleanup replay gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 74 ++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d57da8437293..cbbbe07f57eb 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2430,6 +2430,8 @@ pub struct Mux { resource_close_after_commit: Mutex>>, #[cfg(test)] discovered_terminal_termination_requests: Mutex)>>, + #[cfg(test)] + terminal_host_cleanup_hook: Mutex bool + Send + Sync>>>, browser_runtime: BrowserRuntimeSlot, active_render_attachments: Arc, deadline_fanout_pool: DeadlineFanoutPool, @@ -2785,6 +2787,8 @@ impl Mux { resource_close_after_commit: Mutex::new(None), #[cfg(test)] discovered_terminal_termination_requests: Mutex::new(Vec::new()), + #[cfg(test)] + terminal_host_cleanup_hook: Mutex::new(None), browser_runtime: BrowserRuntimeSlot::default(), active_render_attachments: Arc::new(AtomicUsize::new(0)), deadline_fanout_pool: DeadlineFanoutPool::new(), @@ -7753,6 +7757,10 @@ impl Mux { &self, records: TerminalHostRecords, ) -> anyhow::Result<()> { + #[cfg(test)] + if let Some(hook) = self.terminal_host_cleanup_hook.lock().unwrap().clone() { + anyhow::ensure!(hook(), "forced terminal-host cleanup failure"); + } #[cfg(unix)] { for (path, record) in records { @@ -17605,6 +17613,72 @@ mod tests { ); } + #[test] + fn local_workspace_close_retries_cleanup_from_its_committed_receipt() { + let mux = test_mux(); + let surface = mux.new_workspace(Some("local-cleanup-retry".into()), None).unwrap(); + let workspace = mux.with_state(|state| state.workspaces[0].id); + let attempts = Arc::new(AtomicUsize::new(0)); + *mux.terminal_host_cleanup_hook.lock().unwrap() = Some(Arc::new({ + let attempts = attempts.clone(); + move || attempts.fetch_add(1, Ordering::SeqCst) != 0 + })); + + let error = mux.close_workspace_at_revision(workspace, None).unwrap_err(); + assert!(error.to_string().contains("forced terminal-host cleanup failure")); + assert!(mux.with_state(|state| state.workspaces.is_empty())); + let committed_revision = mux.workspace_registry.lock().unwrap().snapshot().unwrap().revision; + + assert_eq!( + mux.close_workspace_at_revision(workspace, None).unwrap(), + Some(committed_revision) + ); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + *mux.terminal_host_cleanup_hook.lock().unwrap() = None; + surface.kill(); + mux.shutdown().unwrap(); + } + + #[test] + fn resource_workspace_close_retries_cleanup_before_committed_success() { + let mux = test_mux(); + let surface = mux.new_workspace(Some("resource-cleanup-retry".into()), None).unwrap(); + let workspace = mux.with_state(|state| state.workspaces[0].public_id.to_string()); + let attempts = Arc::new(AtomicUsize::new(0)); + *mux.terminal_host_cleanup_hook.lock().unwrap() = Some(Arc::new({ + let attempts = attempts.clone(); + move || attempts.fetch_add(1, Ordering::SeqCst) != 0 + })); + let params = serde_json::json!({ + "machine":"current", + "session":"current", + "workspace":workspace, + }); + + let first = public_request( + &mux, + "cleanup-first", + "workspace.close", + params.clone(), + Some("cleanup-retry-effect"), + ); + assert!(first.get("error").is_some(), "postcommit cleanup failure was acknowledged"); + assert!(mux.with_state(|state| state.workspaces.is_empty())); + + let replay = public_request( + &mux, + "cleanup-replay", + "workspace.close", + params, + Some("cleanup-retry-effect"), + ); + assert_eq!(replay["result"]["replayed"], true); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + *mux.terminal_host_cleanup_hook.lock().unwrap() = None; + surface.kill(); + mux.shutdown().unwrap(); + } + #[cfg(unix)] #[test] fn workspace_close_cleanup_is_driven_by_live_host_records() { From 4c7bc6e335cb30e532f8492136d8e1d6f1a37989 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:05:08 -0700 Subject: [PATCH 322/521] test(tui): require provider peer reaping --- .../cmux-tui-core/src/provider_management.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index 2696126f2738..5ad205edff2a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -427,6 +427,7 @@ mod tests { use super::*; use crate::SurfaceOptions; use std::sync::Arc; + use std::time::{Duration, Instant}; const MUX_GENERATION: &str = "0123456789abcdef0123456789abcdef"; const AUTHORITY_ONE: &str = "provider-authority-one-0000000000000001"; @@ -451,6 +452,27 @@ mod tests { assert_eq!(response.error.unwrap().code, "access_denied"); } + #[test] + fn completed_provider_peer_threads_are_reaped_while_server_runs() { + let completed = std::thread::spawn(|| {}); + let deadline = Instant::now() + Duration::from_secs(1); + while !completed.is_finished() { + assert!(Instant::now() < deadline, "peer thread did not finish"); + std::thread::yield_now(); + } + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let active = std::thread::spawn(move || release_rx.recv().unwrap()); + let mut peers = vec![completed, active]; + + reap_finished_peer_threads(&mut peers); + + assert_eq!(peers.len(), 1, "completed peer handle remained retained"); + release_tx.send(()).unwrap(); + for peer in peers { + peer.join().unwrap(); + } + } + #[cfg(target_os = "linux")] #[test] fn explicit_unsupported_response_is_the_only_upgrade_signal() { From be3545b9358862840219f33db3dedc91f583bc93 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:43:01 -0700 Subject: [PATCH 323/521] fix(tui): close cleanup lifecycle gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 99 +++++++++++++++++-- .../src/mux/resource_topology.rs | 40 +++++--- .../cmux-tui-core/src/provider_management.rs | 14 +++ 3 files changed, 130 insertions(+), 23 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index cbbbe07f57eb..eb6a86fabdc8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -12192,6 +12192,21 @@ impl Mux { let _creation_fence = project_resource.then(|| self.resource_creation_execution.lock().unwrap()); let authority = self.authorize_workspace_lifecycle_mutation(authorization, "close")?; + let mutation = local_workspace_close_mutation(id, key, project_resource); + let fingerprint = workspace_close_fingerprint(id, key); + { + let registry = self.workspace_registry.lock().unwrap(); + if let Some(commit) = registry.replay(&mutation, &fingerprint)? { + let result = workspace_mutation_result(&commit)?; + let workspace = result + .workspace + .context("stored local workspace close result is missing its workspace")?; + drop(registry); + self.terminate_workspace_hosts_from_receipt(&commit.result)?; + drop(authority); + return Ok(Some((workspace, result.key, result.revision))); + } + } let resolved = { let state = self.state.lock().unwrap(); Self::require_workspace_revision(&state, expected_revision)?; @@ -12200,7 +12215,6 @@ impl Mux { let Some((resolved_target, _)) = resolved else { return Ok(None); }; - let mutation = WorkspaceMutation::local("cmux-tui"); let result = self.close_workspace_with_mutation_inner( id, key, @@ -12223,11 +12237,7 @@ impl Mux { mutation: &WorkspaceMutation, project_resource: bool, ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "op": "close-workspace", - "workspace": target, - "key": requested_key, - }); + let fingerprint = workspace_close_fingerprint(target, requested_key); { let registry = self.workspace_registry.lock().unwrap(); if let Some(commit) = registry.replay(mutation, &fingerprint)? { @@ -15890,6 +15900,45 @@ fn workspace_mutation_result(commit: &RegistryCommit) -> anyhow::Result, key: Option<&str>) -> Value { + serde_json::json!({ + "op": "close-workspace", + "workspace": target, + "key": key, + }) +} + +fn local_workspace_close_mutation( + target: Option, + key: Option<&str>, + project_resource: bool, +) -> WorkspaceMutation { + let mut digest = Sha256::new(); + digest.update(b"cmux-tui/local-workspace-close/1\0"); + digest.update([u8::from(project_resource)]); + match target { + Some(target) => { + digest.update([1]); + digest.update(target.to_be_bytes()); + } + None => digest.update([0]), + } + match key { + Some(key) => { + digest.update([1]); + digest.update( + u64::try_from(key.len()).expect("workspace key length fits u64").to_be_bytes(), + ); + digest.update(key.as_bytes()); + } + None => digest.update([0]), + } + let payload = + digest.finalize()[..16].iter().map(|byte| format!("{byte:02x}")).collect::(); + WorkspaceMutation::new(format!("local-workspace-close-{payload}"), "cmux-tui") + .expect("derived local workspace close mutation is valid") +} + fn terminal_host_cleanup_receipt(terminals: &[(String, Option)]) -> Value { Value::Array( terminals @@ -15946,6 +15995,12 @@ fn terminal_host_cleanup_identities( Ok(identities) } +fn remove_terminal_host_cleanup_receipt(result: &mut Value) { + if let Some(result) = result.as_object_mut() { + result.remove(TERMINAL_HOST_CLEANUP_RECEIPT_FIELD); + } +} + fn close_surface_delta( state: &State, notifications: &HashMap, @@ -17627,7 +17682,8 @@ mod tests { let error = mux.close_workspace_at_revision(workspace, None).unwrap_err(); assert!(error.to_string().contains("forced terminal-host cleanup failure")); assert!(mux.with_state(|state| state.workspaces.is_empty())); - let committed_revision = mux.workspace_registry.lock().unwrap().snapshot().unwrap().revision; + let committed_revision = + mux.workspace_registry.lock().unwrap().snapshot().unwrap().revision; assert_eq!( mux.close_workspace_at_revision(workspace, None).unwrap(), @@ -17643,7 +17699,10 @@ mod tests { fn resource_workspace_close_retries_cleanup_before_committed_success() { let mux = test_mux(); let surface = mux.new_workspace(Some("resource-cleanup-retry".into()), None).unwrap(); - let workspace = mux.with_state(|state| state.workspaces[0].public_id.to_string()); + let (workspace_id, workspace) = mux.with_state(|state| { + (state.workspaces[0].id, state.workspaces[0].public_id.to_string()) + }); + let selectors = mux.ordinary_workspace_selectors(workspace_id).unwrap(); let attempts = Arc::new(AtomicUsize::new(0)); *mux.terminal_host_cleanup_hook.lock().unwrap() = Some(Arc::new({ let attempts = attempts.clone(); @@ -17664,6 +17723,26 @@ mod tests { ); assert!(first.get("error").is_some(), "postcommit cleanup failure was acknowledged"); assert!(mux.with_state(|state| state.workspaces.is_empty())); + let fingerprint = serde_json::json!({ + "operation":"workspace.close", + "selectors":selectors, + "fields":{}, + }); + let preparation = mux + .workspace_registry + .lock() + .unwrap() + .lookup_resource_effect("cleanup-retry-effect", "workspace.close", &fingerprint) + .unwrap() + .unwrap(); + let ResourceEffectPreparation::Committed { + outcome: ResourceEffectOutcome::Success(stored_result), + .. + } = preparation + else { + panic!("postcommit cleanup failure lost its committed success receipt"); + }; + assert_eq!(terminal_host_cleanup_identities(&stored_result).unwrap().len(), 1); let replay = public_request( &mux, @@ -17673,6 +17752,10 @@ mod tests { Some("cleanup-retry-effect"), ); assert_eq!(replay["result"]["replayed"], true); + assert!( + !replay.to_string().contains(TERMINAL_HOST_CLEANUP_RECEIPT_FIELD), + "public replay leaked its internal cleanup receipt" + ); assert_eq!(attempts.load(Ordering::SeqCst), 2); *mux.terminal_host_cleanup_hook.lock().unwrap() = None; surface.kill(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 945230b57778..db7aea2265e4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -1786,7 +1786,9 @@ impl Mux { }; match preparation { ResourceEffectPreparation::Committed { outcome, revision } => match outcome { - ResourceEffectOutcome::Success(result) => { + ResourceEffectOutcome::Success(mut result) => { + self.terminate_workspace_hosts_from_receipt(&result)?; + remove_terminal_host_cleanup_receipt(&mut result); Ok(ResourcePatchCommit { revision, result, replayed: true }) } ResourceEffectOutcome::Failure(error) => Err(anyhow::Error::new(error)), @@ -2064,22 +2066,35 @@ impl Mux { terminal_host_cleanup_receipt(&workspace_host_identities), ); } - let projection = + let cleanup_identities = if plan.closed_workspace_key.is_some() { + workspace_host_identities + } else { + std::mem::take(&mut plan.unmaterialized_terminals) + }; + let host_cleanup = self.prepare_terminal_host_cleanup_at_root( + terminal_host_root.as_deref(), + &cleanup_identities, + )?; + let mut projection = self.resource_effect_projection_locked(®istry, &mut plan.state, json!({}))?; + projection + .result + .as_object_mut() + .context("resource close result is not an object")? + .insert( + TERMINAL_HOST_CLEANUP_RECEIPT_FIELD.to_string(), + terminal_host_cleanup_receipt(&cleanup_identities), + ); let closed_public_ids = if let Some(workspace_key) = plan.closed_workspace_key.as_deref() { registry.terminal_resource_ids_in_workspace(workspace_key)? } else { Self::terminal_public_ids_for_hosted(®istry, &plan.terminals)? }; - let workspace_host_cleanup = self.prepare_terminal_host_cleanup_at_root( - terminal_host_root.as_deref(), - &workspace_host_identities, - )?; #[cfg(test)] if let Some(hook) = self.resource_projection_before_commit.lock().unwrap().clone() { hook(); } - let close = registry.commit_resource_close_patch( + let mut close = registry.commit_resource_close_patch( idempotency_key, operation_name, fingerprint, @@ -2108,25 +2123,20 @@ impl Mux { drop(registry); drop(workspace_lifecycle); - let unmaterialized_termination = if plan.closed_workspace_key.is_none() { - self.terminate_discovered_terminal_hosts(&plan.unmaterialized_terminals) - } else { - Ok(()) - }; self.notify_terminal_exit_waiters(closed_public_ids); self.publish_resource_event(); for surface in &plan.removed { self.purge_surface_side_tables(surface.id); } self.retire_surface_runtimes(plan.removed); - let workspace_termination = self.terminate_prepared_terminal_hosts(workspace_host_cleanup); + let host_termination = self.terminate_prepared_terminal_hosts(host_cleanup); self.emit_tree_delta(plan.delta, plan.selection_resync); for screen in plan.changed_screens { self.emit(MuxEvent::LayoutChanged(screen)); } self.emit_empty_if_current(empty_revision); - unmaterialized_termination?; - workspace_termination?; + host_termination?; + remove_terminal_host_cleanup_receipt(&mut close.resource.result); Ok(close.resource) } diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index 5ad205edff2a..6b1536bcd158 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -225,6 +225,19 @@ pub struct ProviderManagementServer { thread: Option>, } +#[cfg(any(target_os = "linux", test))] +fn reap_finished_peer_threads(peers: &mut Vec>) { + let mut index = 0; + while index < peers.len() { + if peers[index].is_finished() { + let peer = peers.swap_remove(index); + let _ = peer.join(); + } else { + index += 1; + } + } +} + #[cfg(target_os = "linux")] impl Drop for ProviderManagementServer { fn drop(&mut self) { @@ -256,6 +269,7 @@ pub fn serve( let next_peer = AtomicU64::new(1); let mut peers = Vec::new(); while !thread_shutdown.load(Ordering::Acquire) { + reap_finished_peer_threads(&mut peers); let stream = match listener.accept() { Ok((stream, _)) => stream, Err(error) if error.kind() == io::ErrorKind::WouldBlock => { From 9f3f44d4071944c0cb88806de3299db7be0c1573 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:10:49 -0700 Subject: [PATCH 324/521] test(tui): expose identify SDK schema drift --- .../python/tests/test_schema_roundtrip.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmux-tui/bindings/python/tests/test_schema_roundtrip.py b/cmux-tui/bindings/python/tests/test_schema_roundtrip.py index ba05c370931c..2958b1a0e116 100644 --- a/cmux-tui/bindings/python/tests/test_schema_roundtrip.py +++ b/cmux-tui/bindings/python/tests/test_schema_roundtrip.py @@ -86,6 +86,32 @@ def _sample( class SchemaRoundTripTests(unittest.TestCase): + def test_identify_decodes_shutdown_cleanup_health(self) -> None: + result = decode_command_result( + "identify", + { + "app": "cmux-tui", + "version": "0.1.0", + "protocol": 10, + "session": "main", + "pid": 12345, + "registry_id": "registry", + "generation": "generation", + "workspace_revision": 1, + "terminal_revision": 2, + "daemon_handoff": 1, + "shutdown_cleanup": { + "pending": 3, + "retrying": True, + "degraded": False, + }, + }, + ) + + self.assertEqual(result.shutdown_cleanup.pending, 3) + self.assertTrue(result.shutdown_cleanup.retrying) + self.assertFalse(result.shutdown_cleanup.degraded) + def test_every_command_request_encodes(self) -> None: for command, definition in SCHEMA["commands"].items(): with self.subTest(command=command): From 1164a68d5f25206d303160ce725c0b350ebabf9f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:16:39 -0700 Subject: [PATCH 325/521] fix(tui): declare identify cleanup status --- cmux-tui/bindings/cpp/.cmux-sdk-manifest.json | 10 +- .../cpp/include/cmux/raw/generated/models.hpp | 17 +++- .../cpp/src/raw/generated/protocol.cpp | 60 ++++++++++++ .../bindings/go/raw/.cmux-sdk-manifest.json | 16 ++-- .../bindings/go/raw/generated_commands.go | 2 +- cmux-tui/bindings/go/raw/generated_events.go | 2 +- .../bindings/go/raw/generated_metadata.go | 4 +- .../go/raw/generated_presence_test.go | 68 +++++++++++++- cmux-tui/bindings/go/raw/generated_types.go | 72 ++++++++++++--- .../src/com/cmux/raw/.cmux-sdk-manifest.json | 13 ++- .../java/src/com/cmux/raw/IdentifyResult.java | 17 +++- .../java/src/com/cmux/raw/Protocol.java | 2 +- .../com/cmux/raw/ShutdownCleanupStatus.java | 91 +++++++++++++++++++ .../raw/_generated/.cmux-sdk-manifest.json | 16 ++-- .../python/cmux/raw/_generated/_schema.py | 2 +- .../python/cmux/raw/_generated/codec.py | 1 + .../python/cmux/raw/_generated/metadata.py | 2 +- .../python/cmux/raw/_generated/models.py | 10 ++ .../src/generated/.cmux-sdk-manifest.json | 14 +-- .../bindings/rust/src/generated/commands.rs | 2 +- .../bindings/rust/src/generated/events.rs | 2 +- .../bindings/rust/src/generated/metadata.rs | 4 +- cmux-tui/bindings/rust/src/generated/mod.rs | 2 +- cmux-tui/bindings/rust/src/generated/types.rs | 12 ++- .../src/raw/generated/.cmux-sdk-manifest.json | 16 ++-- .../typescript/src/raw/generated/commands.ts | 2 +- .../typescript/src/raw/generated/events.ts | 2 +- .../typescript/src/raw/generated/index.ts | 2 +- .../typescript/src/raw/generated/metadata.ts | 43 ++++++++- .../typescript/src/raw/generated/types.ts | 9 +- .../src/raw/generated/.cmux-sdk-manifest.json | 10 +- .../zig/src/raw/generated/presence_test.zig | 1 + .../zig/src/raw/generated/protocol.zig | 10 +- cmux-tui/spec/commands.md | 6 +- cmux-tui/spec/sdk-schema.json | 39 ++++++++ 35 files changed, 492 insertions(+), 89 deletions(-) create mode 100644 cmux-tui/bindings/java/src/com/cmux/raw/ShutdownCleanupStatus.java diff --git a/cmux-tui/bindings/cpp/.cmux-sdk-manifest.json b/cmux-tui/bindings/cpp/.cmux-sdk-manifest.json index 670f660fd4ea..91be301d7b3f 100644 --- a/cmux-tui/bindings/cpp/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/cpp/.cmux-sdk-manifest.json @@ -12,17 +12,17 @@ }, { "path": "include/cmux/raw/generated/models.hpp", - "sha256": "689d095b4df5327e494eca5d097cbab5c7e1315b82f82dcba7bc47233a74ef64", - "size": 108434 + "sha256": "35efd3f44b4523adfdc9be4edd5459f36bf99a815703afc7dd9edb5c99f2030b", + "size": 108919 }, { "path": "src/raw/generated/protocol.cpp", - "sha256": "6ed3eb2a74e50a4e810c22354c72adf8e71bd50ff18bf78f383867fd53bc3fd9", - "size": 717205 + "sha256": "dfca5f4abdaf443bd377b98505de8ead23c28f1520873aa1f90a380973fd9d40", + "size": 719874 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "cpp", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/cpp/include/cmux/raw/generated/models.hpp b/cmux-tui/bindings/cpp/include/cmux/raw/generated/models.hpp index c7f57ccf90a3..55edfdb49c70 100644 --- a/cmux-tui/bindings/cpp/include/cmux/raw/generated/models.hpp +++ b/cmux-tui/bindings/cpp/include/cmux/raw/generated/models.hpp @@ -14,7 +14,7 @@ namespace cmux::raw { inline constexpr std::uint32_t kMuxProtocolVersion = 10U; -inline constexpr std::string_view kProtocolIrSha256 = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa"; +inline constexpr std::string_view kProtocolIrSha256 = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d"; struct AgentRecord; enum class AgentReportSource; @@ -84,6 +84,7 @@ struct ResolveTerminalResult; struct RunResult; struct Screen; struct SetCellPixelsResult; +struct ShutdownCleanupStatus; struct ShutdownDaemonResult; struct SidebarPluginResult; struct Size; @@ -1062,6 +1063,13 @@ struct IdentifyRequest { friend bool operator==(const IdentifyRequest&, const IdentifyRequest&) = default; }; +struct ShutdownCleanupStatus { + bool degraded{}; + std::uint64_t pending{}; + bool retrying{}; + friend bool operator==(const ShutdownCleanupStatus&, const ShutdownCleanupStatus&) = default; +}; + struct IdentifyResult { Field build_commit{}; std::optional> capabilities{}; @@ -1071,6 +1079,7 @@ struct IdentifyResult { std::uint32_t protocol{}; std::string registry_id{}; std::string session{}; + std::optional shutdown_cleanup{}; std::uint64_t terminal_revision{}; std::string version{}; std::uint64_t workspace_revision{}; @@ -2678,6 +2687,12 @@ struct Codec { static Result decode(const Json& value); }; +template <> +struct Codec { + static Result encode(const ShutdownCleanupStatus& value); + static Result decode(const Json& value); +}; + template <> struct Codec { static Result encode(const ShutdownDaemonResult& value); diff --git a/cmux-tui/bindings/cpp/src/raw/generated/protocol.cpp b/cmux-tui/bindings/cpp/src/raw/generated/protocol.cpp index d2c9d0fd1732..cc84039e1f22 100644 --- a/cmux-tui/bindings/cpp/src/raw/generated/protocol.cpp +++ b/cmux-tui/bindings/cpp/src/raw/generated/protocol.cpp @@ -1252,6 +1252,11 @@ Result Codec::encode(const IdentifyResult& value) { auto encoded_session = encode_value(value.session); if (!encoded_session) return std::move(encoded_session).error(); object.emplace("session", std::move(encoded_session).value()); + if (value.shutdown_cleanup) { + auto encoded = encode_value(*value.shutdown_cleanup); + if (!encoded) return std::move(encoded).error(); + object.emplace("shutdown_cleanup", std::move(encoded).value()); + } auto encoded_terminal_revision = encode_value(value.terminal_revision); if (!encoded_terminal_revision) return std::move(encoded_terminal_revision).error(); object.emplace("terminal_revision", std::move(encoded_terminal_revision).value()); @@ -1357,6 +1362,12 @@ Result Codec::decode(const Json& value) { if (!decoded) return std::move(decoded).error(); result.session = std::move(decoded).value(); } + const Json* field_shutdown_cleanup = value.find("shutdown_cleanup"); + if (field_shutdown_cleanup) { + auto decoded = decode_value(*field_shutdown_cleanup); + if (!decoded) return std::move(decoded).error(); + result.shutdown_cleanup = std::move(decoded).value(); + } const Json* field_terminal_revision = value.find("terminal_revision"); if (!field_terminal_revision) { return make_error(ErrorCode::decode, "missing required field 'terminal_revision'"); @@ -3922,6 +3933,55 @@ Result Codec::decode(const Json& value return result; } +Result Codec::encode(const ShutdownCleanupStatus& value) { + (void)value; + Json::Object object; + auto encoded_degraded = encode_value(value.degraded); + if (!encoded_degraded) return std::move(encoded_degraded).error(); + object.emplace("degraded", std::move(encoded_degraded).value()); + auto encoded_pending = encode_value(value.pending); + if (!encoded_pending) return std::move(encoded_pending).error(); + object.emplace("pending", std::move(encoded_pending).value()); + auto encoded_retrying = encode_value(value.retrying); + if (!encoded_retrying) return std::move(encoded_retrying).error(); + object.emplace("retrying", std::move(encoded_retrying).value()); + return Json(std::move(object)); +} + +Result Codec::decode(const Json& value) { + auto source = value.as_object(); + if (!source) return std::move(source).error(); + ShutdownCleanupStatus result{}; + const Json* field_degraded = value.find("degraded"); + if (!field_degraded) { + return make_error(ErrorCode::decode, "missing required field 'degraded'"); + } + if (field_degraded) { + auto decoded = decode_value(*field_degraded); + if (!decoded) return std::move(decoded).error(); + result.degraded = std::move(decoded).value(); + } + const Json* field_pending = value.find("pending"); + if (!field_pending) { + return make_error(ErrorCode::decode, "missing required field 'pending'"); + } + if (field_pending) { + auto decoded = decode_value(*field_pending); + if (!decoded) return std::move(decoded).error(); + result.pending = std::move(decoded).value(); + } + const Json* field_retrying = value.find("retrying"); + if (!field_retrying) { + return make_error(ErrorCode::decode, "missing required field 'retrying'"); + } + if (field_retrying) { + auto decoded = decode_value(*field_retrying); + if (!decoded) return std::move(decoded).error(); + result.retrying = std::move(decoded).value(); + } + return result; +} + Result Codec::encode(const ShutdownDaemonResult& value) { (void)value; Json::Object object; diff --git a/cmux-tui/bindings/go/raw/.cmux-sdk-manifest.json b/cmux-tui/bindings/go/raw/.cmux-sdk-manifest.json index 2924b1d007eb..f05b89538227 100644 --- a/cmux-tui/bindings/go/raw/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/go/raw/.cmux-sdk-manifest.json @@ -2,32 +2,32 @@ "files": [ { "path": "generated_commands.go", - "sha256": "cedd6280ba7aecb8c6d75cf62b0b99e4d89b2ca3c35c13a197693c6130fbce85", + "sha256": "c423e10a6b2d58d5ff59d30dbe888903522e4fe2860a33b5896ad36f57c63308", "size": 320630 }, { "path": "generated_events.go", - "sha256": "73d02de8a2aa63b508e1fe3a5aba35f021c19cb280e92f568a43269f2c266774", + "sha256": "f0025f22892c58d21e43736101e2f321769e9994155c1643e45fe611a5a2abd1", "size": 105515 }, { "path": "generated_metadata.go", - "sha256": "160ebc00479c3d552cf4c309eb56cb622d7bd5a418092e6b21ca61f447ffef9b", + "sha256": "f1999a248b2762926d7b92a0279cf929b690aae161ea5283c1c4dc777165cae7", "size": 44205 }, { "path": "generated_presence_test.go", - "sha256": "bcfbc98132e58463e960bc56d566440012734c1c1a59f2fd93b1546c64c48e88", - "size": 871216 + "sha256": "abb05c23a3fb3560b27ed2fa8714fb537164d22aaab98620a0c1768ddf58fa91", + "size": 874464 }, { "path": "generated_types.go", - "sha256": "b224f4290715ede1d7c0eba26e1892b144bbf7cf2be600dc33bde5c9913de341", - "size": 216980 + "sha256": "73f42a31d0f9b0275da1c5cd4597bb92f4f216581c044dc2a01f88e7826cfd7b", + "size": 218705 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "go", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/go/raw/generated_commands.go b/cmux-tui/bindings/go/raw/generated_commands.go index 76a54f020f5c..4d2a704aed95 100644 --- a/cmux-tui/bindings/go/raw/generated_commands.go +++ b/cmux-tui/bindings/go/raw/generated_commands.go @@ -1,5 +1,5 @@ // Code generated by cmux-tui SDK codegen. DO NOT EDIT. -// Mux protocol 10; IR SHA-256 a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// Mux protocol 10; IR SHA-256 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. package raw diff --git a/cmux-tui/bindings/go/raw/generated_events.go b/cmux-tui/bindings/go/raw/generated_events.go index ff993501b15f..dcea108df60f 100644 --- a/cmux-tui/bindings/go/raw/generated_events.go +++ b/cmux-tui/bindings/go/raw/generated_events.go @@ -1,5 +1,5 @@ // Code generated by cmux-tui SDK codegen. DO NOT EDIT. -// Mux protocol 10; IR SHA-256 a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// Mux protocol 10; IR SHA-256 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. package raw diff --git a/cmux-tui/bindings/go/raw/generated_metadata.go b/cmux-tui/bindings/go/raw/generated_metadata.go index 9b58984aee07..6d2c3d1c93fe 100644 --- a/cmux-tui/bindings/go/raw/generated_metadata.go +++ b/cmux-tui/bindings/go/raw/generated_metadata.go @@ -1,12 +1,12 @@ // Code generated by cmux-tui SDK codegen. DO NOT EDIT. -// Mux protocol 10; IR SHA-256 a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// Mux protocol 10; IR SHA-256 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. package raw const ( SDKSchemaVersion = 2 MuxProtocolVersion = 10 - SDKIRSHA256 = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa" + SDKIRSHA256 = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d" ) type Authority string diff --git a/cmux-tui/bindings/go/raw/generated_presence_test.go b/cmux-tui/bindings/go/raw/generated_presence_test.go index ad4e7260ce1c..77a8d1feebff 100644 --- a/cmux-tui/bindings/go/raw/generated_presence_test.go +++ b/cmux-tui/bindings/go/raw/generated_presence_test.go @@ -1,5 +1,5 @@ // Code generated by cmux-tui SDK codegen. DO NOT EDIT. -// Mux protocol 10; IR SHA-256 a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// Mux protocol 10; IR SHA-256 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. package raw @@ -369,6 +369,28 @@ func TestGeneratedSchemaPresenceRoundTrips(t *testing.T) { } assertGeneratedFieldJSON(t, presentValue, "ghostty_commit", true, "\"value\"") }) + t.Run("IdentifyResult.ShutdownCleanup", func(t *testing.T) { + var missing IdentifyResult + if err := json.Unmarshal([]byte("{\"app\":\"cmux-tui\",\"daemon_handoff\":1,\"generation\":\"value\",\"pid\":1,\"protocol\":1,\"registry_id\":\"value\",\"session\":\"value\",\"terminal_revision\":1,\"version\":\"value\",\"workspace_revision\":1}"), &missing); err != nil { + t.Fatal(err) + } + if missing.ShutdownCleanup != nil { + t.Fatal("ShutdownCleanup did not preserve absence") + } + assertGeneratedFieldJSON(t, missing, "shutdown_cleanup", false, "") + var nullValue IdentifyResult + if err := json.Unmarshal([]byte("{\"app\":\"cmux-tui\",\"daemon_handoff\":1,\"generation\":\"value\",\"pid\":1,\"protocol\":1,\"registry_id\":\"value\",\"session\":\"value\",\"terminal_revision\":1,\"version\":\"value\",\"workspace_revision\":1,\"shutdown_cleanup\":null}"), &nullValue); err == nil { + t.Fatal("non-nullable field shutdown_cleanup accepted null") + } + var presentValue IdentifyResult + if err := json.Unmarshal([]byte("{\"app\":\"cmux-tui\",\"daemon_handoff\":1,\"generation\":\"value\",\"pid\":1,\"protocol\":1,\"registry_id\":\"value\",\"session\":\"value\",\"terminal_revision\":1,\"version\":\"value\",\"workspace_revision\":1,\"shutdown_cleanup\":{\"degraded\":true,\"pending\":1,\"retrying\":true}}"), &presentValue); err != nil { + t.Fatal(err) + } + if presentValue.ShutdownCleanup == nil { + t.Fatal("ShutdownCleanup lost its value") + } + assertGeneratedFieldJSON(t, presentValue, "shutdown_cleanup", true, "{\"degraded\":true,\"pending\":1,\"retrying\":true}") + }) t.Run("LayoutSplit.Split", func(t *testing.T) { var missing LayoutSplit if err := json.Unmarshal([]byte("{\"a\":{\"pane\":1,\"type\":\"leaf\"},\"b\":{\"pane\":1,\"type\":\"leaf\"},\"dir\":\"right\",\"ratio\":1.5,\"type\":\"split\"}"), &missing); err != nil { @@ -12058,6 +12080,24 @@ func TestGeneratedRequiredFieldsRejectOmission(t *testing.T) { t.Fatal("missing required field resizes decoded successfully") } }) + t.Run("ShutdownCleanupStatus.Degraded", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"pending\":1,\"retrying\":true}"), &decoded); err == nil { + t.Fatal("missing required field degraded decoded successfully") + } + }) + t.Run("ShutdownCleanupStatus.Pending", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"degraded\":true,\"retrying\":true}"), &decoded); err == nil { + t.Fatal("missing required field pending decoded successfully") + } + }) + t.Run("ShutdownCleanupStatus.Retrying", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"degraded\":true,\"pending\":1}"), &decoded); err == nil { + t.Fatal("missing required field retrying decoded successfully") + } + }) t.Run("ShutdownDaemonResult.Accepted", func(t *testing.T) { var decoded ShutdownDaemonResult if err := json.Unmarshal([]byte("{\"generation\":\"value\",\"pid\":1}"), &decoded); err == nil { @@ -15451,6 +15491,24 @@ func TestGeneratedRequiredNonnullableFieldsRejectNull(t *testing.T) { t.Fatal("required non-nullable field resizes accepted null") } }) + t.Run("ShutdownCleanupStatus.Degraded", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"degraded\":null,\"pending\":1,\"retrying\":true}"), &decoded); err == nil { + t.Fatal("required non-nullable field degraded accepted null") + } + }) + t.Run("ShutdownCleanupStatus.Pending", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"degraded\":true,\"pending\":null,\"retrying\":true}"), &decoded); err == nil { + t.Fatal("required non-nullable field pending accepted null") + } + }) + t.Run("ShutdownCleanupStatus.Retrying", func(t *testing.T) { + var decoded ShutdownCleanupStatus + if err := json.Unmarshal([]byte("{\"degraded\":true,\"pending\":1,\"retrying\":null}"), &decoded); err == nil { + t.Fatal("required non-nullable field retrying accepted null") + } + }) t.Run("ShutdownDaemonResult.Accepted", func(t *testing.T) { var decoded ShutdownDaemonResult if err := json.Unmarshal([]byte("{\"accepted\":null,\"generation\":\"value\",\"pid\":1}"), &decoded); err == nil { @@ -18619,12 +18677,12 @@ func TestGeneratedConstrainedFieldsRejectUnknownValuesOnMarshal(t *testing.T) { } const ( - generatedFieldShapeCount = 431 + generatedFieldShapeCount = 432 generatedOptionalNullableFieldCount = 287 generatedRequiredNullableFieldCount = 61 - generatedOptionalNonnullableFieldCount = 83 - generatedRequiredFieldCount = 595 - generatedRequiredNonnullableFieldCount = 534 + generatedOptionalNonnullableFieldCount = 84 + generatedRequiredFieldCount = 598 + generatedRequiredNonnullableFieldCount = 537 generatedConstrainedFieldCount = 72 generatedEncodedConstrainedFieldCount = 72 ) diff --git a/cmux-tui/bindings/go/raw/generated_types.go b/cmux-tui/bindings/go/raw/generated_types.go index 9b014309fb25..59990903ae0c 100644 --- a/cmux-tui/bindings/go/raw/generated_types.go +++ b/cmux-tui/bindings/go/raw/generated_types.go @@ -1,5 +1,5 @@ // Code generated by cmux-tui SDK codegen. DO NOT EDIT. -// Mux protocol 10; IR SHA-256 a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// Mux protocol 10; IR SHA-256 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. package raw @@ -1833,6 +1833,7 @@ type IdentifyResult struct { Protocol uint32 `json:"protocol"` RegistryID string `json:"registry_id"` Session string `json:"session"` + ShutdownCleanup *ShutdownCleanupStatus `json:"shutdown_cleanup,omitempty"` TerminalRevision uint64 `json:"terminal_revision"` Version string `json:"version"` WorkspaceRevision uint64 `json:"workspace_revision"` @@ -1890,19 +1891,20 @@ func (value *IdentifyResult) UnmarshalJSON(data []byte) error { return fmt.Errorf("decode IdentifyResult: expected object") } var fields struct { - App *IdentifyResultApp `json:"app"` - BuildCommit Presence[string] `json:"build_commit"` - Capabilities optionalNonNullJSON[[]string] `json:"capabilities"` - DaemonHandoff *IdentifyResultDaemonHandoff `json:"daemon_handoff"` - Generation *string `json:"generation"` - GhosttyCommit Presence[string] `json:"ghostty_commit"` - PID *uint32 `json:"pid"` - Protocol *uint32 `json:"protocol"` - RegistryID *string `json:"registry_id"` - Session *string `json:"session"` - TerminalRevision *uint64 `json:"terminal_revision"` - Version *string `json:"version"` - WorkspaceRevision *uint64 `json:"workspace_revision"` + App *IdentifyResultApp `json:"app"` + BuildCommit Presence[string] `json:"build_commit"` + Capabilities optionalNonNullJSON[[]string] `json:"capabilities"` + DaemonHandoff *IdentifyResultDaemonHandoff `json:"daemon_handoff"` + Generation *string `json:"generation"` + GhosttyCommit Presence[string] `json:"ghostty_commit"` + PID *uint32 `json:"pid"` + Protocol *uint32 `json:"protocol"` + RegistryID *string `json:"registry_id"` + Session *string `json:"session"` + ShutdownCleanup optionalNonNullJSON[ShutdownCleanupStatus] `json:"shutdown_cleanup"` + TerminalRevision *uint64 `json:"terminal_revision"` + Version *string `json:"version"` + WorkspaceRevision *uint64 `json:"workspace_revision"` } if err := json.Unmarshal(data, &fields); err != nil { return fmt.Errorf("decode IdentifyResult: %w", err) @@ -1955,6 +1957,12 @@ func (value *IdentifyResult) UnmarshalJSON(data []byte) error { return fmt.Errorf("decode IdentifyResult: required field session is missing or null") } decoded.Session = *fields.Session + if fields.ShutdownCleanup.null { + return fmt.Errorf("decode IdentifyResult: non-nullable field shutdown_cleanup is null") + } + if fields.ShutdownCleanup.set { + decoded.ShutdownCleanup = &fields.ShutdownCleanup.value + } if fields.TerminalRevision == nil { return fmt.Errorf("decode IdentifyResult: required field terminal_revision is missing or null") } @@ -4620,6 +4628,42 @@ func (value *SetCellPixelsResult) UnmarshalJSON(data []byte) error { return nil } +type ShutdownCleanupStatus struct { + Degraded bool `json:"degraded"` + Pending uint64 `json:"pending"` + Retrying bool `json:"retrying"` +} + +func (value *ShutdownCleanupStatus) UnmarshalJSON(data []byte) error { + if !isJSONObject(data) { + return fmt.Errorf("decode ShutdownCleanupStatus: expected object") + } + var fields struct { + Degraded *bool `json:"degraded"` + Pending *uint64 `json:"pending"` + Retrying *bool `json:"retrying"` + } + if err := json.Unmarshal(data, &fields); err != nil { + return fmt.Errorf("decode ShutdownCleanupStatus: %w", err) + } + type wire ShutdownCleanupStatus + var decoded wire + if fields.Degraded == nil { + return fmt.Errorf("decode ShutdownCleanupStatus: required field degraded is missing or null") + } + decoded.Degraded = *fields.Degraded + if fields.Pending == nil { + return fmt.Errorf("decode ShutdownCleanupStatus: required field pending is missing or null") + } + decoded.Pending = *fields.Pending + if fields.Retrying == nil { + return fmt.Errorf("decode ShutdownCleanupStatus: required field retrying is missing or null") + } + decoded.Retrying = *fields.Retrying + *value = ShutdownCleanupStatus(decoded) + return nil +} + type ShutdownDaemonResult struct { Accepted bool `json:"accepted"` Generation string `json:"generation"` diff --git a/cmux-tui/bindings/java/src/com/cmux/raw/.cmux-sdk-manifest.json b/cmux-tui/bindings/java/src/com/cmux/raw/.cmux-sdk-manifest.json index 1678353efd6c..8eb8e2be42d5 100644 --- a/cmux-tui/bindings/java/src/com/cmux/raw/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/java/src/com/cmux/raw/.cmux-sdk-manifest.json @@ -467,8 +467,8 @@ }, { "path": "IdentifyResult.java", - "sha256": "c15df399314639229bf9e16b856613e1690f0a6e1b1f94450696c30d6e3be3ec", - "size": 10038 + "sha256": "7eca17b7d1ee7967fe3ca60175312ca169c9bf1251a848187fd68660907a0532", + "size": 10847 }, { "path": "IdsRequest.java", @@ -737,7 +737,7 @@ }, { "path": "Protocol.java", - "sha256": "10bb781385b0b61f6951cf8de0f09c9229c44a9c6c5a6aec154cb94b52bb2bd7", + "sha256": "cb2ba912ffe131f6a3796cfe8fe1eb62c0914ec82fbc7340971c22ab8566d33d", "size": 3967 }, { @@ -1025,6 +1025,11 @@ "sha256": "5eac6261d6384c422c517ca06b9f05034be307b9b8747830dcbb7f1c1ad8e9f5", "size": 2021 }, + { + "path": "ShutdownCleanupStatus.java", + "sha256": "92f639034ae93e376ac02c7d935a5190ce177757cb32418af096cdaf7f74b545", + "size": 3389 + }, { "path": "ShutdownDaemonRequest.java", "sha256": "b60b647ed6dc21658f10c7428df62efe67b766309b4878336d6eef4d360ec051", @@ -1317,7 +1322,7 @@ } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "java", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/java/src/com/cmux/raw/IdentifyResult.java b/cmux-tui/bindings/java/src/com/cmux/raw/IdentifyResult.java index f8b725898b6f..587c3140ea98 100644 --- a/cmux-tui/bindings/java/src/com/cmux/raw/IdentifyResult.java +++ b/cmux-tui/bindings/java/src/com/cmux/raw/IdentifyResult.java @@ -19,6 +19,7 @@ public final class IdentifyResult implements WireValue { private final long protocol; private final String registryId; private final String session; + private final Field shutdownCleanup; private final UInt64 terminalRevision; private final String version; private final UInt64 workspaceRevision; @@ -37,6 +38,7 @@ private IdentifyResult(Builder builder) { this.registryId = Wire.nonNull(builder.registryId, "registry_id"); if (!builder.sessionSet) throw new IllegalArgumentException("session is required"); this.session = Wire.nonNull(builder.session, "session"); + this.shutdownCleanup = builder.shutdownCleanup; if (!builder.terminalRevisionSet) throw new IllegalArgumentException("terminal_revision is required"); this.terminalRevision = Wire.nonNull(builder.terminalRevision, "terminal_revision"); if (!builder.versionSet) throw new IllegalArgumentException("version is required"); @@ -57,6 +59,7 @@ private IdentifyResult(Builder builder) { public long protocol() { return protocol; } public String registryId() { return registryId; } public String session() { return session; } + public Field shutdownCleanup() { return shutdownCleanup; } public UInt64 terminalRevision() { return terminalRevision; } public String version() { return version; } public UInt64 workspaceRevision() { return workspaceRevision; } @@ -90,6 +93,10 @@ public static IdentifyResult fromWire(Object value) { builder.registryId(Wire.string(rawRegistryId, "IdentifyResult.registry_id")); Object rawSession = Wire.required(object, "session"); builder.session(Wire.string(rawSession, "IdentifyResult.session")); + Object rawShutdownCleanup = Wire.optional(object, "shutdown_cleanup"); + if (!Wire.isMissing(rawShutdownCleanup)) { + builder.shutdownCleanup(ShutdownCleanupStatus.fromWire(rawShutdownCleanup)); + } Object rawTerminalRevision = Wire.required(object, "terminal_revision"); builder.terminalRevision(Wire.uint64(rawTerminalRevision, "IdentifyResult.terminal_revision")); Object rawVersion = Wire.required(object, "version"); @@ -112,6 +119,7 @@ public Map toWire() { Wire.put(object, "protocol", protocol); Wire.put(object, "registry_id", registryId); Wire.put(object, "session", session); + Wire.put(object, "shutdown_cleanup", shutdownCleanup); Wire.put(object, "terminal_revision", terminalRevision); Wire.put(object, "version", version); Wire.put(object, "workspace_revision", workspaceRevision); @@ -121,11 +129,11 @@ public Map toWire() { @Override public boolean equals(Object other) { if (!(other instanceof IdentifyResult that)) return false; - return Objects.equals(buildCommit, that.buildCommit) && Objects.equals(capabilities, that.capabilities) && Objects.equals(generation, that.generation) && Objects.equals(ghosttyCommit, that.ghosttyCommit) && Objects.equals(pid, that.pid) && Objects.equals(protocol, that.protocol) && Objects.equals(registryId, that.registryId) && Objects.equals(session, that.session) && Objects.equals(terminalRevision, that.terminalRevision) && Objects.equals(version, that.version) && Objects.equals(workspaceRevision, that.workspaceRevision); + return Objects.equals(buildCommit, that.buildCommit) && Objects.equals(capabilities, that.capabilities) && Objects.equals(generation, that.generation) && Objects.equals(ghosttyCommit, that.ghosttyCommit) && Objects.equals(pid, that.pid) && Objects.equals(protocol, that.protocol) && Objects.equals(registryId, that.registryId) && Objects.equals(session, that.session) && Objects.equals(shutdownCleanup, that.shutdownCleanup) && Objects.equals(terminalRevision, that.terminalRevision) && Objects.equals(version, that.version) && Objects.equals(workspaceRevision, that.workspaceRevision); } @Override - public int hashCode() { return Objects.hash(buildCommit, capabilities, generation, ghosttyCommit, pid, protocol, registryId, session, terminalRevision, version, workspaceRevision); } + public int hashCode() { return Objects.hash(buildCommit, capabilities, generation, ghosttyCommit, pid, protocol, registryId, session, shutdownCleanup, terminalRevision, version, workspaceRevision); } @Override public String toString() { return "IdentifyResult" + toWire(); } @@ -144,6 +152,7 @@ public static final class Builder { private boolean registryIdSet; private String session; private boolean sessionSet; + private Field shutdownCleanup = Field.omitted(); private UInt64 terminalRevision; private boolean terminalRevisionSet; private String version; @@ -188,6 +197,10 @@ public Builder session(String value) { this.sessionSet = true; return this; } + public Builder shutdownCleanup(ShutdownCleanupStatus value) { + this.shutdownCleanup = Field.of(value); + return this; + } public Builder terminalRevision(UInt64 value) { this.terminalRevision = value; this.terminalRevisionSet = true; diff --git a/cmux-tui/bindings/java/src/com/cmux/raw/Protocol.java b/cmux-tui/bindings/java/src/com/cmux/raw/Protocol.java index 8e82b2a60965..f0375b718a81 100644 --- a/cmux-tui/bindings/java/src/com/cmux/raw/Protocol.java +++ b/cmux-tui/bindings/java/src/com/cmux/raw/Protocol.java @@ -9,7 +9,7 @@ public final class Protocol { public static final String SDK_VERSION = "1.0.0"; public static final int VERSION = 10; public static final int SCHEMA_VERSION = 2; - public static final String IR_SHA256 = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa"; + public static final String IR_SHA256 = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d"; private Protocol() {} public static ProtocolEvent decodeEvent(Object value) { diff --git a/cmux-tui/bindings/java/src/com/cmux/raw/ShutdownCleanupStatus.java b/cmux-tui/bindings/java/src/com/cmux/raw/ShutdownCleanupStatus.java new file mode 100644 index 000000000000..5a002c53eb62 --- /dev/null +++ b/cmux-tui/bindings/java/src/com/cmux/raw/ShutdownCleanupStatus.java @@ -0,0 +1,91 @@ +// Generated from cmux-tui/spec/sdk-schema.json. DO NOT EDIT. +package com.cmux.raw; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + + +public final class ShutdownCleanupStatus implements WireValue { + private final boolean degraded; + private final UInt64 pending; + private final boolean retrying; + + private ShutdownCleanupStatus(Builder builder) { + if (!builder.degradedSet) throw new IllegalArgumentException("degraded is required"); + this.degraded = builder.degraded; + if (!builder.pendingSet) throw new IllegalArgumentException("pending is required"); + this.pending = Wire.nonNull(builder.pending, "pending"); + if (!builder.retryingSet) throw new IllegalArgumentException("retrying is required"); + this.retrying = builder.retrying; + } + + public static Builder builder() { return new Builder(); } + + public boolean degraded() { return degraded; } + public UInt64 pending() { return pending; } + public boolean retrying() { return retrying; } + + public static ShutdownCleanupStatus fromWire(Object value) { + Map object = Wire.object(value, "ShutdownCleanupStatus"); + Builder builder = builder(); + Object rawDegraded = Wire.required(object, "degraded"); + builder.degraded(Wire.bool(rawDegraded, "ShutdownCleanupStatus.degraded")); + Object rawPending = Wire.required(object, "pending"); + builder.pending(Wire.uint64(rawPending, "ShutdownCleanupStatus.pending")); + Object rawRetrying = Wire.required(object, "retrying"); + builder.retrying(Wire.bool(rawRetrying, "ShutdownCleanupStatus.retrying")); + return builder.build(); + } + + @Override + public Map toWire() { + LinkedHashMap object = new LinkedHashMap<>(); + Wire.put(object, "degraded", degraded); + Wire.put(object, "pending", pending); + Wire.put(object, "retrying", retrying); + return Collections.unmodifiableMap(object); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ShutdownCleanupStatus that)) return false; + return Objects.equals(degraded, that.degraded) && Objects.equals(pending, that.pending) && Objects.equals(retrying, that.retrying); + } + + @Override + public int hashCode() { return Objects.hash(degraded, pending, retrying); } + + @Override + public String toString() { return "ShutdownCleanupStatus" + toWire(); } + + public static final class Builder { + private Boolean degraded; + private boolean degradedSet; + private UInt64 pending; + private boolean pendingSet; + private Boolean retrying; + private boolean retryingSet; + + public Builder degraded(boolean value) { + this.degraded = value; + this.degradedSet = true; + return this; + } + public Builder pending(UInt64 value) { + this.pending = value; + this.pendingSet = true; + return this; + } + public Builder retrying(boolean value) { + this.retrying = value; + this.retryingSet = true; + return this; + } + public ShutdownCleanupStatus build() { return new ShutdownCleanupStatus(this); } + } +} diff --git a/cmux-tui/bindings/python/cmux/raw/_generated/.cmux-sdk-manifest.json b/cmux-tui/bindings/python/cmux/raw/_generated/.cmux-sdk-manifest.json index 10fa9531a48b..ee9486ca4e9d 100644 --- a/cmux-tui/bindings/python/cmux/raw/_generated/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/python/cmux/raw/_generated/.cmux-sdk-manifest.json @@ -7,8 +7,8 @@ }, { "path": "_schema.py", - "sha256": "064d152dcf338b977d21d460b230c42bf1268f7465c45fd816980e6594a9792a", - "size": 136449 + "sha256": "f2834e55191caff45628c82c4b67b2a0aa5268a10a66d9d8334664a1ffdfb715", + "size": 136936 }, { "path": "client.py", @@ -17,22 +17,22 @@ }, { "path": "codec.py", - "sha256": "752f3cce023236711fe66e6e16a9bf5c75abd89fdd4925080b3b00c5d64d7f05", - "size": 26103 + "sha256": "e4c43840a191f23211678abb2fac4ebe9181369563005faf325d63664409f3ef", + "size": 26168 }, { "path": "metadata.py", - "sha256": "91373724168adf5532894a0ae99a7c2797988bba3fe1b5e51a77fc10f9bc8819", + "sha256": "1776c149717fa474060452149c4ff1dafe9c9beed5c4851c6ea4f35b337ae72e", "size": 40973 }, { "path": "models.py", - "sha256": "c0d99209724a7b50c88d97bef9f7b1179f43d08c9fe1e370e820d51b97386e55", - "size": 75064 + "sha256": "2e02f5e1f9b24d4c715a46d119e941f70037da037d1af60782c3c26efbaf80f2", + "size": 75364 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "python", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/python/cmux/raw/_generated/_schema.py b/cmux-tui/bindings/python/cmux/raw/_generated/_schema.py index 41deaa4e9f30..7ef1c897a114 100644 --- a/cmux-tui/bindings/python/cmux/raw/_generated/_schema.py +++ b/cmux-tui/bindings/python/cmux/raw/_generated/_schema.py @@ -5,4 +5,4 @@ import json -SCHEMA = json.loads('{"$schema":"./sdk-schema.schema.json","commands":{"apply-layout":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["layout must contain at least one leaf or stack member.","cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ApplyLayoutResult"},"since":6,"stream":null},"attach-surface":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows must be supplied together.","Browser surfaces reject mode:render."],"fields":{"cols":{"capability":"attach-initial-size","default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"mode":{"default":"bytes","nullable":true,"presence":"optional","since":7,"type":{"kind":"enum","values":["bytes","render"]}},"rows":{"capability":"attach-initial-size","default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":{"event_names":["browser-state","colors-changed","detached","frame","notification","output","overflow","render-delta","render-state","resized","scroll-changed","vt-state"],"kind":"attach","mode_field":"mode","modes":{"browser":["browser-state","frame","notification","scroll-changed","overflow","detached"],"bytes":["vt-state","output","resized","colors-changed","scroll-changed","notification","overflow","detached"],"render":["render-state","render-delta","scroll-changed","overflow","detached"]},"ordering":"Initial state precedes the command response. Later surface events preserve order. A resized replay replaces the previous byte mirror; overflow ends that surface stream.","terminal_event":"detached"}},"browser-activate":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-back":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-forward":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-frame-presented":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Acknowledges the exact rendered browser frame for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-insert-text":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-key":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up"]}},"modifiers":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"windows_virtual_key_code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-key-press":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","One request preserves the atomic press sequence through the bounded input queue."],"request":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"modifiers":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"windows_virtual_key_code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-mouse":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; coordinates are CSS pixels.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"button":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"click_count":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"frame_seq":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up","move"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-mouse-guarded":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Browser surfaces only; coordinates are CSS pixels.","The frame sequence must be the exact presented token for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"button":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"click_count":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up","move"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-navigate":{"authority":"frontend","capability":null,"constraints":["Queue acknowledgement only; observe browser-state for outcome.","Navigation is latest-wins."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-reload":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-wheel":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; values are CSS pixels.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"delta_y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"frame_seq":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-wheel-guarded":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Browser surfaces only; values are CSS pixels.","The frame sequence must be the exact presented token for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"delta_y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"clear-history":{"authority":"control","capability":"clear-history-v1","constraints":["PTY surfaces only.","Failed responses classify error_delivery as known-not-delivered or ambiguous."],"request":{"additional_properties":false,"fields":{"fallback_key":{"capability":"clear-history-key-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"TerminalKeyInput"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"clear-window-title":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"close-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-provider-managed-workspace":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Call only after the external provider durably accepts the close."],"request":{"additional_properties":false,"constraints":["workspace and key must identify the same live provider-managed workspace."],"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProviderWorkspaceMutationResult"},"since":9,"stream":null},"close-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-surface":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-terminal":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"CloseTerminalResult"},"since":9,"stream":null},"close-workspace":{"authority":"control","capability":null,"constraints":["Provider-managed workspaces reject this ordinary mutation."],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"copy":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"mode":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["screen","selection","scrollback"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"CopyResult"},"since":6,"stream":null},"create-terminal":{"authority":"control","capability":"workspace-registry-v1","constraints":[],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; when both are supplied they must identify the same workspace.","argv and command are mutually exclusive and must be nonempty when supplied.","cols and rows must be supplied together.","origin and mutation_id are either both present or both absent."],"fields":{"argv":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"command":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"key":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"terminal_id":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"TerminalPlacement"},"since":7,"stream":null},"create-workspace":{"authority":"control","capability":"workspace-registry-v1","constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent.","At most 4096 live workspaces may exist; tombstoned keys cannot be reused."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"key":{"constraints":[{"format":"lowercase canonical UUID"}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":7,"stream":null},"detach-client":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"export-layout":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"screen":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ExportLayoutResult"},"since":6,"stream":null},"focus-direction":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"FocusDirectionResult"},"since":6,"stream":null},"focus-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"get-cell-pixels":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"GetCellPixelsResult"},"since":6,"stream":null},"get-frontend-projection":{"authority":"control","capability":null,"constraints":["Each identifier is nonempty, contains no control character, and is at most 128 bytes."],"request":{"additional_properties":false,"fields":{"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"FrontendProjection"},"since":7,"stream":null},"identify":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"IdentifyResult"},"since":5,"stream":null},"ids":{"authority":"control","capability":null,"constraints":["Short ids are snapshot-local labels; command parameters accept numeric ids only."],"request":{"additional_properties":false,"fields":{"kind":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"enum","values":["workspace","screen","pane","surface"]}}},"kind":"object"},"result":{"kind":"ref","name":"IdsResult"},"since":6,"stream":null},"list-agents":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"state":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"AgentState"}},"surface":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ListAgentsResult"},"since":6,"stream":null},"list-clients":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"items":{"kind":"ref","name":"ClientInfo"},"kind":"array"},"since":6,"stream":null},"list-terminals":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"ListTerminalsResult"},"since":9,"stream":null},"list-workspaces":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"Tree"},"since":5,"stream":null},"mark-workspaces-provider-managed":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Authority must match the value provisioned before this mux generation accepted control clients."],"request":{"additional_properties":false,"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"mint-terminal-renderer":{"authority":"frontend","capability":null,"constraints":["Only terminal-host-backed PTYs can mint one-use renderer credentials."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"ttl_ms":{"constraints":[{"maximum":60000,"minimum":1}],"default":30000,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"MintTerminalRendererResult"},"since":9,"stream":null},"move-tab":{"authority":"control","capability":null,"constraints":["An out-of-range index clamps to the destination end."],"request":{"additional_properties":false,"fields":{"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"move-terminal":{"authority":"control","capability":null,"constraints":["A move to the current workspace still commits a terminal revision with changed:false."],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"MoveTerminalResult"},"since":9,"stream":null},"move-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"new-browser-tab":{"authority":"control","capability":null,"constraints":["Bootstrap and navigation failures are asynchronous browser-state outcomes."],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":9,"stream":null},"new-pane-right":{"authority":"control","capability":"viewport-splits-v1","constraints":[],"request":{"additional_properties":false,"constraints":["Omitted width defaults to two thirds.","cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"width":{"constraints":[{"maximum":1.0,"minimum":0.1}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":9,"stream":null},"new-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-tab":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"notify":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"body":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"level":{"default":"info","nullable":true,"presence":"optional","type":{"kind":"ref","name":"NotificationLevel"}},"surface":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"title":{"constraints":[{"min_length":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"NotifyResult"},"since":6,"stream":null},"pairing-response":{"authority":"local-admin","capability":null,"constraints":["The request id must identify a live, unexpired pairing challenge."],"request":{"additional_properties":false,"fields":{"approve":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":7,"stream":null},"pane-neighbor":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"PaneNeighborResult"},"since":6,"stream":null},"ping":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"PingResult"},"since":6,"stream":null},"process-info":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProcessInfoResult"},"since":6,"stream":null},"put-frontend-projection":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent.","Serialized projection must be at most 1048576 bytes."],"fields":{"expected_generation":{"default":null,"description":"Accepted by the current decoder but ignored for projection writes.","nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_projection_revision":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"description":"Accepted by the current decoder but ignored for projection writes.","nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"projection":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"schema_version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"FrontendProjection"},"since":7,"stream":null},"read-screen":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReadScreenResult"},"since":5,"stream":null},"read-scrollback":{"authority":"control","capability":null,"constraints":["PTY surfaces only; row indexes are snapshot-relative and not durable."],"request":{"additional_properties":false,"fields":{"count":{"constraints":[{"maximum":65535,"minimum":0}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"start":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReadScrollbackResult"},"since":7,"stream":null},"release-surface-size":{"authority":"control","capability":null,"constraints":["An absent lease is a successful no-op."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":7,"stream":null},"reload-config":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"additional_properties":false,"fields":{"path":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"reloaded":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}}},"kind":"object"},"since":6,"stream":null},"rename-pane":{"authority":"control","capability":null,"constraints":["An empty name clears the pane name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-provider-managed-workspace":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Call only after the external provider durably accepts the rename."],"request":{"additional_properties":false,"constraints":["workspace and key must identify the same live provider-managed workspace."],"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProviderWorkspaceMutationResult"},"since":9,"stream":null},"rename-screen":{"authority":"control","capability":null,"constraints":["An empty name clears the screen name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-surface":{"authority":"control","capability":null,"constraints":["An empty name clears the surface name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-workspace":{"authority":"control","capability":null,"constraints":["Provider-managed workspaces reject this ordinary mutation."],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"report-agent":{"authority":"control","capability":null,"constraints":["A stored hook report outranks later socket reports until another hook report or surface close."],"request":{"additional_properties":false,"fields":{"session":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentReportSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReportAgentResult"},"since":6,"stream":null},"resize-surface":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"cols":{"constraints":[{"clamped_maximum":10000,"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"constraints":[{"clamped_maximum":10000,"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ResizeSurfaceResult"},"since":5,"stream":null},"resolve-terminal":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"terminal_id":{"constraints":[{"format":"UUIDv4 hex without dashes","pattern":"^[0-9a-f]{32}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"ResolveTerminalResult"},"since":9,"stream":null},"run":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["Exactly one of argv and command must be supplied.","pane and new_workspace:true are mutually exclusive.","key is valid only with new_workspace:true.","cols and rows affect sizing only when both are present."],"fields":{"argv":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"command":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"key":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"new_workspace":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"RunResult"},"since":6,"stream":null},"scroll-surface":{"authority":"control","capability":null,"constraints":["PTY surfaces only; negative values scroll up."],"request":{"additional_properties":false,"fields":{"delta":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-tab":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"send":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both are present, UTF-8 text bytes precede decoded bytes."],"fields":{"bytes":{"constraints":[{"encoding":"standard base64"}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Base64"}},"paste":{"default":false,"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"send-key":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"keys":{"constraints":[{"syntax":"lowercase modifier+key chords"}],"nullable":false,"presence":"required","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"set-cell-pixels":{"authority":"frontend","capability":null,"constraints":["Accepted browser resizes complete asynchronously."],"request":{"additional_properties":false,"fields":{"height_px":{"constraints":[{"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"width_px":{"constraints":[{"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SetCellPixelsResult"},"since":6,"stream":null},"set-client-info":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"capabilities":{"constraints":["Advertises additive client capabilities for this connection."],"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array"}},"kind":{"constraints":["Control characters become spaces; at most 64 Unicode characters are retained."],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"constraints":["Control characters become spaces; at most 64 Unicode characters are retained."],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"set-client-sizing":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["exclusive:true requires client and enabled:true.","Omitting client is valid only with enabled:true and restores all clients for the surface."],"fields":{"client":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"enabled":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"exclusive":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"set-default-colors":{"authority":"control","capability":null,"constraints":["Color strings are exactly #rrggbb.","With complete:true, absent optional values reset to built-in defaults."],"request":{"additional_properties":false,"fields":{"bg":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"complete":{"default":false,"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"cursor":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"CursorStyle"}},"fg":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"palette":{"constraints":["Decimal string keys are palette indexes 0 through 255."],"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"set-ratio":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"set-split-ratio":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"split":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"transaction":{"capability":"layout-undo-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":8,"stream":null},"set-viewport-pane-width":{"authority":"control","capability":"viewport-column-resize-v1","constraints":["width must be finite."],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"transaction":{"capability":"layout-undo-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"width":{"constraints":[{"maximum":1.0,"minimum":0.1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"set-window-title":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"title":{"constraints":["C0 controls are sanitized before OSC output."],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"shutdown":{"authority":"local-admin","capability":"server-shutdown-v1","constraints":["The command is accepted only on a trusted Unix-classified transport.","Success means every tracked pane process terminated before the bounded shutdown deadline."],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"shutdown-daemon":{"authority":"local-admin","capability":null,"constraints":["pid and generation must match the latest identify result.","force bypasses native-browser ownership only; the identity fence and trusted-local authority still apply.","Clients must require daemon-handoff-force-v1 before sending force:true.","The daemon exits only after the success response is queued."],"request":{"additional_properties":false,"fields":{"force":{"capability":"daemon-handoff-force-v1","default":false,"nullable":false,"presence":"optional","since":10,"type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"ShutdownDaemonResult"},"since":9,"stream":null},"sidebar-plugin":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"relaunch":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SidebarPluginResult"},"since":6,"stream":null},"split":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"subscribe":{"authority":"frontend","capability":null,"constraints":["subscribe sends no initial tree snapshot.","surface filtering occurs before the bounded mailbox."],"request":{"additional_properties":false,"fields":{"surface":{"capability":"surface-subscribe-filter","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"Id"}},"tree_events":{"default":"coarse","nullable":true,"presence":"optional","since":7,"type":{"kind":"enum","values":["coarse","deltas"]}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":{"event_names":["bell","client-attached","client-changed","client-detached","client-list-invalidated","config-reload-requested","empty","frontend-projection-changed","layout-changed","notification","overflow","pairing-requested","pairing-resolved","pane-added","pane-closed","screen-added","screen-closed","screen-renamed","scroll-changed","status","surface-exited","surface-output","surface-resize-failed","surface-resized","tab-added","tab-closed","tab-renamed","terminal-registry-changed","title-changed","tree-changed","window-title-requested","workspace-added","workspace-closed","workspace-moved","workspace-renamed"],"kind":"subscribe","mode_field":"tree_events","modes":{"coarse":["tree-changed"],"deltas":["workspace-added","workspace-closed","workspace-renamed","workspace-moved","screen-added","screen-closed","screen-renamed","pane-added","pane-closed","tab-added","tab-closed","tab-renamed","tree-changed"]},"ordering":"Response and event objects may interleave. Events preserve enqueue order per subscription. Delta workspace revisions are serialized in durable commit order; overflow ends the stream and requires resubscribe plus snapshot.","terminal_event":null}},"swap-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["Exactly one of dir and target must be supplied."],"fields":{"dir":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"target":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"terminal-events":{"authority":"control","capability":null,"constraints":["Consumers apply only contiguous revisions for one registry_id and generation."],"request":{"additional_properties":false,"fields":{"after_revision":{"default":0,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"TerminalEventsResult"},"since":9,"stream":null},"undo-layout":{"authority":"control","capability":"layout-undo-v1","constraints":["Clients must reject incomplete or contradictory result variants."],"request":{"additional_properties":false,"constraints":["confirm_close requires the exact preview revision."],"fields":{"confirm_close":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"revision":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"LayoutUndoResult"},"since":9,"stream":null},"vt-state":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"VtStateResult"},"since":5,"stream":null},"wait-for":{"authority":"control","capability":null,"constraints":["Blocks subsequent requests on this connection; SDKs should use a dedicated connection."],"request":{"additional_properties":false,"fields":{"pattern":{"constraints":[{"syntax":"Rust regex"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"timeout_ms":{"description":"Zero performs one immediate check.","nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"WaitForResult"},"since":6,"stream":null},"zoom-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"mode":{"default":"toggle","nullable":true,"presence":"optional","type":{"kind":"enum","values":["toggle","on","off"]}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ZoomPaneResult"},"since":6,"stream":null}},"events":{"bell":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"bell"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"browser-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"browser-state"}},"frame":{"description":"The initial browser-state includes the latest frame when one exists; later state updates omit it.","nullable":true,"presence":"optional","type":{"kind":"ref","name":"BrowserFrame"}},"frames_stalled":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"status":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["starting","live","failed"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["attach-browser"]},"client-attached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-attached"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"transport":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["unix","ws"]}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-changed"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-detached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-detached"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-list-invalidated":{"capability":null,"emission":"serialized-never-emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-list-invalidated"}}},"kind":"object"},"since":9,"streams":["subscribe"]},"colors-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"cursor":{"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"nullable":true,"presence":"optional","type":{"kind":"ref","name":"CursorStyle"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"colors-changed"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"palette":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"surface":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["attach-byte"]},"config-reload-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"config-reload-requested"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"detached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"detached"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte","attach-render","attach-browser"]},"empty":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"empty"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"frame":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"frame"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"since":6,"streams":["attach-browser"]},"frontend-projection-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"frontend-projection-changed"}},"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"projection_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"graphics-status":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["kitty-image-budget-worker-start-failed carries error.","kitty-image-budget-update-failed carries retry_exhausted and summary.","cell-pixel-update-retries-exhausted carries attempts, remaining, cell_width, and cell_height."],"fields":{"attempts":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cell_height":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cell_width":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"graphics-status"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["kitty-image-budget-worker-start-failed","kitty-image-budget-update-failed","cell-pixel-update-retries-exhausted"]}},"remaining":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"retry_exhausted":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"summary":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":10,"streams":["subscribe"]},"layout-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"layout-changed"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"notification":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"body":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"notification"}},"level":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"NotificationLevel"}},"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe","attach-byte","attach-browser"]},"output":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"colors":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"TerminalColors"}},"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"output"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte"]},"overflow":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["scope and surface are either both present for attach overflow or both absent for subscribe overflow."],"fields":{"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"overflow"}},"scope":{"nullable":false,"presence":"optional","type":{"kind":"literal","value":"surface"}},"surface":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe","attach-byte","attach-render","attach-browser"]},"pairing-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pairing-requested"}},"expires_in":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"peer":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"pairing-resolved":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pairing-resolved"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"pane-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Pane"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pane-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"pane-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Pane"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pane-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"render-delta":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["size is present if and only if the surface resized; every resize has full:true."],"fields":{"cursor":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderCursor"}},"default_bg":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"default_fg":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"render-delta"}},"full":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"graphics":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderGraphicsDelta"}},"history_epoch":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"scrollback_rows":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"size":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"Size"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["attach-render"]},"render-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cursor":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderCursor"}},"default_bg":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"default_fg":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"render-state"}},"graphics":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderGraphics"}},"history_epoch":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"scrollback_rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"size":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Size"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["attach-render"]},"resized":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["At least one of replay and data is present; replay is canonical from protocol 7."],"fields":{"colors":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"TerminalColors"}},"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"description":"Protocol 6 compatibility field.","nullable":false,"presence":"optional","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"resized"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"replay":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"Base64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["attach-byte"]},"screen-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"screen-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"screen-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-renamed"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"scroll-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"at_bottom":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"scroll-changed"}},"offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["subscribe","attach-byte","attach-render","attach-browser"]},"status":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"status"}},"message":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-exited":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-exited"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-output":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-output"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-resize-failed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-resize-failed"}},"reservation_id":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"retry_after_ms":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"surface-resized":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-resized"}},"reservation_id":{"nullable":true,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"tab-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"tab-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"tab-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-renamed"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"terminal-registry-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"terminal-registry-changed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"refetch":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"terminal-events-or-list-terminals"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":9,"streams":["subscribe"]},"title-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"title-changed"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"tree-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tree-changed"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"vt-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"colors":{"nullable":false,"presence":"optional","since":6,"type":{"kind":"ref","name":"TerminalColors"}},"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"vt-state"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte"]},"window-title-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"window-title-requested"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"workspace-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-added"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-closed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-moved":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-moved"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-renamed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]}},"ir_sha256":"a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa","profiles":{"control":{"description":"Base authenticated session-control commands available to ordinary SDK clients.","inherits":[]},"frontend":{"description":"Rendering, input, presentation, subscribe, and attach commands.","inherits":["control"]},"local-admin":{"description":"Trusted local administration commands.","inherits":["control"],"transport":"Unix-classified transport, including direct Unix and the current stdio relay"},"provider-authority":{"description":"Provider-owned workspace mutation commands.","inherits":["control"],"requires_authority":true}},"protocol":{"id_type":"uint64","javascript_id_policy":"All protocol identifiers are uint64 JSON numbers. JavaScript and TypeScript SDKs must decode them losslessly as bigint (or validated decimal strings at their public boundary), and must not expose IEEE-754 number ids. Pairing request ids, revisions, timestamps, frame sequences, and reservation ids follow the same rule.","name":"cmux-tui-mux","version":10},"schema_version":2,"types":{"AgentRecord":{"additional_properties":false,"fields":{"session":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"updated_at_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"AgentReportSource":{"kind":"enum","values":["socket","hook"]},"AgentSource":{"kind":"enum","values":["detected","socket","hook"]},"AgentState":{"kind":"enum","values":["working","blocked","idle","done","unknown"]},"AppliedPane":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ApplyLayoutResult":{"additional_properties":false,"fields":{"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"AppliedPane"},"kind":"array"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Base64":{"kind":"alias","target":{"kind":"scalar","name":"string"}},"BrowserFrame":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"CellPixelFailure":{"additional_properties":false,"fields":{"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"CellPixelResize":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"reservation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"CellPixelSurface":{"additional_properties":false,"fields":{"height_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"width_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"ClientInfo":{"additional_properties":false,"fields":{"attached":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}},"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"connected_seconds":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"self":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"sizes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"ClientSize"},"kind":"array"}},"transport":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ClientTransport"}}},"kind":"object"},"ClientSize":{"additional_properties":false,"fields":{"cols":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"size_participating":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ClientTransport":{"kind":"enum","values":["local","unix","ws"]},"CloseTerminalResult":{"additional_properties":false,"fields":{"already_closed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"closed":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ColorHex":{"kind":"alias","target":{"kind":"scalar","name":"string"}},"CopyResult":{"additional_properties":false,"fields":{"mode":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["screen","selection","scrollback"]}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"CursorStyle":{"kind":"enum","values":["block","underline","bar"]},"DeadPane":{"additional_properties":false,"fields":{"dead":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"DeclarativeLayout":{"kind":"tagged_union","tag":"type","variants":{"leaf":{"additional_properties":false,"fields":{"command":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"leaf"}}},"kind":"object"},"split":{"additional_properties":false,"fields":{"a":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"b":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"split"}}},"kind":"object"},"stack":{"additional_properties":false,"fields":{"expanded":{"constraints":["Must identify a member of panes."],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array","min_items":1}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"stack"}}},"kind":"object"}}},"EmptyResult":{"additional_properties":false,"fields":{},"kind":"object"},"ExportLayoutResult":{"additional_properties":false,"fields":{"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"ExportedPane"},"kind":"array"}}},"kind":"object"},"ExportedPane":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surfaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}}},"kind":"object"},"FocusDirectionResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"FrontendProjection":{"additional_properties":false,"fields":{"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"projection":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"projection_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"replayed":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"schema_version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"GetCellPixelsResult":{"additional_properties":false,"fields":{"height_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surfaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelSurface"},"kind":"array"}},"width_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"Id":{"kind":"alias","target":{"kind":"scalar","name":"uint64"}},"IdMapping":{"additional_properties":false,"fields":{"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["workspace","screen","pane","surface"]}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"IdentifyResult":{"additional_properties":false,"fields":{"app":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"cmux-tui"}},"build_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"capabilities":{"default":[],"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array"}},"daemon_handoff":{"nullable":false,"presence":"required","since":9,"type":{"kind":"literal","value":1}},"generation":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"string"}},"ghostty_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"protocol":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"registry_id":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"string"}},"session":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","since":9,"type":{"kind":"scalar","name":"uint64"}},"version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace_revision":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"IdsResult":{"additional_properties":false,"fields":{"ids":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"IdMapping"},"kind":"array"}}},"kind":"object"},"JsonValue":{"kind":"opaque_json","reason":"The wire field intentionally carries a frontend-authored or runtime-authored arbitrary JSON document."},"KittyGraphicsState":{"additional_properties":false,"fields":{"alternate_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"alternate_replay_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_bytes":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"inflight_bytes":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"placements":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"primary_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"primary_replay_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"replay_cursor_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"KittyImageAlias":{"additional_properties":false,"fields":{"image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_number":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"Layout":{"kind":"tagged_union","tag":"type","variants":{"leaf":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"leaf"}}},"kind":"object"},"split":{"additional_properties":false,"fields":{"a":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"b":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"ratio":{"constraints":[{"maximum":0.95,"minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"split":{"description":"Stable for the lifetime of this split node.","nullable":false,"presence":"optional","since":8,"type":{"kind":"ref","name":"Id"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"split"}}},"kind":"object"},"stack":{"additional_properties":false,"fields":{"expanded":{"constraints":["Must identify a member of panes."],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array","min_items":1}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"stack"}}},"kind":"object"}}},"LayoutUndoConfirmationRequired":{"additional_properties":false,"fields":{"closes_panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}},"confirmation_required":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"undone":{"nullable":false,"presence":"required","type":{"kind":"literal","value":false}}},"kind":"object"},"LayoutUndoResult":{"kind":"untagged_union","variants":[{"kind":"ref","name":"LayoutUndoUndone"},{"kind":"ref","name":"LayoutUndoConfirmationRequired"}]},"LayoutUndoUndone":{"additional_properties":false,"fields":{"confirmation_required":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"literal","value":false}},"revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"undone":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}}},"kind":"object"},"ListAgentsResult":{"additional_properties":false,"fields":{"agents":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"AgentRecord"},"kind":"array"}}},"kind":"object"},"ListTerminalsResult":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"terminals":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"TerminalRecord"},"kind":"array"}}},"kind":"object"},"LivePane":{"additional_properties":false,"fields":{"active_tab":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"focused_at":{"default":0,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"tabs":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Tab"},"kind":"array"}}},"kind":"object"},"MintTerminalRendererResult":{"additional_properties":false,"fields":{"endpoint":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"incarnation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"rights":{"constraints":[{"current_value":7}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"token":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"ttl_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"MoveTerminalResult":{"additional_properties":false,"fields":{"changed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"screen":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"NotificationLevel":{"kind":"enum","values":["info","warning","error"]},"NotificationMarker":{"additional_properties":false,"fields":{"level":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"NotificationLevel"}},"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"unread":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}}},"kind":"object"},"NotifyResult":{"additional_properties":false,"fields":{"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Pane":{"kind":"untagged_union","variants":[{"kind":"ref","name":"LivePane"},{"kind":"ref","name":"DeadPane"}]},"PaneDirection":{"kind":"enum","values":["left","right","up","down"]},"PaneNeighborResult":{"additional_properties":false,"fields":{"pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"PingResult":{"additional_properties":false,"fields":{"build_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"ghostty_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"ok":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"protocol":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"ProcessInfoResult":{"additional_properties":false,"fields":{"command":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"cwd":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"ProviderWorkspaceMutationResult":{"additional_properties":false,"fields":{"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ReadScreenResult":{"additional_properties":false,"fields":{"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"ReadScrollbackResult":{"additional_properties":false,"fields":{"epoch":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"start":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"total":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"RenderCursor":{"additional_properties":false,"fields":{"blink":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"color":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"style":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"CursorStyle"}},"visible":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"x":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"y":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"RenderGraphicFormat":{"kind":"enum","values":["rgb","rgba"]},"RenderGraphicImage":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"format":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderGraphicFormat"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"RenderGraphicPlacement":{"additional_properties":false,"fields":{"anchor_col":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"anchor_row":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"columns":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"grid_cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"grid_rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"ordinal":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"pixel_height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"pixel_width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"placement_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_x":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_y":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"viewport_col":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}},"viewport_row":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}},"viewport_visible":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"x_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"y_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"z":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}}},"kind":"object"},"RenderGraphics":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicImage"},"kind":"array"}},"placements":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderGraphicPlacement"},"kind":"array"}},"removed_image_ids":{"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"uint32"},"kind":"array"}}},"kind":"object"},"RenderGraphicsDelta":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicImage"},"kind":"array"}},"placements":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicPlacement"},"kind":"array"}},"removed_image_ids":{"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"uint32"},"kind":"array"}}},"kind":"object"},"RenderRow":{"additional_properties":false,"fields":{"row":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"runs":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRun"},"kind":"array"}}},"kind":"object"},"RenderRun":{"additional_properties":false,"fields":{"attrs":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"underline":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderUnderline"}},"width_hint":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"RenderUnderline":{"kind":"enum","values":["single","double","curly","dotted","dashed"]},"ReportAgentResult":{"additional_properties":false,"fields":{"session":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentReportSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ResizeSurfaceResult":{"additional_properties":false,"fields":{"accepted":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"reservation_id":{"nullable":true,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ResolveTerminalResult":{"additional_properties":false,"fields":{"exit":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"launch_spec":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"RunResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"required","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","since":9,"type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Screen":{"additional_properties":false,"fields":{"active":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"active_pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Pane"},"kind":"array"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"zoomed_pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"SetCellPixelsResult":{"additional_properties":false,"fields":{"failures":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelFailure"},"kind":"array"}},"resizes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelResize"},"kind":"array"}}},"kind":"object"},"ShutdownDaemonResult":{"additional_properties":false,"fields":{"accepted":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"SidebarPluginResult":{"additional_properties":false,"fields":{"error":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"retry_after_ms":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Size":{"additional_properties":false,"fields":{"cols":{"constraints":[{"maximum":10000,"minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"constraints":[{"maximum":10000,"minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"SplitDirection":{"kind":"enum","values":["right","down"]},"SurfaceResult":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Tab":{"additional_properties":false,"fields":{"browser_error":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"browser_frames_stalled":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"boolean"}},"browser_source":{"nullable":true,"presence":"required","type":{"kind":"enum","values":["external","launched"]}},"browser_status":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"enum","values":["starting","live","failed"]}},"dead":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["pty","browser"]}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"notification":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"NotificationMarker"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"size":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Size"}},"supports_clear_history_key_fallback":{"capability":"clear-history-key-v1","nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_resource_id":{"constraints":[{"pattern":"^term_[0-9a-f]{32}$"}],"nullable":true,"presence":"optional","since":10,"type":{"kind":"scalar","name":"string"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalColors":{"additional_properties":false,"fields":{"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"cursor":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"CursorStyle"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"palette":{"constraints":["Decimal string keys are palette indexes 0 through 255."],"nullable":false,"presence":"optional","since":7,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}}},"kind":"object"},"TerminalEventsResult":{"additional_properties":false,"fields":{"events":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"TerminalRegistryEvent"},"kind":"array"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"TerminalKey":{"kind":"enum","values":["unidentified","backquote","backslash","bracket-left","bracket-right","comma","digit0","digit1","digit2","digit3","digit4","digit5","digit6","digit7","digit8","digit9","equal","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","minus","period","quote","semicolon","slash","backspace","enter","space","tab","delete","end","home","insert","page-down","page-up","arrow-down","arrow-left","arrow-right","arrow-up","numpad0","numpad1","numpad2","numpad3","numpad4","numpad5","numpad6","numpad7","numpad8","numpad9","numpad-add","numpad-backspace","numpad-comma","numpad-decimal","numpad-divide","numpad-enter","numpad-equal","numpad-multiply","numpad-subtract","numpad-up","numpad-down","numpad-right","numpad-left","numpad-begin","numpad-home","numpad-end","numpad-insert","numpad-delete","numpad-page-up","numpad-page-down","escape","f1","f2","f3","f4","f5","f6","f7","f8","f9","f10","f11","f12","f13","f14","f15","f16","f17","f18","f19","f20"]},"TerminalKeyAction":{"kind":"enum","values":["press","release","repeat"]},"TerminalKeyInput":{"additional_properties":false,"constraints":["consumed_mods must be a subset of mods.","unshifted_codepoint, shifted_codepoint, and base_layout_codepoint contain exactly one Unicode scalar when present.","utf8 contains no control characters.","macos_option_as_alt may be false only when Alt is active and consumed."],"fields":{"action":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"TerminalKeyAction"}},"base_layout_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"composing":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"consumed_mods":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalModifiers"}},"key":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalKey"}},"macos_option_as_alt":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"mods":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalModifiers"}},"shifted_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"unshifted_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"utf8":{"constraints":[{"max_length":4096}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalLifecycle":{"kind":"enum","values":["launching","adopting","running","exited","tombstoned"]},"TerminalModifiers":{"additional_properties":false,"fields":{"alt":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"caps_lock":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"control":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"num_lock":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"shift":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"super":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}}},"kind":"object"},"TerminalPlacement":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"lifecycle":{"nullable":true,"presence":"required","type":{"kind":"literal","value":"running"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"TerminalRecord":{"additional_properties":false,"fields":{"exit":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"launch_spec":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"terminal_id":{"constraints":[{"format":"UUIDv4 hex without dashes","pattern":"^[0-9a-f]{32}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalRegistryEvent":{"additional_properties":false,"fields":{"kind":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"result":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Tree":{"additional_properties":false,"fields":{"generation":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"pane_revision":{"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"registry_id":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"workspace_revision":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"workspaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Workspace"},"kind":"array"}}},"kind":"object"},"VtStateResult":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"constraints":[{"encoding":"standard base64"}],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"WaitForResult":{"additional_properties":false,"fields":{"elapsed_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"matched":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Workspace":{"additional_properties":false,"fields":{"active":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"key":{"capability":"workspace-registry-v1","constraints":[{"format":"lowercase canonical UUID"}],"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"screens":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Screen"},"kind":"array"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"WorkspaceMutationResult":{"additional_properties":false,"fields":{"changed":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ZoomPaneResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"zoomed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"zoomed_pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"}}}') +SCHEMA = json.loads('{"$schema":"./sdk-schema.schema.json","commands":{"apply-layout":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["layout must contain at least one leaf or stack member.","cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ApplyLayoutResult"},"since":6,"stream":null},"attach-surface":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows must be supplied together.","Browser surfaces reject mode:render."],"fields":{"cols":{"capability":"attach-initial-size","default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"mode":{"default":"bytes","nullable":true,"presence":"optional","since":7,"type":{"kind":"enum","values":["bytes","render"]}},"rows":{"capability":"attach-initial-size","default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":{"event_names":["browser-state","colors-changed","detached","frame","notification","output","overflow","render-delta","render-state","resized","scroll-changed","vt-state"],"kind":"attach","mode_field":"mode","modes":{"browser":["browser-state","frame","notification","scroll-changed","overflow","detached"],"bytes":["vt-state","output","resized","colors-changed","scroll-changed","notification","overflow","detached"],"render":["render-state","render-delta","scroll-changed","overflow","detached"]},"ordering":"Initial state precedes the command response. Later surface events preserve order. A resized replay replaces the previous byte mirror; overflow ends that surface stream.","terminal_event":"detached"}},"browser-activate":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-back":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-forward":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-frame-presented":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Acknowledges the exact rendered browser frame for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-insert-text":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-key":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up"]}},"modifiers":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"windows_virtual_key_code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-key-press":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only.","One request preserves the atomic press sequence through the bounded input queue."],"request":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"modifiers":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"windows_virtual_key_code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-mouse":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; coordinates are CSS pixels.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"button":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"click_count":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"frame_seq":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up","move"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-mouse-guarded":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Browser surfaces only; coordinates are CSS pixels.","The frame sequence must be the exact presented token for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"button":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"click_count":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["down","up","move"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"browser-navigate":{"authority":"frontend","capability":null,"constraints":["Queue acknowledgement only; observe browser-state for outcome.","Navigation is latest-wins."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-reload":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; queue acknowledgement is not page-load success."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-wheel":{"authority":"frontend","capability":null,"constraints":["Browser surfaces only; values are CSS pixels.","The bounded disposable input queue drops newest input when full."],"request":{"additional_properties":false,"fields":{"delta_y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"frame_seq":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"browser-wheel-guarded":{"authority":"frontend","capability":"browser-pointer-frame-guard-v1","constraints":["Browser surfaces only; values are CSS pixels.","The frame sequence must be the exact presented token for this connection.","Requires browser-pointer-frame-guard-v1."],"request":{"additional_properties":false,"fields":{"delta_y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"frame_seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"x_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}},"y_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"clear-history":{"authority":"control","capability":"clear-history-v1","constraints":["PTY surfaces only.","Failed responses classify error_delivery as known-not-delivered or ambiguous."],"request":{"additional_properties":false,"fields":{"fallback_key":{"capability":"clear-history-key-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"TerminalKeyInput"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"clear-window-title":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"close-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-provider-managed-workspace":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Call only after the external provider durably accepts the close."],"request":{"additional_properties":false,"constraints":["workspace and key must identify the same live provider-managed workspace."],"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProviderWorkspaceMutationResult"},"since":9,"stream":null},"close-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-surface":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"close-terminal":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"CloseTerminalResult"},"since":9,"stream":null},"close-workspace":{"authority":"control","capability":null,"constraints":["Provider-managed workspaces reject this ordinary mutation."],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"copy":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"mode":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["screen","selection","scrollback"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"CopyResult"},"since":6,"stream":null},"create-terminal":{"authority":"control","capability":"workspace-registry-v1","constraints":[],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; when both are supplied they must identify the same workspace.","argv and command are mutually exclusive and must be nonempty when supplied.","cols and rows must be supplied together.","origin and mutation_id are either both present or both absent."],"fields":{"argv":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"command":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"key":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"terminal_id":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"TerminalPlacement"},"since":7,"stream":null},"create-workspace":{"authority":"control","capability":"workspace-registry-v1","constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent.","At most 4096 live workspaces may exist; tombstoned keys cannot be reused."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"key":{"constraints":[{"format":"lowercase canonical UUID"}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":7,"stream":null},"detach-client":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"export-layout":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"screen":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ExportLayoutResult"},"since":6,"stream":null},"focus-direction":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"FocusDirectionResult"},"since":6,"stream":null},"focus-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"get-cell-pixels":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"GetCellPixelsResult"},"since":6,"stream":null},"get-frontend-projection":{"authority":"control","capability":null,"constraints":["Each identifier is nonempty, contains no control character, and is at most 128 bytes."],"request":{"additional_properties":false,"fields":{"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"FrontendProjection"},"since":7,"stream":null},"identify":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"IdentifyResult"},"since":5,"stream":null},"ids":{"authority":"control","capability":null,"constraints":["Short ids are snapshot-local labels; command parameters accept numeric ids only."],"request":{"additional_properties":false,"fields":{"kind":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"enum","values":["workspace","screen","pane","surface"]}}},"kind":"object"},"result":{"kind":"ref","name":"IdsResult"},"since":6,"stream":null},"list-agents":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"state":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"AgentState"}},"surface":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ListAgentsResult"},"since":6,"stream":null},"list-clients":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"items":{"kind":"ref","name":"ClientInfo"},"kind":"array"},"since":6,"stream":null},"list-terminals":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"ListTerminalsResult"},"since":9,"stream":null},"list-workspaces":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"Tree"},"since":5,"stream":null},"mark-workspaces-provider-managed":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Authority must match the value provisioned before this mux generation accepted control clients."],"request":{"additional_properties":false,"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"mint-terminal-renderer":{"authority":"frontend","capability":null,"constraints":["Only terminal-host-backed PTYs can mint one-use renderer credentials."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"ttl_ms":{"constraints":[{"maximum":60000,"minimum":1}],"default":30000,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"MintTerminalRendererResult"},"since":9,"stream":null},"move-tab":{"authority":"control","capability":null,"constraints":["An out-of-range index clamps to the destination end."],"request":{"additional_properties":false,"fields":{"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"move-terminal":{"authority":"control","capability":null,"constraints":["A move to the current workspace still commits a terminal revision with changed:false."],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"MoveTerminalResult"},"since":9,"stream":null},"move-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"new-browser-tab":{"authority":"control","capability":null,"constraints":["Bootstrap and navigation failures are asynchronous browser-state outcomes."],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":9,"stream":null},"new-pane-right":{"authority":"control","capability":"viewport-splits-v1","constraints":[],"request":{"additional_properties":false,"constraints":["Omitted width defaults to two thirds.","cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"width":{"constraints":[{"maximum":1.0,"minimum":0.1}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":9,"stream":null},"new-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-tab":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"new-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"notify":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"body":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"level":{"default":"info","nullable":true,"presence":"optional","type":{"kind":"ref","name":"NotificationLevel"}},"surface":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"title":{"constraints":[{"min_length":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"NotifyResult"},"since":6,"stream":null},"pairing-response":{"authority":"local-admin","capability":null,"constraints":["The request id must identify a live, unexpired pairing challenge."],"request":{"additional_properties":false,"fields":{"approve":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":7,"stream":null},"pane-neighbor":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"PaneNeighborResult"},"since":6,"stream":null},"ping":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"PingResult"},"since":6,"stream":null},"process-info":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProcessInfoResult"},"since":6,"stream":null},"put-frontend-projection":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent.","Serialized projection must be at most 1048576 bytes."],"fields":{"expected_generation":{"default":null,"description":"Accepted by the current decoder but ignored for projection writes.","nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"expected_projection_revision":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"description":"Accepted by the current decoder but ignored for projection writes.","nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"projection":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"schema_version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"FrontendProjection"},"since":7,"stream":null},"read-screen":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReadScreenResult"},"since":5,"stream":null},"read-scrollback":{"authority":"control","capability":null,"constraints":["PTY surfaces only; row indexes are snapshot-relative and not durable."],"request":{"additional_properties":false,"fields":{"count":{"constraints":[{"maximum":65535,"minimum":0}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"start":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReadScrollbackResult"},"since":7,"stream":null},"release-surface-size":{"authority":"control","capability":null,"constraints":["An absent lease is a successful no-op."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":7,"stream":null},"reload-config":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"additional_properties":false,"fields":{"path":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"reloaded":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}}},"kind":"object"},"since":6,"stream":null},"rename-pane":{"authority":"control","capability":null,"constraints":["An empty name clears the pane name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-provider-managed-workspace":{"authority":"provider-authority","capability":"provider-managed-workspace-authority-v2","constraints":["Call only after the external provider durably accepts the rename."],"request":{"additional_properties":false,"constraints":["workspace and key must identify the same live provider-managed workspace."],"fields":{"authority":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ProviderWorkspaceMutationResult"},"since":9,"stream":null},"rename-screen":{"authority":"control","capability":null,"constraints":["An empty name clears the screen name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-surface":{"authority":"control","capability":null,"constraints":["An empty name clears the surface name."],"request":{"additional_properties":false,"fields":{"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"rename-workspace":{"authority":"control","capability":null,"constraints":["Provider-managed workspaces reject this ordinary mutation."],"request":{"additional_properties":false,"constraints":["At least one of workspace and key must be supplied; both must identify the same workspace when supplied.","origin and mutation_id are either both present or both absent."],"fields":{"expected_generation":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"expected_revision":{"aliases":["expected_terminal_revision"],"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"key":{"capability":"workspace-registry-v1","default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"mutation_id":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"name":{"constraints":[{"max_utf8_bytes":1024}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"default":null,"nullable":true,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"workspace":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"WorkspaceMutationResult"},"since":5,"stream":null},"report-agent":{"authority":"control","capability":null,"constraints":["A stored hook report outranks later socket reports until another hook report or surface close."],"request":{"additional_properties":false,"fields":{"session":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentReportSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ReportAgentResult"},"since":6,"stream":null},"resize-surface":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"cols":{"constraints":[{"clamped_maximum":10000,"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"constraints":[{"clamped_maximum":10000,"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ResizeSurfaceResult"},"since":5,"stream":null},"resolve-terminal":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"terminal_id":{"constraints":[{"format":"UUIDv4 hex without dashes","pattern":"^[0-9a-f]{32}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"ResolveTerminalResult"},"since":9,"stream":null},"run":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["Exactly one of argv and command must be supplied.","pane and new_workspace:true are mutually exclusive.","key is valid only with new_workspace:true.","cols and rows affect sizing only when both are present."],"fields":{"argv":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"command":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"key":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"name":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"new_workspace":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"RunResult"},"since":6,"stream":null},"scroll-surface":{"authority":"control","capability":null,"constraints":["PTY surfaces only; negative values scroll up."],"request":{"additional_properties":false,"fields":{"delta":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-screen":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-tab":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"select-workspace":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both index and delta are supplied, index wins."],"fields":{"delta":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"int64"}},"index":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"send":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["When both are present, UTF-8 text bytes precede decoded bytes."],"fields":{"bytes":{"constraints":[{"encoding":"standard base64"}],"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Base64"}},"paste":{"default":false,"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"text":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"send-key":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"keys":{"constraints":[{"syntax":"lowercase modifier+key chords"}],"nullable":false,"presence":"required","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"set-cell-pixels":{"authority":"frontend","capability":null,"constraints":["Accepted browser resizes complete asynchronously."],"request":{"additional_properties":false,"fields":{"height_px":{"constraints":[{"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"width_px":{"constraints":[{"clamped_minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SetCellPixelsResult"},"since":6,"stream":null},"set-client-info":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"capabilities":{"constraints":["Advertises additive client capabilities for this connection."],"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array"}},"kind":{"constraints":["Control characters become spaces; at most 64 Unicode characters are retained."],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"name":{"constraints":["Control characters become spaces; at most 64 Unicode characters are retained."],"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"set-client-sizing":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["exclusive:true requires client and enabled:true.","Omitting client is valid only with enabled:true and restores all clients for the surface."],"fields":{"client":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"enabled":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"exclusive":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":10,"stream":null},"set-default-colors":{"authority":"control","capability":null,"constraints":["Color strings are exactly #rrggbb.","With complete:true, absent optional values reset to built-in defaults."],"request":{"additional_properties":false,"fields":{"bg":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"complete":{"default":false,"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"cursor":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"CursorStyle"}},"fg":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"palette":{"constraints":["Decimal string keys are palette indexes 0 through 255."],"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"ColorHex"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"set-ratio":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":null},"set-split-ratio":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"split":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"transaction":{"capability":"layout-undo-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":8,"stream":null},"set-viewport-pane-width":{"authority":"control","capability":"viewport-column-resize-v1","constraints":["width must be finite."],"request":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"transaction":{"capability":"layout-undo-v1","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"width":{"constraints":[{"maximum":1.0,"minimum":0.1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"set-window-title":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"title":{"constraints":["C0 controls are sanitized before OSC output."],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"shutdown":{"authority":"local-admin","capability":"server-shutdown-v1","constraints":["The command is accepted only on a trusted Unix-classified transport.","Success means every tracked pane process terminated before the bounded shutdown deadline."],"request":{"additional_properties":false,"fields":{},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":9,"stream":null},"shutdown-daemon":{"authority":"local-admin","capability":null,"constraints":["pid and generation must match the latest identify result.","force bypasses native-browser ownership only; the identity fence and trusted-local authority still apply.","Clients must require daemon-handoff-force-v1 before sending force:true.","The daemon exits only after the success response is queued."],"request":{"additional_properties":false,"fields":{"force":{"capability":"daemon-handoff-force-v1","default":false,"nullable":false,"presence":"optional","since":10,"type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"result":{"kind":"ref","name":"ShutdownDaemonResult"},"since":9,"stream":null},"sidebar-plugin":{"authority":"frontend","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"relaunch":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SidebarPluginResult"},"since":6,"stream":null},"split":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["cols and rows affect sizing only when both are present."],"fields":{"cols":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"rows":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"result":{"kind":"ref","name":"SurfaceResult"},"since":5,"stream":null},"subscribe":{"authority":"frontend","capability":null,"constraints":["subscribe sends no initial tree snapshot.","surface filtering occurs before the bounded mailbox."],"request":{"additional_properties":false,"fields":{"surface":{"capability":"surface-subscribe-filter","default":null,"nullable":true,"presence":"optional","since":9,"type":{"kind":"ref","name":"Id"}},"tree_events":{"default":"coarse","nullable":true,"presence":"optional","since":7,"type":{"kind":"enum","values":["coarse","deltas"]}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":5,"stream":{"event_names":["bell","client-attached","client-changed","client-detached","client-list-invalidated","config-reload-requested","empty","frontend-projection-changed","layout-changed","notification","overflow","pairing-requested","pairing-resolved","pane-added","pane-closed","screen-added","screen-closed","screen-renamed","scroll-changed","status","surface-exited","surface-output","surface-resize-failed","surface-resized","tab-added","tab-closed","tab-renamed","terminal-registry-changed","title-changed","tree-changed","window-title-requested","workspace-added","workspace-closed","workspace-moved","workspace-renamed"],"kind":"subscribe","mode_field":"tree_events","modes":{"coarse":["tree-changed"],"deltas":["workspace-added","workspace-closed","workspace-renamed","workspace-moved","screen-added","screen-closed","screen-renamed","pane-added","pane-closed","tab-added","tab-closed","tab-renamed","tree-changed"]},"ordering":"Response and event objects may interleave. Events preserve enqueue order per subscription. Delta workspace revisions are serialized in durable commit order; overflow ends the stream and requires resubscribe plus snapshot.","terminal_event":null}},"swap-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"constraints":["Exactly one of dir and target must be supplied."],"fields":{"dir":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"PaneDirection"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"target":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"EmptyResult"},"since":6,"stream":null},"terminal-events":{"authority":"control","capability":null,"constraints":["Consumers apply only contiguous revisions for one registry_id and generation."],"request":{"additional_properties":false,"fields":{"after_revision":{"default":0,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"TerminalEventsResult"},"since":9,"stream":null},"undo-layout":{"authority":"control","capability":"layout-undo-v1","constraints":["Clients must reject incomplete or contradictory result variants."],"request":{"additional_properties":false,"constraints":["confirm_close requires the exact preview revision."],"fields":{"confirm_close":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"revision":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"LayoutUndoResult"},"since":9,"stream":null},"vt-state":{"authority":"control","capability":null,"constraints":["PTY surfaces only."],"request":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"VtStateResult"},"since":5,"stream":null},"wait-for":{"authority":"control","capability":null,"constraints":["Blocks subsequent requests on this connection; SDKs should use a dedicated connection."],"request":{"additional_properties":false,"fields":{"pattern":{"constraints":[{"syntax":"Rust regex"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"timeout_ms":{"description":"Zero performs one immediate check.","nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"result":{"kind":"ref","name":"WaitForResult"},"since":6,"stream":null},"zoom-pane":{"authority":"control","capability":null,"constraints":[],"request":{"additional_properties":false,"fields":{"mode":{"default":"toggle","nullable":true,"presence":"optional","type":{"kind":"enum","values":["toggle","on","off"]}},"pane":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"result":{"kind":"ref","name":"ZoomPaneResult"},"since":6,"stream":null}},"events":{"bell":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"bell"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"browser-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"browser-state"}},"frame":{"description":"The initial browser-state includes the latest frame when one exists; later state updates omit it.","nullable":true,"presence":"optional","type":{"kind":"ref","name":"BrowserFrame"}},"frames_stalled":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"status":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["starting","live","failed"]}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"url":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["attach-browser"]},"client-attached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-attached"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"transport":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["unix","ws"]}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-changed"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-detached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-detached"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"client-list-invalidated":{"capability":null,"emission":"serialized-never-emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"client-list-invalidated"}}},"kind":"object"},"since":9,"streams":["subscribe"]},"colors-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"cursor":{"nullable":true,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"nullable":true,"presence":"optional","type":{"kind":"ref","name":"CursorStyle"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"colors-changed"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"palette":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"surface":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["attach-byte"]},"config-reload-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"config-reload-requested"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"detached":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"detached"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte","attach-render","attach-browser"]},"empty":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"empty"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"frame":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"frame"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"since":6,"streams":["attach-browser"]},"frontend-projection-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"frontend-projection-changed"}},"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"projection_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"graphics-status":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["kitty-image-budget-worker-start-failed carries error.","kitty-image-budget-update-failed carries retry_exhausted and summary.","cell-pixel-update-retries-exhausted carries attempts, remaining, cell_width, and cell_height."],"fields":{"attempts":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cell_height":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"cell_width":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"graphics-status"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["kitty-image-budget-worker-start-failed","kitty-image-budget-update-failed","cell-pixel-update-retries-exhausted"]}},"remaining":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"retry_exhausted":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"summary":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":10,"streams":["subscribe"]},"layout-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"layout-changed"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"notification":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"body":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"notification"}},"level":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"NotificationLevel"}},"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe","attach-byte","attach-browser"]},"output":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"colors":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"TerminalColors"}},"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"output"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte"]},"overflow":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["scope and surface are either both present for attach overflow or both absent for subscribe overflow."],"fields":{"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"overflow"}},"scope":{"nullable":false,"presence":"optional","type":{"kind":"literal","value":"surface"}},"surface":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe","attach-byte","attach-render","attach-browser"]},"pairing-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"code":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pairing-requested"}},"expires_in":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"peer":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"pairing-resolved":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pairing-resolved"}},"request":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"pane-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Pane"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pane-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"pane-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Pane"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"pane-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"render-delta":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["size is present if and only if the surface resized; every resize has full:true."],"fields":{"cursor":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderCursor"}},"default_bg":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"default_fg":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"ColorHex"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"render-delta"}},"full":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"graphics":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderGraphicsDelta"}},"history_epoch":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"scrollback_rows":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"size":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"Size"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["attach-render"]},"render-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cursor":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderCursor"}},"default_bg":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"default_fg":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"render-state"}},"graphics":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderGraphics"}},"history_epoch":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"scrollback_rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"size":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Size"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["attach-render"]},"resized":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["At least one of replay and data is present; replay is canonical from protocol 7."],"fields":{"colors":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"TerminalColors"}},"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"description":"Protocol 6 compatibility field.","nullable":false,"presence":"optional","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"resized"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"replay":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"ref","name":"Base64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["attach-byte"]},"screen-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"screen-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"screen-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Screen"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"screen-renamed"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"scroll-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"at_bottom":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"scroll-changed"}},"offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":6,"streams":["subscribe","attach-byte","attach-render","attach-browser"]},"status":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"status"}},"message":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-exited":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-exited"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-output":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-output"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"surface-resize-failed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-resize-failed"}},"reservation_id":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"retry_after_ms":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe"]},"surface-resized":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"surface-resized"}},"reservation_id":{"nullable":true,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"tab-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-added"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"tab-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-closed"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"tab-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Tab"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tab-renamed"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"terminal-registry-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"terminal-registry-changed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"refetch":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"terminal-events-or-list-terminals"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":9,"streams":["subscribe"]},"title-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"title-changed"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"title":{"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"tree-changed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"tree-changed"}}},"kind":"object"},"since":5,"streams":["subscribe"]},"vt-state":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"colors":{"nullable":false,"presence":"optional","since":6,"type":{"kind":"ref","name":"TerminalColors"}},"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"vt-state"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"since":5,"streams":["attach-byte"]},"window-title-requested":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"fields":{"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"window-title-requested"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"since":6,"streams":["subscribe"]},"workspace-added":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-added"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-closed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-closed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-moved":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-moved"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]},"workspace-renamed":{"capability":null,"emission":"emitted","payload":{"additional_properties":false,"constraints":["origin and mutation_id are either both present or both absent."],"fields":{"entity":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Workspace"}},"event":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"workspace-renamed"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"since":7,"streams":["subscribe-deltas"]}},"ir_sha256":"56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d","profiles":{"control":{"description":"Base authenticated session-control commands available to ordinary SDK clients.","inherits":[]},"frontend":{"description":"Rendering, input, presentation, subscribe, and attach commands.","inherits":["control"]},"local-admin":{"description":"Trusted local administration commands.","inherits":["control"],"transport":"Unix-classified transport, including direct Unix and the current stdio relay"},"provider-authority":{"description":"Provider-owned workspace mutation commands.","inherits":["control"],"requires_authority":true}},"protocol":{"id_type":"uint64","javascript_id_policy":"All protocol identifiers are uint64 JSON numbers. JavaScript and TypeScript SDKs must decode them losslessly as bigint (or validated decimal strings at their public boundary), and must not expose IEEE-754 number ids. Pairing request ids, revisions, timestamps, frame sequences, and reservation ids follow the same rule.","name":"cmux-tui-mux","version":10},"schema_version":2,"types":{"AgentRecord":{"additional_properties":false,"fields":{"session":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"updated_at_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"AgentReportSource":{"kind":"enum","values":["socket","hook"]},"AgentSource":{"kind":"enum","values":["detected","socket","hook"]},"AgentState":{"kind":"enum","values":["working","blocked","idle","done","unknown"]},"AppliedPane":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ApplyLayoutResult":{"additional_properties":false,"fields":{"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"AppliedPane"},"kind":"array"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Base64":{"kind":"alias","target":{"kind":"scalar","name":"string"}},"BrowserFrame":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"seq":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"CellPixelFailure":{"additional_properties":false,"fields":{"error":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"CellPixelResize":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"reservation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"CellPixelSurface":{"additional_properties":false,"fields":{"height_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"width_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"ClientInfo":{"additional_properties":false,"fields":{"attached":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}},"client":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"connected_seconds":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"kind":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"self":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"sizes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"ClientSize"},"kind":"array"}},"transport":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"ClientTransport"}}},"kind":"object"},"ClientSize":{"additional_properties":false,"fields":{"cols":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"size_participating":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ClientTransport":{"kind":"enum","values":["local","unix","ws"]},"CloseTerminalResult":{"additional_properties":false,"fields":{"already_closed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"closed":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ColorHex":{"kind":"alias","target":{"kind":"scalar","name":"string"}},"CopyResult":{"additional_properties":false,"fields":{"mode":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["screen","selection","scrollback"]}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"CursorStyle":{"kind":"enum","values":["block","underline","bar"]},"DeadPane":{"additional_properties":false,"fields":{"dead":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"DeclarativeLayout":{"kind":"tagged_union","tag":"type","variants":{"leaf":{"additional_properties":false,"fields":{"command":{"default":null,"nullable":true,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array","min_items":1}},"cwd":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"leaf"}}},"kind":"object"},"split":{"additional_properties":false,"fields":{"a":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"b":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"DeclarativeLayout"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"ratio":{"constraints":[{"clamped_maximum":0.95,"clamped_minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"split"}}},"kind":"object"},"stack":{"additional_properties":false,"fields":{"expanded":{"constraints":["Must identify a member of panes."],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array","min_items":1}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"stack"}}},"kind":"object"}}},"EmptyResult":{"additional_properties":false,"fields":{},"kind":"object"},"ExportLayoutResult":{"additional_properties":false,"fields":{"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"ExportedPane"},"kind":"array"}}},"kind":"object"},"ExportedPane":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surfaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}}},"kind":"object"},"FocusDirectionResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"FrontendProjection":{"additional_properties":false,"fields":{"frontend":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"projection":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"projection_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"replayed":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"schema_version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"scope":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"subject_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"GetCellPixelsResult":{"additional_properties":false,"fields":{"height_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"surfaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelSurface"},"kind":"array"}},"width_px":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"Id":{"kind":"alias","target":{"kind":"scalar","name":"uint64"}},"IdMapping":{"additional_properties":false,"fields":{"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["workspace","screen","pane","surface"]}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"IdentifyResult":{"additional_properties":false,"fields":{"app":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"cmux-tui"}},"build_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"capabilities":{"default":[],"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"string"},"kind":"array"}},"daemon_handoff":{"nullable":false,"presence":"required","since":9,"type":{"kind":"literal","value":1}},"generation":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"string"}},"ghostty_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"protocol":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"registry_id":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"string"}},"session":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"shutdown_cleanup":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"ShutdownCleanupStatus"}},"terminal_revision":{"nullable":false,"presence":"required","since":9,"type":{"kind":"scalar","name":"uint64"}},"version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace_revision":{"nullable":false,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"IdsResult":{"additional_properties":false,"fields":{"ids":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"IdMapping"},"kind":"array"}}},"kind":"object"},"JsonValue":{"kind":"opaque_json","reason":"The wire field intentionally carries a frontend-authored or runtime-authored arbitrary JSON document."},"KittyGraphicsState":{"additional_properties":false,"fields":{"alternate_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"alternate_replay_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_bytes":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"inflight_bytes":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"placements":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"primary_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"primary_replay_next_image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"replay_cursor_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"KittyImageAlias":{"additional_properties":false,"fields":{"image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_number":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"Layout":{"kind":"tagged_union","tag":"type","variants":{"leaf":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"leaf"}}},"kind":"object"},"split":{"additional_properties":false,"fields":{"a":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"b":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"dir":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"SplitDirection"}},"ratio":{"constraints":[{"maximum":0.95,"minimum":0.05}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"float32"}},"split":{"description":"Stable for the lifetime of this split node.","nullable":false,"presence":"optional","since":8,"type":{"kind":"ref","name":"Id"}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"split"}}},"kind":"object"},"stack":{"additional_properties":false,"fields":{"expanded":{"constraints":["Must identify a member of panes."],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array","min_items":1}},"type":{"nullable":false,"presence":"required","type":{"kind":"literal","value":"stack"}}},"kind":"object"}}},"LayoutUndoConfirmationRequired":{"additional_properties":false,"fields":{"closes_panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Id"},"kind":"array"}},"confirmation_required":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"undone":{"nullable":false,"presence":"required","type":{"kind":"literal","value":false}}},"kind":"object"},"LayoutUndoResult":{"kind":"untagged_union","variants":[{"kind":"ref","name":"LayoutUndoUndone"},{"kind":"ref","name":"LayoutUndoConfirmationRequired"}]},"LayoutUndoUndone":{"additional_properties":false,"fields":{"confirmation_required":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"literal","value":false}},"revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"undone":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}}},"kind":"object"},"ListAgentsResult":{"additional_properties":false,"fields":{"agents":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"AgentRecord"},"kind":"array"}}},"kind":"object"},"ListTerminalsResult":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"terminals":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"TerminalRecord"},"kind":"array"}}},"kind":"object"},"LivePane":{"additional_properties":false,"fields":{"active_tab":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"focused_at":{"default":0,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint64"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"tabs":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Tab"},"kind":"array"}}},"kind":"object"},"MintTerminalRendererResult":{"additional_properties":false,"fields":{"endpoint":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"incarnation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"rights":{"constraints":[{"current_value":7}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"token":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"ttl_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"MoveTerminalResult":{"additional_properties":false,"fields":{"changed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"screen":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"NotificationLevel":{"kind":"enum","values":["info","warning","error"]},"NotificationMarker":{"additional_properties":false,"fields":{"level":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"NotificationLevel"}},"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"unread":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}}},"kind":"object"},"NotifyResult":{"additional_properties":false,"fields":{"notification":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Pane":{"kind":"untagged_union","variants":[{"kind":"ref","name":"LivePane"},{"kind":"ref","name":"DeadPane"}]},"PaneDirection":{"kind":"enum","values":["left","right","up","down"]},"PaneNeighborResult":{"additional_properties":false,"fields":{"pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"PingResult":{"additional_properties":false,"fields":{"build_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"ghostty_commit":{"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"ok":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"protocol":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"version":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"ProcessInfoResult":{"additional_properties":false,"fields":{"command":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"cwd":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"ProviderWorkspaceMutationResult":{"additional_properties":false,"fields":{"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ReadScreenResult":{"additional_properties":false,"fields":{"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"ReadScrollbackResult":{"additional_properties":false,"fields":{"epoch":{"nullable":false,"presence":"required","since":10,"type":{"kind":"scalar","name":"uint64"}},"rows":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRow"},"kind":"array"}},"start":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"total":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"RenderCursor":{"additional_properties":false,"fields":{"blink":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"color":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"style":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"CursorStyle"}},"visible":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"x":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"y":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"RenderGraphicFormat":{"kind":"enum","values":["rgb","rgba"]},"RenderGraphicImage":{"additional_properties":false,"fields":{"data":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"format":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"RenderGraphicFormat"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"RenderGraphicPlacement":{"additional_properties":false,"fields":{"anchor_col":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}},"anchor_row":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint32"}},"columns":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"grid_cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"grid_rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"image_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"ordinal":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"pixel_height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"pixel_width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"placement_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_height":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_width":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_x":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"source_y":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"viewport_col":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}},"viewport_row":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}},"viewport_visible":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"x_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"y_offset":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"z":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"int32"}}},"kind":"object"},"RenderGraphics":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicImage"},"kind":"array"}},"placements":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderGraphicPlacement"},"kind":"array"}},"removed_image_ids":{"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"uint32"},"kind":"array"}}},"kind":"object"},"RenderGraphicsDelta":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"images":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicImage"},"kind":"array"}},"placements":{"nullable":false,"presence":"optional","type":{"items":{"kind":"ref","name":"RenderGraphicPlacement"},"kind":"array"}},"removed_image_ids":{"nullable":false,"presence":"optional","type":{"items":{"kind":"scalar","name":"uint32"},"kind":"array"}}},"kind":"object"},"RenderRow":{"additional_properties":false,"fields":{"row":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"runs":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"RenderRun"},"kind":"array"}}},"kind":"object"},"RenderRun":{"additional_properties":false,"fields":{"attrs":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}},"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"underline":{"nullable":false,"presence":"optional","type":{"kind":"ref","name":"RenderUnderline"}},"width_hint":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"RenderUnderline":{"kind":"enum","values":["single","double","curly","dotted","dashed"]},"ReportAgentResult":{"additional_properties":false,"fields":{"session":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"source":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentReportSource"}},"state":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"AgentState"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"ResizeSurfaceResult":{"additional_properties":false,"fields":{"accepted":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"reservation_id":{"nullable":true,"presence":"required","since":7,"type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ResolveTerminalResult":{"additional_properties":false,"fields":{"exit":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"launch_spec":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"RunResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"required","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","since":9,"type":{"kind":"scalar","name":"string"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Screen":{"additional_properties":false,"fields":{"active":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"active_pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"layout":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Layout"}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"panes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Pane"},"kind":"array"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"zoomed_pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"SetCellPixelsResult":{"additional_properties":false,"fields":{"failures":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelFailure"},"kind":"array"}},"resizes":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"CellPixelResize"},"kind":"array"}}},"kind":"object"},"ShutdownCleanupStatus":{"additional_properties":false,"fields":{"degraded":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"pending":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"retrying":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}}},"kind":"object"},"ShutdownDaemonResult":{"additional_properties":false,"fields":{"accepted":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"pid":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint32"}}},"kind":"object"},"SidebarPluginResult":{"additional_properties":false,"fields":{"error":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"retry_after_ms":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"surface":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"Size":{"additional_properties":false,"fields":{"cols":{"constraints":[{"maximum":10000,"minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"rows":{"constraints":[{"maximum":10000,"minimum":1}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"SplitDirection":{"kind":"enum","values":["right","down"]},"SurfaceResult":{"additional_properties":false,"fields":{"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Tab":{"additional_properties":false,"fields":{"browser_error":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"browser_frames_stalled":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"boolean"}},"browser_source":{"nullable":true,"presence":"required","type":{"kind":"enum","values":["external","launched"]}},"browser_status":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"enum","values":["starting","live","failed"]}},"dead":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"kind":{"nullable":false,"presence":"required","type":{"kind":"enum","values":["pty","browser"]}},"name":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"notification":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"NotificationMarker"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}},"size":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Size"}},"supports_clear_history_key_fallback":{"capability":"clear-history-key-v1","nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"boolean"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"optional","since":9,"type":{"kind":"scalar","name":"string"}},"terminal_resource_id":{"constraints":[{"pattern":"^term_[0-9a-f]{32}$"}],"nullable":true,"presence":"optional","since":10,"type":{"kind":"scalar","name":"string"}},"title":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalColors":{"additional_properties":false,"fields":{"bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"cursor":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"ColorHex"}},"cursor_blink":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"scalar","name":"boolean"}},"cursor_style":{"nullable":true,"presence":"optional","since":6,"type":{"kind":"ref","name":"CursorStyle"}},"fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"palette":{"constraints":["Decimal string keys are palette indexes 0 through 255."],"nullable":false,"presence":"optional","since":7,"type":{"kind":"map","values":{"kind":"ref","name":"ColorHex"}}},"selection_bg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}},"selection_fg":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"ColorHex"}}},"kind":"object"},"TerminalEventsResult":{"additional_properties":false,"fields":{"events":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"TerminalRegistryEvent"},"kind":"array"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"TerminalKey":{"kind":"enum","values":["unidentified","backquote","backslash","bracket-left","bracket-right","comma","digit0","digit1","digit2","digit3","digit4","digit5","digit6","digit7","digit8","digit9","equal","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","minus","period","quote","semicolon","slash","backspace","enter","space","tab","delete","end","home","insert","page-down","page-up","arrow-down","arrow-left","arrow-right","arrow-up","numpad0","numpad1","numpad2","numpad3","numpad4","numpad5","numpad6","numpad7","numpad8","numpad9","numpad-add","numpad-backspace","numpad-comma","numpad-decimal","numpad-divide","numpad-enter","numpad-equal","numpad-multiply","numpad-subtract","numpad-up","numpad-down","numpad-right","numpad-left","numpad-begin","numpad-home","numpad-end","numpad-insert","numpad-delete","numpad-page-up","numpad-page-down","escape","f1","f2","f3","f4","f5","f6","f7","f8","f9","f10","f11","f12","f13","f14","f15","f16","f17","f18","f19","f20"]},"TerminalKeyAction":{"kind":"enum","values":["press","release","repeat"]},"TerminalKeyInput":{"additional_properties":false,"constraints":["consumed_mods must be a subset of mods.","unshifted_codepoint, shifted_codepoint, and base_layout_codepoint contain exactly one Unicode scalar when present.","utf8 contains no control characters.","macos_option_as_alt may be false only when Alt is active and consumed."],"fields":{"action":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"ref","name":"TerminalKeyAction"}},"base_layout_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"composing":{"default":false,"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"consumed_mods":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalModifiers"}},"key":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalKey"}},"macos_option_as_alt":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"mods":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalModifiers"}},"shifted_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"unshifted_codepoint":{"default":null,"nullable":true,"presence":"optional","type":{"kind":"scalar","name":"string"}},"utf8":{"constraints":[{"max_length":4096}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalLifecycle":{"kind":"enum","values":["launching","adopting","running","exited","tombstoned"]},"TerminalModifiers":{"additional_properties":false,"fields":{"alt":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"caps_lock":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"control":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"num_lock":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"shift":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"super":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}}},"kind":"object"},"TerminalPlacement":{"additional_properties":false,"fields":{"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"lifecycle":{"nullable":true,"presence":"required","type":{"kind":"literal","value":"running"}},"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"screen":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"surface":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"terminal_id":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"},"TerminalRecord":{"additional_properties":false,"fields":{"exit":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"launch_spec":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"lifecycle":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"TerminalLifecycle"}},"terminal_id":{"constraints":[{"format":"UUIDv4 hex without dashes","pattern":"^[0-9a-f]{32}$"}],"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_incarnation":{"nullable":true,"presence":"required","type":{"kind":"scalar","name":"string"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"TerminalRegistryEvent":{"additional_properties":false,"fields":{"kind":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"mutation_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"origin":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"result":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"JsonValue"}},"terminal_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"workspace_key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Tree":{"additional_properties":false,"fields":{"generation":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"pane_revision":{"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"registry_id":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"terminal_revision":{"nullable":false,"presence":"optional","since":9,"type":{"kind":"scalar","name":"uint64"}},"workspace_revision":{"capability":"workspace-registry-v1","nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"uint64"}},"workspaces":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Workspace"},"kind":"array"}}},"kind":"object"},"VtStateResult":{"additional_properties":false,"fields":{"cols":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}},"data":{"constraints":[{"encoding":"standard base64"}],"nullable":false,"presence":"required","type":{"kind":"ref","name":"Base64"}},"kitty_graphics_state":{"nullable":false,"presence":"optional","since":10,"type":{"kind":"ref","name":"KittyGraphicsState"}},"kitty_image_aliases":{"nullable":false,"presence":"optional","since":9,"type":{"items":{"kind":"ref","name":"KittyImageAlias"},"kind":"array"}},"rows":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint16"}}},"kind":"object"},"WaitForResult":{"additional_properties":false,"fields":{"elapsed_ms":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"matched":{"nullable":false,"presence":"required","type":{"kind":"literal","value":true}},"text":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}}},"kind":"object"},"Workspace":{"additional_properties":false,"fields":{"active":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"id":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"key":{"capability":"workspace-registry-v1","constraints":[{"format":"lowercase canonical UUID"}],"nullable":false,"presence":"optional","since":7,"type":{"kind":"scalar","name":"string"}},"name":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"screens":{"nullable":false,"presence":"required","type":{"items":{"kind":"ref","name":"Screen"},"kind":"array"}},"short_id":{"constraints":[{"pattern":"^[a-z0-9]{6}$"}],"nullable":false,"presence":"optional","since":6,"type":{"kind":"scalar","name":"string"}}},"kind":"object"},"WorkspaceMutationResult":{"additional_properties":false,"fields":{"changed":{"nullable":false,"presence":"optional","type":{"kind":"scalar","name":"boolean"}},"generation":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"index":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}},"key":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"registry_id":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"string"}},"replayed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"workspace":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"workspace_revision":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"uint64"}}},"kind":"object"},"ZoomPaneResult":{"additional_properties":false,"fields":{"pane":{"nullable":false,"presence":"required","type":{"kind":"ref","name":"Id"}},"zoomed":{"nullable":false,"presence":"required","type":{"kind":"scalar","name":"boolean"}},"zoomed_pane":{"nullable":true,"presence":"required","type":{"kind":"ref","name":"Id"}}},"kind":"object"}}}') diff --git a/cmux-tui/bindings/python/cmux/raw/_generated/codec.py b/cmux-tui/bindings/python/cmux/raw/_generated/codec.py index 350675042591..ad7886ba692b 100644 --- a/cmux-tui/bindings/python/cmux/raw/_generated/codec.py +++ b/cmux-tui/bindings/python/cmux/raw/_generated/codec.py @@ -72,6 +72,7 @@ class ProtocolDecodeError(ValueError): 'types/RunResult': models.RunResult, 'types/Screen': models.Screen, 'types/SetCellPixelsResult': models.SetCellPixelsResult, + 'types/ShutdownCleanupStatus': models.ShutdownCleanupStatus, 'types/ShutdownDaemonResult': models.ShutdownDaemonResult, 'types/SidebarPluginResult': models.SidebarPluginResult, 'types/Size': models.Size, diff --git a/cmux-tui/bindings/python/cmux/raw/_generated/metadata.py b/cmux-tui/bindings/python/cmux/raw/_generated/metadata.py index 199a10bc6951..46066ac96051 100644 --- a/cmux-tui/bindings/python/cmux/raw/_generated/metadata.py +++ b/cmux-tui/bindings/python/cmux/raw/_generated/metadata.py @@ -8,7 +8,7 @@ SCHEMA_VERSION = 2 MUX_PROTOCOL = 10 -IR_SHA256 = 'a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa' +IR_SHA256 = '56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d' @dataclass(frozen=True) diff --git a/cmux-tui/bindings/python/cmux/raw/_generated/models.py b/cmux-tui/bindings/python/cmux/raw/_generated/models.py index 4315df95005a..c178059ee309 100644 --- a/cmux-tui/bindings/python/cmux/raw/_generated/models.py +++ b/cmux-tui/bindings/python/cmux/raw/_generated/models.py @@ -419,6 +419,7 @@ class IdentifyResult: build_commit: Union[str, None, MissingType] = field(default=MISSING) capabilities: Union[List[str], MissingType] = field(default=MISSING) ghostty_commit: Union[str, None, MissingType] = field(default=MISSING) + shutdown_cleanup: Union[ShutdownCleanupStatus, MissingType] = field(default=MISSING) @dataclass(frozen=True) @@ -755,6 +756,14 @@ class SetCellPixelsResult: resizes: List[CellPixelResize] +@dataclass(frozen=True) +class ShutdownCleanupStatus: + __cmux_schema_path__: ClassVar[str] = 'types/ShutdownCleanupStatus' + degraded: bool + pending: int + retrying: bool + + @dataclass(frozen=True) class ShutdownDaemonResult: __cmux_schema_path__: ClassVar[str] = 'types/ShutdownDaemonResult' @@ -2307,6 +2316,7 @@ class WorkspaceRenamedEvent(EventBase): 'RunResult', 'Screen', 'SetCellPixelsResult', + 'ShutdownCleanupStatus', 'ShutdownDaemonResult', 'SidebarPluginResult', 'Size', diff --git a/cmux-tui/bindings/rust/src/generated/.cmux-sdk-manifest.json b/cmux-tui/bindings/rust/src/generated/.cmux-sdk-manifest.json index 3d3d00decd90..8eee43c4c548 100644 --- a/cmux-tui/bindings/rust/src/generated/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/rust/src/generated/.cmux-sdk-manifest.json @@ -2,32 +2,32 @@ "files": [ { "path": "commands.rs", - "sha256": "e9eb6e4dd84035790be07c830a005496bd260ac9fce78e45a7af3b81b1aa5c4b", + "sha256": "6e2103672a67db8853d7b301908889f03efaeb5394f0beff28d29aebc9ac861d", "size": 56698 }, { "path": "events.rs", - "sha256": "79727898e91158d9dedb23fb3818da1ac0c93241117e26bb44bdb62ecc890f76", + "sha256": "9c678a4485daacce8d0f31bdafaad5a644ad579339df1aa061f718b52369f603", "size": 40682 }, { "path": "metadata.rs", - "sha256": "cf786154337a9255a02e12e56101e523c591a78a956b515353382528954242b8", + "sha256": "189bda5fa15aeb97e5fba49bd1fe7481f24d38c287c62ad8937ce40967d1b467", "size": 34495 }, { "path": "mod.rs", - "sha256": "0018a4e897123ef737aa0cbdff084e1adb32801d845965f286cb55339735c17b", + "sha256": "8005f4fa062c45a2d4ddcf53db2deeae7f07e82e8f807fd98eb39b4862fc884c", "size": 365 }, { "path": "types.rs", - "sha256": "0e068347ff1964b4c8e90b9b175da387f9b67610080d6282f4d8d22c07c214ac", - "size": 34273 + "sha256": "78741638ca31b6177be486a1a58aed0e1fcefcffd2e9b285dc5783f2d82ffb82", + "size": 34646 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "rust", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/rust/src/generated/commands.rs b/cmux-tui/bindings/rust/src/generated/commands.rs index 1634f70a5a41..55c01827a608 100644 --- a/cmux-tui/bindings/rust/src/generated/commands.rs +++ b/cmux-tui/bindings/rust/src/generated/commands.rs @@ -1,5 +1,5 @@ // This file is generated. Do not edit by hand. -// cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. // The emitter owns this layout so generation is independent of the installed rustfmt. use super::metadata::*; diff --git a/cmux-tui/bindings/rust/src/generated/events.rs b/cmux-tui/bindings/rust/src/generated/events.rs index 153458de26bd..9ea05c768db7 100644 --- a/cmux-tui/bindings/rust/src/generated/events.rs +++ b/cmux-tui/bindings/rust/src/generated/events.rs @@ -1,5 +1,5 @@ // This file is generated. Do not edit by hand. -// cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. // The emitter owns this layout so generation is independent of the installed rustfmt. use super::metadata::*; diff --git a/cmux-tui/bindings/rust/src/generated/metadata.rs b/cmux-tui/bindings/rust/src/generated/metadata.rs index 55c9c63cd873..239cee6e50b9 100644 --- a/cmux-tui/bindings/rust/src/generated/metadata.rs +++ b/cmux-tui/bindings/rust/src/generated/metadata.rs @@ -1,12 +1,12 @@ // This file is generated. Do not edit by hand. -// cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. // The emitter owns this layout so generation is independent of the installed rustfmt. use crate::{CommandMetadata, EventMetadata, ProfileMetadata, StreamMetadata}; pub const SDK_SCHEMA_VERSION: u32 = 2; pub const MUX_PROTOCOL_VERSION: u32 = 10; -pub const SDK_IR_SHA256: &str = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa"; +pub const SDK_IR_SHA256: &str = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d"; #[rustfmt::skip] pub const CONTROL_PROFILE: ProfileMetadata = ProfileMetadata { diff --git a/cmux-tui/bindings/rust/src/generated/mod.rs b/cmux-tui/bindings/rust/src/generated/mod.rs index f843e6452ac5..96286622be8c 100644 --- a/cmux-tui/bindings/rust/src/generated/mod.rs +++ b/cmux-tui/bindings/rust/src/generated/mod.rs @@ -1,5 +1,5 @@ // This file is generated. Do not edit by hand. -// cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. // The emitter owns this layout so generation is independent of the installed rustfmt. mod commands; diff --git a/cmux-tui/bindings/rust/src/generated/types.rs b/cmux-tui/bindings/rust/src/generated/types.rs index 0603a7ab6ac0..888b161d767b 100644 --- a/cmux-tui/bindings/rust/src/generated/types.rs +++ b/cmux-tui/bindings/rust/src/generated/types.rs @@ -1,5 +1,5 @@ // This file is generated. Do not edit by hand. -// cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. +// cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. // The emitter owns this layout so generation is independent of the installed rustfmt. use crate::{Nullable, Optional}; @@ -298,6 +298,8 @@ pub struct IdentifyResult { pub protocol: u32, pub registry_id: String, pub session: String, + #[serde(default, deserialize_with = "crate::presence::deserialize_optional_non_null", skip_serializing_if = "Option::is_none")] + pub shutdown_cleanup: Option, pub terminal_revision: u64, pub version: String, pub workspace_revision: u64, @@ -714,6 +716,14 @@ pub struct SetCellPixelsResult { pub resizes: Vec, } +#[rustfmt::skip] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ShutdownCleanupStatus { + pub degraded: bool, + pub pending: u64, + pub retrying: bool, +} + #[rustfmt::skip] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ShutdownDaemonResult { diff --git a/cmux-tui/bindings/typescript/src/raw/generated/.cmux-sdk-manifest.json b/cmux-tui/bindings/typescript/src/raw/generated/.cmux-sdk-manifest.json index 5b38880ad936..c878b0e03f48 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/typescript/src/raw/generated/.cmux-sdk-manifest.json @@ -2,32 +2,32 @@ "files": [ { "path": "commands.ts", - "sha256": "328e2ddb5eb067d9e1e21008fe73333dd7dd4c272fd42d96f37657c1389ad82b", + "sha256": "c1fefa266793b861ec17331dacab59903248d2c8827ce18ba0c3d373f4b0f7d2", "size": 44010 }, { "path": "events.ts", - "sha256": "2e93759e25f1594bd97e98022e2579d53e9f857e6912ce09e5004ec44597af85", + "sha256": "44c08915cec137eee1cfb6b47e6f5c7c19019f49deede1a4b667a49bcaa20ea4", "size": 14850 }, { "path": "index.ts", - "sha256": "38a6b65cc5044bb1de9109746ec107aa7698f8204d62f56796fb1c7e5deb0b71", + "sha256": "034937ce6675b3c230f61167820f6376584c0a4e265e8b0aee8a68bf9ecde64b", "size": 272 }, { "path": "metadata.ts", - "sha256": "e5171d32f1a7c31e6912527e2b1b8bda31ce617017008be7f0c4c21f614c7bf1", - "size": 246585 + "sha256": "ea5a2f099491eb39400c2eef13f2b3b2145ed7dad61bfef37fe987a7d4b6d830", + "size": 247424 }, { "path": "types.ts", - "sha256": "6d1a1e975e610db5ebe3d561dab727cb97a993ad7b1194f93c7dc6cbc4d5e2bb", - "size": 14855 + "sha256": "ef6d412fa176d96b1a0f17071e88270bf31868948d61f24f2c8fd766c0d40ec4", + "size": 15010 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "typescript", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/typescript/src/raw/generated/commands.ts b/cmux-tui/bindings/typescript/src/raw/generated/commands.ts index 5c0706a3c022..0be3842b7475 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/commands.ts +++ b/cmux-tui/bindings/typescript/src/raw/generated/commands.ts @@ -1,5 +1,5 @@ /* This file is generated. Do not edit by hand. */ -/* cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. */ +/* cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. */ import type * as T from "./types.js"; diff --git a/cmux-tui/bindings/typescript/src/raw/generated/events.ts b/cmux-tui/bindings/typescript/src/raw/generated/events.ts index 5cdb43affcc7..ce06fe0ee59e 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/events.ts +++ b/cmux-tui/bindings/typescript/src/raw/generated/events.ts @@ -1,5 +1,5 @@ /* This file is generated. Do not edit by hand. */ -/* cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. */ +/* cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. */ import type * as T from "./types.js"; diff --git a/cmux-tui/bindings/typescript/src/raw/generated/index.ts b/cmux-tui/bindings/typescript/src/raw/generated/index.ts index 7be670d69c66..e58a8f906b27 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/index.ts +++ b/cmux-tui/bindings/typescript/src/raw/generated/index.ts @@ -1,5 +1,5 @@ /* This file is generated. Do not edit by hand. */ -/* cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. */ +/* cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. */ export * from "./types.js"; export * from "./commands.js"; diff --git a/cmux-tui/bindings/typescript/src/raw/generated/metadata.ts b/cmux-tui/bindings/typescript/src/raw/generated/metadata.ts index 3c6add2c268e..3950bc6f0b78 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/metadata.ts +++ b/cmux-tui/bindings/typescript/src/raw/generated/metadata.ts @@ -1,10 +1,10 @@ /* This file is generated. Do not edit by hand. */ -/* cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. */ +/* cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. */ export const SDK_SCHEMA_VERSION = 2 as const; export const MUX_PROTOCOL_VERSION = 10 as const; -export const SDK_IR_SHA256 = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa" as const; +export const SDK_IR_SHA256 = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d" as const; export const PROTOCOL = { "id_type": "uint64", "javascript_id_policy": "All protocol identifiers are uint64 JSON numbers. JavaScript and TypeScript SDKs must decode them losslessly as bigint (or validated decimal strings at their public boundary), and must not expose IEEE-754 number ids. Pairing request ids, revisions, timestamps, frame sequences, and reservation ids follow the same rule.", @@ -2472,6 +2472,15 @@ export const TYPE_SCHEMAS: Readonly> = { "name": "string" } }, + "shutdown_cleanup": { + "nullable": false, + "presence": "optional", + "since": 10, + "type": { + "kind": "ref", + "name": "ShutdownCleanupStatus" + } + }, "terminal_revision": { "nullable": false, "presence": "required", @@ -4173,6 +4182,36 @@ export const TYPE_SCHEMAS: Readonly> = { }, "kind": "object" }, + "ShutdownCleanupStatus": { + "additional_properties": false, + "fields": { + "degraded": { + "nullable": false, + "presence": "required", + "type": { + "kind": "scalar", + "name": "boolean" + } + }, + "pending": { + "nullable": false, + "presence": "required", + "type": { + "kind": "scalar", + "name": "uint64" + } + }, + "retrying": { + "nullable": false, + "presence": "required", + "type": { + "kind": "scalar", + "name": "boolean" + } + } + }, + "kind": "object" + }, "ShutdownDaemonResult": { "additional_properties": false, "fields": { diff --git a/cmux-tui/bindings/typescript/src/raw/generated/types.ts b/cmux-tui/bindings/typescript/src/raw/generated/types.ts index 81f7c24acce2..4379ba42e2b6 100644 --- a/cmux-tui/bindings/typescript/src/raw/generated/types.ts +++ b/cmux-tui/bindings/typescript/src/raw/generated/types.ts @@ -1,5 +1,5 @@ /* This file is generated. Do not edit by hand. */ -/* cmux-tui mux protocol 10, IR a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa. */ +/* cmux-tui mux protocol 10, IR 56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d. */ /** JSON accepted by the wire codec. bigint is serialized as an exact JSON integer. */ @@ -170,6 +170,7 @@ export type IdentifyResult = { "protocol": number; "registry_id": string; "session": string; + "shutdown_cleanup"?: ShutdownCleanupStatus; "terminal_revision": bigint; "version": string; "workspace_revision": bigint; @@ -450,6 +451,12 @@ export type SetCellPixelsResult = { "resizes": Array; }; +export type ShutdownCleanupStatus = { + "degraded": boolean; + "pending": bigint; + "retrying": boolean; +}; + export type ShutdownDaemonResult = { "accepted": true; "generation": string; diff --git a/cmux-tui/bindings/zig/src/raw/generated/.cmux-sdk-manifest.json b/cmux-tui/bindings/zig/src/raw/generated/.cmux-sdk-manifest.json index e4373d54a66c..9a11de3ee2f6 100644 --- a/cmux-tui/bindings/zig/src/raw/generated/.cmux-sdk-manifest.json +++ b/cmux-tui/bindings/zig/src/raw/generated/.cmux-sdk-manifest.json @@ -2,17 +2,17 @@ "files": [ { "path": "presence_test.zig", - "sha256": "d8bd1c7beac5e0fd9389a20f4eab35a2f22c3292f05145b58a89d7279ec4ceeb", - "size": 6345 + "sha256": "3f1b5e387df3ad490bfcfe231dbf7d143d4b973018405bef189dbdbc0aa4e38a", + "size": 6426 }, { "path": "protocol.zig", - "sha256": "746dd6e3c83a35ab9c711bb2821383b013226dbe0047a5f5ca6969fdba97b535", - "size": 142583 + "sha256": "ed16c820b0168a046029c298e3a0eaeaadd152568e5acd3283184c98949e48ee", + "size": 142769 } ], "format": 1, - "ir_sha256": "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa", + "ir_sha256": "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d", "language": "zig", "mux_protocol": 10, "schema_version": 2 diff --git a/cmux-tui/bindings/zig/src/raw/generated/presence_test.zig b/cmux-tui/bindings/zig/src/raw/generated/presence_test.zig index d3ea65abac5c..cc7386d53308 100644 --- a/cmux-tui/bindings/zig/src/raw/generated/presence_test.zig +++ b/cmux-tui/bindings/zig/src/raw/generated/presence_test.zig @@ -66,6 +66,7 @@ test "every generated optional non-null field rejects explicit null" { try expectExplicitNullRejected(protocol.WorkspaceRenamedEvent, "origin"); try expectExplicitNullRejected(protocol.FrontendProjection, "replayed"); try expectExplicitNullRejected(protocol.IdentifyResult, "capabilities"); + try expectExplicitNullRejected(protocol.IdentifyResult, "shutdown_cleanup"); try expectExplicitNullRejected(protocol.LayoutSplit, "split"); try expectExplicitNullRejected(protocol.LayoutUndoUndone, "confirmation_required"); try expectExplicitNullRejected(protocol.LivePane, "focused_at"); diff --git a/cmux-tui/bindings/zig/src/raw/generated/protocol.zig b/cmux-tui/bindings/zig/src/raw/generated/protocol.zig index 1490033f1171..7922ed9f3bd7 100644 --- a/cmux-tui/bindings/zig/src/raw/generated/protocol.zig +++ b/cmux-tui/bindings/zig/src/raw/generated/protocol.zig @@ -7,7 +7,7 @@ const client_runtime = @import("../client.zig"); pub const schema_version: u16 = 2; pub const mux_protocol: u16 = 10; -pub const ir_sha256 = "a9918d4cebc182e832fc2e08c8808125a7e4d4f6e4ed389f10b4cbb1478245aa"; +pub const ir_sha256 = "56597ffacc6ef7d83023966ca55a6f176ebc27d34f45256d41dff5985684105d"; pub const AgentRecord = struct { session: wire.Nullable([]const u8), @@ -350,12 +350,14 @@ pub const IdentifyResult = struct { protocol: u32, registry_id: []const u8, session: []const u8, + shutdown_cleanup: ?ShutdownCleanupStatus = null, terminal_revision: u64, version: []const u8, workspace_revision: u64, pub const cmux_wire_optional_nonnull_fields = [_][]const u8{ "capabilities", + "shutdown_cleanup", }; }; @@ -825,6 +827,12 @@ pub const SetCellPixelsResult = struct { resizes: []const CellPixelResize, }; +pub const ShutdownCleanupStatus = struct { + degraded: bool, + pending: u64, + retrying: bool, +}; + pub const ShutdownDaemonResult = struct { accepted: bool, generation: []const u8, diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 3cfb8c2ee66e..cc1763218897 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -187,13 +187,15 @@ Params: none. Result: ```text -object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:string|null,protocol:uint32,capabilities:array,session:string,pid:uint32,registry_id:string,generation:string,workspace_revision:uint64} +object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:string|null,protocol:uint32,capabilities:array,session:string,pid:uint32,registry_id:string,generation:string,workspace_revision:uint64,terminal_revision:uint64,daemon_handoff:1,shutdown_cleanup?:object{pending:uint64,retrying:boolean,degraded:boolean}} ``` `build_commit` and `ghostty_commit` are additive build-stamp fields. Current source builds derive them from the repository and terminal-engine checkout. A dirty source build appends a deterministic content fingerprint. Distribution builds may override both stamps explicitly. Older binaries can omit the fields or return `null`, so clients must preserve compatibility with those identities. `capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `daemon-handoff-force-v1` advertises the optional `force` field on `shutdown-daemon`. `browser-pointer-frame-guard-v1` advertises authoritative `pointer_frame_seq` and `pointer_frame_floor_seq` browser attach/frame state plus the additive `browser-frame-presented`, `browser-mouse-guarded`, and `browser-wheel-guarded` commands. Each admitted bitmap receives a new guard even when its document and dimensions match the previous bitmap. The reported floor through latest range proves route membership only. `browser-frame-presented` advances one exact acknowledged token for that connection, and only that token authorizes a new guarded pointer action. A guarded pointer command implicitly acknowledges its own token. Each connection retains one token, while the bounded browser input queue owns actions admitted before a later presentation. Navigation or geometry changes clear the range and all acknowledgements. An accepted press keeps its original guard for motion across ordinary repaints while document and geometry remain valid; invalidation suppresses further motion but retains its balancing release. A capable client echoes that value in `set-client-info`; browser attach requires the bilateral capability while PTY attach remains available without it. The legacy `browser-mouse` and `browser-wheel` schemas retain their optional guard, but guarded servers reject a missing guard before surface lookup. `viewport-splits-v1` advertises `new-pane-right` and the `Screen.viewport_splits` field. `viewport-column-resize-v1` advertises `set-viewport-pane-width` and `Screen.viewport_base_width`. `layout-undo-v1` advertises server-owned structural layout history and `undo-layout`. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. `server-shutdown-v1` advertises atomic destructive shutdown; clients may use the compatibility cleanup path only when this capability is absent. +`shutdown_cleanup` reports unresolved terminal cleanup, whether a retry is active, and whether cleanup has exhausted its automatic retries. It is an additive protocol 10 field, so clients must accept its absence from older protocol 10 servers. + Errors: | Error | Condition | @@ -214,7 +216,7 @@ Example: ```json {"id":1,"cmd":"identify"} -{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"0.1.0","build_commit":"abc123","ghostty_commit":"def456","protocol":10,"capabilities":["attach-initial-size","surface-subscribe-filter","workspace-registry-v1","daemon-handoff-force-v1","browser-pointer-frame-guard-v1","viewport-splits-v1","viewport-column-resize-v1","layout-undo-v1","clear-history-v1","clear-history-key-v1","provider-managed-workspace-authority-v2","server-shutdown-v1"],"session":"main","pid":12345}} +{"id":1,"ok":true,"data":{"app":"cmux-tui","version":"0.1.0","build_commit":"abc123","ghostty_commit":"def456","protocol":10,"capabilities":["attach-initial-size","surface-subscribe-filter","workspace-registry-v1","daemon-handoff-force-v1","browser-pointer-frame-guard-v1","viewport-splits-v1","viewport-column-resize-v1","layout-undo-v1","clear-history-v1","clear-history-key-v1","provider-managed-workspace-authority-v2","server-shutdown-v1"],"session":"main","pid":12345,"registry_id":"registry-123","generation":"generation-456","workspace_revision":12,"terminal_revision":34,"daemon_handoff":1,"shutdown_cleanup":{"pending":0,"retrying":false,"degraded":false}}} ``` The current server reports protocol `10` in this field and in `ping`. Clients must negotiate protocol 8 before requiring stable split ids or sending `set-split-ratio`, protocol 9 before decoding stack layouts or sending `new-pane`, and protocol 10 before using per-surface client sizing. diff --git a/cmux-tui/spec/sdk-schema.json b/cmux-tui/spec/sdk-schema.json index 413bc988172c..e7b4c51b3a55 100644 --- a/cmux-tui/spec/sdk-schema.json +++ b/cmux-tui/spec/sdk-schema.json @@ -1230,6 +1230,36 @@ }, "additional_properties": false }, + "ShutdownCleanupStatus": { + "kind": "object", + "fields": { + "pending": { + "type": { + "kind": "scalar", + "name": "uint64" + }, + "presence": "required", + "nullable": false + }, + "retrying": { + "type": { + "kind": "scalar", + "name": "boolean" + }, + "presence": "required", + "nullable": false + }, + "degraded": { + "type": { + "kind": "scalar", + "name": "boolean" + }, + "presence": "required", + "nullable": false + } + }, + "additional_properties": false + }, "IdentifyResult": { "kind": "object", "fields": { @@ -1345,6 +1375,15 @@ "presence": "required", "nullable": false, "since": 9 + }, + "shutdown_cleanup": { + "type": { + "kind": "ref", + "name": "ShutdownCleanupStatus" + }, + "presence": "optional", + "nullable": false, + "since": 10 } }, "additional_properties": false From 022b4d5d3f00fc7052bc5c4aa43790732bca1cbb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:17:22 -0700 Subject: [PATCH 326/521] test(tui): expose ownerless shutdown health gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index eb6a86fabdc8..da86b2c65e50 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24655,6 +24655,10 @@ mod tests { let error = result.unwrap_err(); assert!(format!("{error:#}").contains("load terminal hosts for server shutdown")); assert!(topology_retained); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 1, retrying: false, degraded: true } + ); } #[cfg(unix)] @@ -25124,8 +25128,16 @@ mod tests { assert!(!mux.daemon_shutdown_requested()); assert!(mux.surface(surface).is_some()); assert!(mux.with_state(|state| state.pane_of(surface).is_some())); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 1, retrying: false, degraded: true } + ); mux.set_terminal_close_failure_for_test(false).unwrap(); assert_eq!(mux.close_all_surfaces_for_shutdown().unwrap(), 1); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 0, retrying: false, degraded: false } + ); assert!(!mux.shutdown_requested()); assert!(!mux.daemon_shutdown_requested()); assert!(mux.surface(surface).is_none()); From 2119c454a638f7b9ccbeab1dbfb79cb5bf797770 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:26:17 -0700 Subject: [PATCH 327/521] fix(tui): own complete shutdown health lifecycle --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 91 ++++++++++++++++++-- cmux-tui/crates/cmux-tui/src/localization.rs | 4 +- cmux-tui/spec/commands.md | 2 +- 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index da86b2c65e50..eb5d88e4c7de 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -1586,6 +1586,66 @@ pub(crate) struct ShutdownCleanupHealth { pub(crate) degraded: bool, } +#[derive(Clone, Copy, Debug, Default)] +struct ShutdownCleanupLifecycleState { + pending: bool, + retrying: bool, + degraded: bool, +} + +#[derive(Default)] +struct ShutdownCleanupLifecycle { + state: Mutex, +} + +impl ShutdownCleanupLifecycle { + /// Publish cleanup before closing the creation fence. The state remains + /// pending until one complete top-level cleanup path succeeds, including + /// failures that have no terminal owner in `ShutdownOwnerLedger`. + fn schedule(&self) { + *self.state.lock().unwrap() = + ShutdownCleanupLifecycleState { pending: true, retrying: true, degraded: false }; + } + + fn begin(&self) -> ShutdownCleanupLifecycleGuard<'_> { + self.schedule(); + ShutdownCleanupLifecycleGuard { lifecycle: self, complete: false } + } + + fn snapshot(&self) -> ShutdownCleanupLifecycleState { + *self.state.lock().unwrap() + } + + fn complete(&self) { + *self.state.lock().unwrap() = ShutdownCleanupLifecycleState::default(); + } + + fn degrade(&self) { + *self.state.lock().unwrap() = + ShutdownCleanupLifecycleState { pending: true, retrying: false, degraded: true }; + } +} + +struct ShutdownCleanupLifecycleGuard<'a> { + lifecycle: &'a ShutdownCleanupLifecycle, + complete: bool, +} + +impl ShutdownCleanupLifecycleGuard<'_> { + fn complete(mut self) { + self.lifecycle.complete(); + self.complete = true; + } +} + +impl Drop for ShutdownCleanupLifecycleGuard<'_> { + fn drop(&mut self) { + if !self.complete { + self.lifecycle.degrade(); + } + } +} + #[derive(Debug, Default)] struct SidebarPluginRuntime { options: Option, @@ -2362,6 +2422,7 @@ pub struct Mux { async_surface_creations: AsyncSurfaceCreationGate, shutdown_coordinator: ShutdownCoordinator, shutdown_owners: ShutdownOwnerLedger, + shutdown_cleanup_lifecycle: ShutdownCleanupLifecycle, shutdown_owner_reconciler: Arc, surface_owner_reservations: AtomicUsize, next_id: AtomicU64, @@ -2720,6 +2781,7 @@ impl Mux { async_surface_creations: AsyncSurfaceCreationGate::default(), shutdown_coordinator: ShutdownCoordinator::default(), shutdown_owners: ShutdownOwnerLedger::default(), + shutdown_cleanup_lifecycle: ShutdownCleanupLifecycle::default(), shutdown_owner_reconciler: Arc::new(ShutdownOwnerReconciler::default()), surface_owner_reservations: AtomicUsize::new(0), next_id: AtomicU64::new(next_id), @@ -8304,14 +8366,14 @@ impl Mux { } fn shutdown_once_until(&self, deadline: Instant) -> anyhow::Result<()> { - self.shutting_down.store(true, Ordering::Release); - self.browser_runtime.stop(); - #[cfg(unix)] - self.request_terminal_adoption_stop(); #[cfg(unix)] crate::process_session::require_stable_process_signaling_until(deadline) .context("preflight process control for daemon exit")?; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; + self.shutting_down.store(true, Ordering::Release); + self.browser_runtime.stop(); + #[cfg(unix)] + self.request_terminal_adoption_stop(); if !self.surface_creations.stop_and_wait_until(deadline) { anyhow::bail!("surface creation remains active at the daemon exit deadline"); } @@ -8358,6 +8420,7 @@ impl Mux { /// absolute deadline guarantees an accepted daemon handoff cannot wedge /// the old process or its socket forever. pub fn shutdown(&self) -> anyhow::Result<()> { + let cleanup_lifecycle = self.shutdown_cleanup_lifecycle.begin(); self.shutdown_owner_reconciler.stop(); let overall_deadline = Instant::now() + self.shutdown_total_timeout(); let mut retry_delay = SERVER_EXIT_RETRY_INITIAL_DELAY; @@ -8369,7 +8432,10 @@ impl Mux { } let attempt_deadline = (now + self.shutdown_attempt_timeout()).min(overall_deadline); match self.shutdown_once_until(attempt_deadline) { - Ok(()) => return Ok(()), + Ok(()) => { + cleanup_lifecycle.complete(); + return Ok(()); + } Err(error) => last_error = Some(error), } let Some(remaining) = overall_deadline.checked_duration_since(Instant::now()) else { @@ -8445,12 +8511,16 @@ impl Mux { } pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { - let pending = self.shutdown_owners.len(); + let owner_pending = self.shutdown_owners.len(); + let lifecycle = self.shutdown_cleanup_lifecycle.snapshot(); let state = self.shutdown_owner_reconciler.state.lock().unwrap(); + let owner_retrying = owner_pending != 0 && state.worker_started && !state.degraded; + let owner_degraded = owner_pending != 0 && (state.degraded || !state.worker_started); + let degraded = lifecycle.degraded || owner_degraded; ShutdownCleanupHealth { - pending, - retrying: pending != 0 && state.worker_started && !state.degraded, - degraded: pending != 0 && (state.degraded || !state.worker_started), + pending: owner_pending.max(usize::from(lifecycle.pending)), + retrying: !degraded && (lifecycle.retrying || owner_retrying), + degraded, } } @@ -8470,6 +8540,7 @@ impl Mux { crate::process_session::require_stable_process_signaling_until(deadline) .context("preflight process control for server shutdown")?; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; + let cleanup_lifecycle = self.shutdown_cleanup_lifecycle.begin(); self.shutting_down.store(true, Ordering::Release); self.browser_runtime.stop(); #[cfg(unix)] @@ -8713,6 +8784,7 @@ impl Mux { anyhow::bail!("{}", failures.join("; ")); } + cleanup_lifecycle.complete(); Ok(closed_count) } @@ -8810,6 +8882,7 @@ impl Mux { /// shutdown path. Durable terminal hosts are disconnected by `shutdown` /// and remain available for the replacement daemon to adopt. pub fn request_daemon_shutdown(&self) { + self.shutdown_cleanup_lifecycle.schedule(); self.shutting_down.store(true, Ordering::Release); self.surface_creations.stop(); self.async_surface_creations.stop(); diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index fd675968020d..e33c4bf80d13 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -719,7 +719,7 @@ static ENGLISH: Catalog = Catalog { process_cleanup_unsupported: "this system does not support safe terminal process cleanup", shutdown_timed_out: "the server did not stop within 10 seconds", cleanup_label: "cleanup", - cleanup_pending_label: "pending panes", + cleanup_pending_label: "pending cleanup tasks", cleanup_retrying: "retrying", cleanup_degraded: "run server stop again", }, @@ -1040,7 +1040,7 @@ static JAPANESE: Catalog = Catalog { process_cleanup_unsupported: "このシステムではターミナルプロセスを安全に終了できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", cleanup_label: "クリーンアップ", - cleanup_pending_label: "保留中のペイン", + cleanup_pending_label: "保留中のクリーンアップ", cleanup_retrying: "再試行中", cleanup_degraded: "server stop をもう一度実行してください", }, diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index cc1763218897..3c8bc299925d 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -194,7 +194,7 @@ object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:s `capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `daemon-handoff-force-v1` advertises the optional `force` field on `shutdown-daemon`. `browser-pointer-frame-guard-v1` advertises authoritative `pointer_frame_seq` and `pointer_frame_floor_seq` browser attach/frame state plus the additive `browser-frame-presented`, `browser-mouse-guarded`, and `browser-wheel-guarded` commands. Each admitted bitmap receives a new guard even when its document and dimensions match the previous bitmap. The reported floor through latest range proves route membership only. `browser-frame-presented` advances one exact acknowledged token for that connection, and only that token authorizes a new guarded pointer action. A guarded pointer command implicitly acknowledges its own token. Each connection retains one token, while the bounded browser input queue owns actions admitted before a later presentation. Navigation or geometry changes clear the range and all acknowledgements. An accepted press keeps its original guard for motion across ordinary repaints while document and geometry remain valid; invalidation suppresses further motion but retains its balancing release. A capable client echoes that value in `set-client-info`; browser attach requires the bilateral capability while PTY attach remains available without it. The legacy `browser-mouse` and `browser-wheel` schemas retain their optional guard, but guarded servers reject a missing guard before surface lookup. `viewport-splits-v1` advertises `new-pane-right` and the `Screen.viewport_splits` field. `viewport-column-resize-v1` advertises `set-viewport-pane-width` and `Screen.viewport_base_width`. `layout-undo-v1` advertises server-owned structural layout history and `undo-layout`. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. `server-shutdown-v1` advertises atomic destructive shutdown; clients may use the compatibility cleanup path only when this capability is absent. -`shutdown_cleanup` reports unresolved terminal cleanup, whether a retry is active, and whether cleanup has exhausted its automatic retries. It is an additive protocol 10 field, so clients must accept its absence from older protocol 10 servers. +`shutdown_cleanup` reports unresolved shutdown work, whether a retry is active, and whether cleanup requires another explicit stop request. Its pending count includes terminal owners and ownerless post-fence work such as durable registry or browser-runtime cleanup. It is an additive protocol 10 field, so clients must accept its absence from older protocol 10 servers. Errors: From d788676b98f1df82c9c90eebc2401dfc32ee1a82 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:36:03 -0700 Subject: [PATCH 328/521] test(tui): expose failed-spawn shutdown gap --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 40 ++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index eb5d88e4c7de..674e6a6ee8d0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24705,6 +24705,46 @@ mod tests { assert!(reused_scan, "a shutdown retry spawned another blocked terminal-host record scan"); } + #[cfg(unix)] + #[test] + fn server_shutdown_waits_for_unpublished_child_reaper_ownership() { + const CHILD_ENV: &str = "CMUX_TUI_TEST_UNPUBLISHED_CHILD_REAPER_SHUTDOWN"; + const TEST_NAME: &str = + "mux::tests::server_shutdown_waits_for_unpublished_child_reaper_ownership"; + if std::env::var_os(CHILD_ENV).is_none() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", TEST_NAME]) + .env(CHILD_ENV, "1") + .status() + .unwrap(); + assert!(status.success(), "shutdown ownership subprocess failed: {status}"); + return; + } + + let mux = test_mux(); + let reservation = crate::process_session::reserve_child_reaper().unwrap(); + let error = mux + .close_all_surfaces_for_shutdown_until(Instant::now() + Duration::from_millis(50)) + .unwrap_err(); + + assert!(format!("{error:#}").contains("child reaper")); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 1, retrying: false, degraded: true } + ); + + drop(reservation); + assert_eq!( + mux.close_all_surfaces_for_shutdown_until(Instant::now() + Duration::from_secs(1)) + .unwrap(), + 0 + ); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 0, retrying: false, degraded: false } + ); + } + #[cfg(unix)] #[test] fn server_shutdown_rejects_malformed_host_records_before_removing_topology() { From 54b7f62983b49e84fb7fbffae62a68447a64c937 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:49:39 -0700 Subject: [PATCH 329/521] fix(tui): fence unpublished child cleanup --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 26 +- .../cmux-tui-core/src/process_session.rs | 274 +++++++++++++++--- cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 +- .../src/terminal_host_runtime.rs | 2 +- cmux-tui/crates/cmux-tui/src/localization.rs | 4 +- cmux-tui/spec/commands.md | 2 +- 6 files changed, 268 insertions(+), 42 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 674e6a6ee8d0..fe25701cb256 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8384,6 +8384,9 @@ impl Mux { if !self.wait_for_terminal_adoption_workers_until(deadline) { anyhow::bail!("terminal adoption remains active at the daemon exit deadline"); } + #[cfg(unix)] + crate::process_session::wait_for_unpublished_child_reaper_until(deadline) + .context("drain unpublished child reaper ownership for daemon exit")?; let surfaces = self.state.lock().unwrap().surfaces.values().cloned().collect::>(); for surface in &surfaces { match surface.shutdown_owner_identity() { @@ -8512,14 +8515,24 @@ impl Mux { pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { let owner_pending = self.shutdown_owners.len(); + #[cfg(unix)] + let unpublished_reaper = crate::process_session::unpublished_child_reaper_health(); + #[cfg(unix)] + let (unpublished_pending, unpublished_degraded) = + (unpublished_reaper.pending, unpublished_reaper.degraded); + #[cfg(not(unix))] + let (unpublished_pending, unpublished_degraded) = (0, false); let lifecycle = self.shutdown_cleanup_lifecycle.snapshot(); let state = self.shutdown_owner_reconciler.state.lock().unwrap(); let owner_retrying = owner_pending != 0 && state.worker_started && !state.degraded; let owner_degraded = owner_pending != 0 && (state.degraded || !state.worker_started); - let degraded = lifecycle.degraded || owner_degraded; + let degraded = lifecycle.degraded || owner_degraded || unpublished_degraded; ShutdownCleanupHealth { - pending: owner_pending.max(usize::from(lifecycle.pending)), - retrying: !degraded && (lifecycle.retrying || owner_retrying), + pending: owner_pending + .saturating_add(unpublished_pending) + .max(usize::from(lifecycle.pending)), + retrying: !degraded + && (lifecycle.retrying || owner_retrying || unpublished_pending != 0), degraded, } } @@ -8555,6 +8568,9 @@ impl Mux { if !self.wait_for_terminal_adoption_workers_until(deadline) { anyhow::bail!("terminal adoption did not stop before the shutdown deadline"); } + #[cfg(unix)] + crate::process_session::wait_for_unpublished_child_reaper_until(deadline) + .context("drain unpublished child reaper ownership for server shutdown")?; #[cfg(unix)] let terminal_host_records = { @@ -24723,6 +24739,10 @@ mod tests { let mux = test_mux(); let reservation = crate::process_session::reserve_child_reaper().unwrap(); + assert_eq!( + mux.shutdown_cleanup_health(), + ShutdownCleanupHealth { pending: 1, retrying: true, degraded: false } + ); let error = mux .close_all_surfaces_for_shutdown_until(Instant::now() + Duration::from_millis(50)) .unwrap_err(); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index e53201746ebd..4d2e3cc5dced 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -59,7 +59,7 @@ fn natural_reap_test_session() -> libc::pid_t { pub(crate) fn reserved_child_reaper_active_for_test() -> usize { let Some(slot) = NATURAL_REAPER.get() else { return 0 }; let slot = slot.lock().unwrap(); - slot.as_ref().map_or(0, |reaper| reaper.active.load(Ordering::Acquire)) + slot.as_ref().map_or(0, |reaper| reaper.activity.active.load(Ordering::Acquire)) } #[cfg(test)] @@ -142,6 +142,118 @@ enum NaturalChildObservation { enum NaturalReaperCommand { Add(NaturalReapRequest), WakeSession(libc::pid_t), + WakeUnpublished, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct UnpublishedChildReaperHealth { + pub(crate) pending: usize, + pub(crate) degraded: bool, +} + +#[derive(Default)] +struct NaturalReaperActivityState { + unpublished: usize, + degraded: usize, + unpublished_degraded: usize, +} + +#[derive(Default)] +struct NaturalReaperActivity { + active: AtomicUsize, + state: Mutex, + changed: Condvar, +} + +impl NaturalReaperActivity { + fn reserve(&self) -> io::Result<()> { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { + (active < NATURAL_REAP_CAPACITY).then_some(active + 1) + }) + .map_err(|_| { + io::Error::new(io::ErrorKind::WouldBlock, "PTY session reaper capacity exhausted") + })?; + self.state.lock().unwrap().unpublished += 1; + self.changed.notify_all(); + Ok(()) + } + + fn attach_owner(&self) { + let mut state = self.state.lock().unwrap(); + state.unpublished = state + .unpublished + .checked_sub(1) + .expect("unpublished PTY child reaper reservation is balanced"); + self.changed.notify_all(); + } + + fn release(&self, owner_attached: bool) { + self.active.fetch_sub(1, Ordering::AcqRel); + if !owner_attached { + let mut state = self.state.lock().unwrap(); + state.unpublished = state + .unpublished + .checked_sub(1) + .expect("unpublished PTY child reaper reservation is balanced"); + self.changed.notify_all(); + } + } + + fn set_degraded(&self, owner_attached: bool, degraded: bool) { + let mut state = self.state.lock().unwrap(); + if degraded { + state.degraded += 1; + if !owner_attached { + state.unpublished_degraded += 1; + } + } else { + state.degraded = state + .degraded + .checked_sub(1) + .expect("degraded PTY child reaper accounting is balanced"); + if !owner_attached { + state.unpublished_degraded = state + .unpublished_degraded + .checked_sub(1) + .expect("unpublished degraded PTY child reaper accounting is balanced"); + } + } + } + + fn unpublished_health(&self) -> UnpublishedChildReaperHealth { + let state = self.state.lock().unwrap(); + UnpublishedChildReaperHealth { + pending: state.unpublished, + degraded: state.unpublished_degraded != 0, + } + } + + fn wait_for_unpublished_until(&self, deadline: Instant) -> io::Result<()> { + let mut state = self.state.lock().unwrap(); + while state.unpublished != 0 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(unpublished_child_reaper_deadline_error(&state)); + }; + let (next, timeout) = self.changed.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.unpublished != 0 { + return Err(unpublished_child_reaper_deadline_error(&state)); + } + } + Ok(()) + } +} + +fn unpublished_child_reaper_deadline_error(state: &NaturalReaperActivityState) -> io::Error { + io::Error::new( + io::ErrorKind::TimedOut, + format!( + "unpublished child reaper ownership did not drain before the shutdown deadline \ + (pending: {}, degraded: {})", + state.unpublished, state.unpublished_degraded + ), + ) } struct NaturalReapRequest { @@ -163,7 +275,7 @@ impl NaturalReapRequest { fn clear_degraded(&mut self) { if self.degraded { self.degraded = false; - self._lease.degraded.fetch_sub(1, Ordering::AcqRel); + self._lease.activity.set_degraded(self._lease.owner_attached, false); } } @@ -186,7 +298,7 @@ impl NaturalReapRequest { if self.attempts >= NATURAL_REAP_MAX_ATTEMPTS { if !self.degraded { self.degraded = true; - self._lease.degraded.fetch_add(1, Ordering::AcqRel); + self._lease.activity.set_degraded(self._lease.owner_attached, true); } self.next_attempt = Instant::now() + NATURAL_REAP_DEGRADED_RETRY; } else { @@ -196,30 +308,49 @@ impl NaturalReapRequest { } } +impl Drop for NaturalReapRequest { + fn drop(&mut self) { + self.clear_degraded(); + } +} + struct NaturalReaper { sender: mpsc::Sender, - active: Arc, - degraded: Arc, + activity: Arc, _worker: std::thread::JoinHandle<()>, } pub(crate) struct ReservedChildReaperLease { sender: mpsc::Sender, - active: Arc, - degraded: Arc, + activity: Arc, + owner_attached: bool, +} + +impl ReservedChildReaperLease { + /// Transfer cleanup authority to a surface or terminal-host owner. + /// + /// Until this transition, shutdown must treat the reservation as the sole + /// owner of any child created after it. Failed-spawn cleanup deliberately + /// keeps the reservation unpublished until the shared reaper completes. + #[must_use] + pub(crate) fn attach_owner(mut self) -> Self { + assert!(!self.owner_attached, "PTY child reaper already has a published owner"); + self.activity.attach_owner(); + self.owner_attached = true; + self + } } impl Drop for ReservedChildReaperLease { fn drop(&mut self) { - self.active.fetch_sub(1, Ordering::AcqRel); + self.activity.release(self.owner_attached); } } impl NaturalReaper { fn start() -> io::Result { let (sender, receiver) = mpsc::channel(); - let active = Arc::new(AtomicUsize::new(0)); - let degraded = Arc::new(AtomicUsize::new(0)); + let activity = Arc::new(NaturalReaperActivity::default()); let worker = std::thread::Builder::new().name("cmux-pty-session-reaper".into()).spawn( move || { #[cfg(test)] @@ -229,21 +360,15 @@ impl NaturalReaper { NATURAL_REAP_WORKERS.fetch_sub(1, Ordering::AcqRel); }, )?; - Ok(Self { sender, active, degraded, _worker: worker }) + Ok(Self { sender, activity, _worker: worker }) } fn lease(&self) -> io::Result { - self.active - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |active| { - (active < NATURAL_REAP_CAPACITY).then_some(active + 1) - }) - .map_err(|_| { - io::Error::new(io::ErrorKind::WouldBlock, "PTY session reaper capacity exhausted") - })?; + self.activity.reserve()?; Ok(ReservedChildReaperLease { sender: self.sender.clone(), - active: self.active.clone(), - degraded: self.degraded.clone(), + activity: self.activity.clone(), + owner_attached: false, }) } } @@ -257,6 +382,35 @@ pub(crate) fn reserve_child_reaper() -> io::Result { slot.as_ref().expect("natural PTY reaper initialized").lease() } +pub(crate) fn unpublished_child_reaper_health() -> UnpublishedChildReaperHealth { + let Some(slot) = NATURAL_REAPER.get() else { + return UnpublishedChildReaperHealth::default(); + }; + let activity = { + let slot = slot.lock().unwrap(); + let Some(reaper) = slot.as_ref() else { + return UnpublishedChildReaperHealth::default(); + }; + reaper.activity.clone() + }; + activity.unpublished_health() +} + +/// Wake ownerless child cleanup and retain the server until every reservation +/// either publishes a higher-level owner or releases its wait ownership. +pub(crate) fn wait_for_unpublished_child_reaper_until(deadline: Instant) -> io::Result<()> { + let Some(slot) = NATURAL_REAPER.get() else { return Ok(()) }; + let (sender, activity) = { + let slot = slot.lock().unwrap(); + let Some(reaper) = slot.as_ref() else { return Ok(()) }; + (reaper.sender.clone(), reaper.activity.clone()) + }; + sender.send(NaturalReaperCommand::WakeUnpublished).map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "unpublished child reaper is unavailable") + })?; + activity.wait_for_unpublished_until(deadline) +} + pub(crate) fn observe_child_without_reaping( pid: libc::pid_t, nonblocking: bool, @@ -621,6 +775,12 @@ fn accept_natural_reaper_command( request.next_attempt = now; } } + NaturalReaperCommand::WakeUnpublished => { + let now = Instant::now(); + for request in pending.iter_mut().filter(|request| !request._lease.owner_attached) { + request.next_attempt = now; + } + } } } @@ -1656,7 +1816,7 @@ mod tests { let (finished_sender, finished_receiver) = mpsc::channel(); let session = natural_reap_test_session(); enqueue_reserved_session_leader( - reserve_child_reaper().unwrap(), + reserve_child_reaper().unwrap().attach_owner(), session, Duration::ZERO, move || { @@ -1699,7 +1859,7 @@ mod tests { let (finished_sender, finished_receiver) = mpsc::channel(); let session = natural_reap_test_session(); enqueue_reserved_session_leader( - reserve_child_reaper().unwrap(), + reserve_child_reaper().unwrap().attach_owner(), session, Duration::ZERO, || Ok(true), @@ -1735,7 +1895,7 @@ mod tests { fn degraded_reap_request_for_test( session: libc::pid_t, next_attempt: Instant, - degraded: Arc, + activity: Arc, ) -> NaturalReapRequest { let (sender, _receiver) = mpsc::channel(); NaturalReapRequest { @@ -1746,11 +1906,7 @@ mod tests { needs_cleanup: Box::new(|| false), prepare_cleanup: Box::new(|| true), finish: Box::new(|_| NaturalReapFinish::Pending), - _lease: ReservedChildReaperLease { - sender, - active: Arc::new(AtomicUsize::new(1)), - degraded, - }, + _lease: ReservedChildReaperLease { sender, activity, owner_attached: true }, next_attempt, retry_delay: NATURAL_REAP_RETRY_MAX, attempts: NATURAL_REAP_MAX_ATTEMPTS, @@ -1758,6 +1914,22 @@ mod tests { } } + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn natural_reaper_activity_for_test( + active: usize, + degraded: usize, + ) -> Arc { + Arc::new(NaturalReaperActivity { + active: AtomicUsize::new(active), + state: Mutex::new(NaturalReaperActivityState { + unpublished: 0, + degraded, + unpublished_degraded: 0, + }), + changed: Condvar::new(), + }) + } + #[cfg(any(target_os = "macos", target_os = "linux"))] fn accept_session_state_change_for_test( session: libc::pid_t, @@ -1771,11 +1943,11 @@ mod tests { fn state_change_wake_preempts_degraded_retry_before_shutdown_deadline() { let now = Instant::now(); let shutdown_deadline = now + Duration::from_secs(5); - let degraded = Arc::new(AtomicUsize::new(1)); + let activity = natural_reaper_activity_for_test(1, 1); let mut pending = vec![degraded_reap_request_for_test( 41, now + Duration::from_secs(30), - degraded.clone(), + activity.clone(), )]; accept_session_state_change_for_test(41, &mut pending); @@ -1784,7 +1956,7 @@ mod tests { "the production degraded retry outlived the server shutdown deadline" ); assert!(pending[0].degraded); - assert_eq!(degraded.load(Ordering::Acquire), 1); + assert_eq!(activity.state.lock().unwrap().degraded, 1); let rescheduled_at = Instant::now(); pending[0].schedule_failure(); @@ -1800,10 +1972,10 @@ mod tests { fn state_change_wake_preserves_unrelated_degraded_backoff() { let now = Instant::now(); let shutdown_deadline = now + Duration::from_secs(5); - let degraded = Arc::new(AtomicUsize::new(2)); + let activity = natural_reaper_activity_for_test(2, 2); let mut pending = vec![ - degraded_reap_request_for_test(41, now + Duration::from_secs(30), degraded.clone()), - degraded_reap_request_for_test(73, now + Duration::from_secs(30), degraded), + degraded_reap_request_for_test(41, now + Duration::from_secs(30), activity.clone()), + degraded_reap_request_for_test(73, now + Duration::from_secs(30), activity), ]; accept_session_state_change_for_test(41, &mut pending); @@ -1814,6 +1986,40 @@ mod tests { ); } + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn shutdown_wake_preempts_only_unpublished_degraded_reapers() { + let now = Instant::now(); + let shutdown_deadline = now + Duration::from_secs(5); + let activity = Arc::new(NaturalReaperActivity { + active: AtomicUsize::new(2), + state: Mutex::new(NaturalReaperActivityState { + unpublished: 1, + degraded: 2, + unpublished_degraded: 1, + }), + changed: Condvar::new(), + }); + let mut unpublished = + degraded_reap_request_for_test(41, now + Duration::from_secs(30), activity.clone()); + unpublished._lease.owner_attached = false; + let published = + degraded_reap_request_for_test(73, now + Duration::from_secs(30), activity.clone()); + let mut pending = vec![unpublished, published]; + + accept_natural_reaper_command(NaturalReaperCommand::WakeUnpublished, &mut pending); + + assert!(pending[0].next_attempt <= shutdown_deadline); + assert!( + pending[1].next_attempt > shutdown_deadline, + "shutdown defeated a published surface reaper's degraded backoff" + ); + assert_eq!( + activity.unpublished_health(), + UnpublishedChildReaperHealth { pending: 1, degraded: true } + ); + } + #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn ignored_sigchld_fails_before_reserving_a_child_reaper() { diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 3e4923647653..f6a79a32a588 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1488,7 +1488,7 @@ impl LocalPtyProcess { let reap_surface = surface; let mut child = Some(child); crate::process_session::enqueue_reserved_session_leader( - reaper, + reaper.attach_owner(), session, NORMAL_EXIT_SESSION_CLEANUP_TIMEOUT, move || { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index aa164d83e59d..7a9e688c42c2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4882,7 +4882,7 @@ mod unix { .expect("terminal-host PTY child retains its reserved session reaper"); let mut child = Some(child); crate::process_session::enqueue_reserved_session_leader( - reaper, + reaper.attach_owner(), session, HOST_KILL_WAIT, move || { diff --git a/cmux-tui/crates/cmux-tui/src/localization.rs b/cmux-tui/crates/cmux-tui/src/localization.rs index e33c4bf80d13..7dda8716610b 100644 --- a/cmux-tui/crates/cmux-tui/src/localization.rs +++ b/cmux-tui/crates/cmux-tui/src/localization.rs @@ -714,7 +714,7 @@ static ENGLISH: Catalog = Catalog { #[cfg(unix)] legacy_signal_failed: "The verified older server process could not be stopped.", shutdown_failed: "the server could not stop cleanly", - shutdown_cleanup_incomplete: "the server could not finish pane cleanup before the shutdown deadline", + shutdown_cleanup_incomplete: "the server could not finish cleanup before the shutdown deadline", shutdown_unsupported: "this server cannot be stopped by this client", process_cleanup_unsupported: "this system does not support safe terminal process cleanup", shutdown_timed_out: "the server did not stop within 10 seconds", @@ -1035,7 +1035,7 @@ static JAPANESE: Catalog = Catalog { #[cfg(unix)] legacy_signal_failed: "検証済みの古いサーバープロセスを停止できませんでした。", shutdown_failed: "サーバーを正常に停止できませんでした", - shutdown_cleanup_incomplete: "シャットダウン期限までにペインのクリーンアップを完了できませんでした", + shutdown_cleanup_incomplete: "シャットダウン期限までにクリーンアップを完了できませんでした", shutdown_unsupported: "このクライアントからこのサーバーを停止できません", process_cleanup_unsupported: "このシステムではターミナルプロセスを安全に終了できません", shutdown_timed_out: "10 秒以内にサーバーが停止しませんでした", diff --git a/cmux-tui/spec/commands.md b/cmux-tui/spec/commands.md index 3c8bc299925d..12e237a358b5 100644 --- a/cmux-tui/spec/commands.md +++ b/cmux-tui/spec/commands.md @@ -194,7 +194,7 @@ object{app:"cmux-tui",version:string,build_commit?:string|null,ghostty_commit?:s `capabilities` is additive build-level feature negotiation within a protocol version. Clients must treat a missing field as an empty list. `daemon-handoff-force-v1` advertises the optional `force` field on `shutdown-daemon`. `browser-pointer-frame-guard-v1` advertises authoritative `pointer_frame_seq` and `pointer_frame_floor_seq` browser attach/frame state plus the additive `browser-frame-presented`, `browser-mouse-guarded`, and `browser-wheel-guarded` commands. Each admitted bitmap receives a new guard even when its document and dimensions match the previous bitmap. The reported floor through latest range proves route membership only. `browser-frame-presented` advances one exact acknowledged token for that connection, and only that token authorizes a new guarded pointer action. A guarded pointer command implicitly acknowledges its own token. Each connection retains one token, while the bounded browser input queue owns actions admitted before a later presentation. Navigation or geometry changes clear the range and all acknowledgements. An accepted press keeps its original guard for motion across ordinary repaints while document and geometry remain valid; invalidation suppresses further motion but retains its balancing release. A capable client echoes that value in `set-client-info`; browser attach requires the bilateral capability while PTY attach remains available without it. The legacy `browser-mouse` and `browser-wheel` schemas retain their optional guard, but guarded servers reject a missing guard before surface lookup. `viewport-splits-v1` advertises `new-pane-right` and the `Screen.viewport_splits` field. `viewport-column-resize-v1` advertises `set-viewport-pane-width` and `Screen.viewport_base_width`. `layout-undo-v1` advertises server-owned structural layout history and `undo-layout`. `provider-managed-workspace-authority-v2` advertises pre-provisioned provider ownership and authority-gated post-provider rename and close commits. `server-shutdown-v1` advertises atomic destructive shutdown; clients may use the compatibility cleanup path only when this capability is absent. -`shutdown_cleanup` reports unresolved shutdown work, whether a retry is active, and whether cleanup requires another explicit stop request. Its pending count includes terminal owners and ownerless post-fence work such as durable registry or browser-runtime cleanup. It is an additive protocol 10 field, so clients must accept its absence from older protocol 10 servers. +`shutdown_cleanup` reports unresolved shutdown work, whether a retry is active, and whether cleanup requires another explicit stop request. Its pending count includes terminal owners, unpublished child-reaper reservations, and ownerless post-fence work such as durable registry or browser-runtime cleanup. It is an additive protocol 10 field, so clients must accept its absence from older protocol 10 servers. Errors: From 6ae03fee945e88601fc3755a2296c54756ef8119 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 20:58:17 -0700 Subject: [PATCH 330/521] test(tui): expose merged shutdown regressions --- cmux-tui/crates/cmux-tui/src/main.rs | 43 +++++++++++++++++++ .../crates/cmux-tui/src/remote_runtime.rs | 21 +++++++++ tests/test_tui_npm_package_artifact.py | 2 + 3 files changed, 66 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 8979ebd7741c..2c0773aa6f5c 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1477,6 +1477,8 @@ struct ServerShutdownCleanup { changed: std::sync::Condvar, #[cfg(test)] shutdown_attempt: std::sync::Mutex>, + #[cfg(test)] + process_owner_timeout: std::sync::Mutex>, } impl ServerShutdownCleanup { @@ -1500,6 +1502,8 @@ impl ServerShutdownCleanup { shutdown_attempt: std::sync::Mutex::new( SERVER_SHUTDOWN_ATTEMPT_FOR_RUN.with(|slot| slot.borrow().clone()), ), + #[cfg(test)] + process_owner_timeout: std::sync::Mutex::new(None), } } @@ -1522,6 +1526,12 @@ impl ServerShutdownCleanup { Some(Arc::new(attempt)); } + #[cfg(test)] + fn set_process_owner_timeout_for_test(&self, timeout: std::time::Duration) { + *self.process_owner_timeout.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(timeout); + } + fn run_once(&self) -> ServerShutdownCleanupOutcome { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let (attempt, retry) = loop { @@ -2606,6 +2616,39 @@ mod tests { assert_shutdown_attempt_returns(|| panic!("forced shutdown panic")); } + #[cfg(unix)] + #[test] + fn server_process_owner_shutdown_obeys_cleanup_deadline() { + let process_owners = Arc::new(ServerProcessOwnersCleanup::new()); + let (release, blocked) = std::sync::mpsc::channel(); + let remote_thread = std::thread::spawn(move || { + let _ = blocked.recv(); + Ok(()) + }); + process_owners + .register_remote(remote_runtime::DaemonRuntimeHandle::from_test_thread(remote_thread)); + process_owners.seal(); + + let mux = Mux::new("bounded-process-owner-test", SurfaceOptions::default()); + let cleanup = Arc::new(ServerShutdownCleanup::new_with_process_owners(mux, process_owners)); + cleanup.set_shutdown_attempt_for_test(|| Ok(())); + cleanup.set_process_owner_timeout_for_test(std::time::Duration::from_millis(50)); + let worker_cleanup = cleanup.clone(); + let (returned, observed) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + let _ = returned.send(worker_cleanup.run_until_complete()); + }); + + let bounded = observed.recv_timeout(std::time::Duration::from_millis(350)); + let _ = release.send(()); + if bounded.is_ok() { + cleanup.prepare_retry(0); + cleanup.run_until_complete().unwrap(); + } + worker.join().unwrap(); + assert!(bounded.is_ok(), "server process-owner shutdown ignored the cleanup deadline"); + } + #[test] fn failed_shutdown_retains_server_until_explicit_retry() { let mux = Mux::new("shutdown-retry-test", SurfaceOptions::default()); diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index a3e5671e2bac..3862264d067b 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -348,6 +348,27 @@ impl DaemonRuntimeHandle { self.thread.as_ref().is_some_and(thread::JoinHandle::is_finished) } + #[cfg(test)] + pub(crate) fn from_test_thread(thread: thread::JoinHandle>) -> Self { + let (shutdown, _) = watch::channel(false); + Self { + info: DaemonRuntimeInfo { + session: "test".into(), + state_dir: PathBuf::new(), + link_socket: PathBuf::new(), + admin_socket: PathBuf::new(), + daemon_fingerprint: String::new(), + routes: Vec::new(), + direct_websocket: None, + iroh_node_id: None, + lifecycle_id: None, + replaceable_sidecar: false, + }, + shutdown, + thread: Some(thread), + } + } + pub fn shutdown(mut self) -> anyhow::Result<()> { #[cfg(debug_assertions)] let shutdown_marker = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER"); diff --git a/tests/test_tui_npm_package_artifact.py b/tests/test_tui_npm_package_artifact.py index c1782106b357..9078df812e80 100644 --- a/tests/test_tui_npm_package_artifact.py +++ b/tests/test_tui_npm_package_artifact.py @@ -96,6 +96,8 @@ def test_build_workflow_executes_stamped_raw_and_packaged_artifacts() -> None: assert build.count('CMUX_TUI_NPM_BOOTSTRAP_VERSION="$VERSION"') == 3 assert 'CMUX_TUI_DISTRIBUTION_VERSION="${{ inputs.version }}"' not in build assert 'if [[ "${{ inputs.package_npm }}" == "true" ]]; then' not in build + assert "EXPECTED_VERSION: ${{ inputs.version }}" in build + assert "(inputs.package_npm || inputs.package_pypi)" not in build assert build.count("verify_artifact_identity.py") == 4 assert "dist/npm-smoke/node_modules/cmux/bin/cmux.js" in build assert "/tmp/cmux-tui-wheel-smoke/bin/cmux" in build From ca5a9e3bc8068a8253fb20f7a77893957a48ffb9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:16:39 -0700 Subject: [PATCH 331/521] fix(tui): close merged release and shutdown gaps --- .github/workflows/cmux-tui-build-package.yml | 23 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 +- cmux-tui/crates/cmux-tui/src/main.rs | 198 +++++++++++++----- .../crates/cmux-tui/src/remote_runtime.rs | 46 +++- 4 files changed, 201 insertions(+), 73 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 3f10cd1788ad..474279cff7ea 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -134,8 +134,8 @@ jobs: - name: Build cmux-tui (native) if: matrix.cross == false env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} PACKAGE_NPM: ${{ inputs.package_npm }} + VERSION: ${{ inputs.version }} working-directory: cmux-tui shell: bash run: | @@ -144,9 +144,10 @@ jobs: unset CMUX_GHOSTTY_SRC CMUX_TUI_BUILD_COMMIT="$(git -C .. rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ../ghostty rev-parse HEAD)" + CMUX_TUI_DISTRIBUTION_VERSION="$VERSION" export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT CMUX_TUI_DISTRIBUTION_VERSION if [[ "$PACKAGE_NPM" == "true" ]]; then - CMUX_TUI_NPM_BOOTSTRAP_VERSION="$CMUX_TUI_DISTRIBUTION_VERSION" + CMUX_TUI_NPM_BOOTSTRAP_VERSION="$VERSION" export CMUX_TUI_NPM_BOOTSTRAP_VERSION fi cargo build -p cmux-tui --bin cmux-tui --release --locked --target ${{ matrix.build_target }} @@ -155,8 +156,8 @@ jobs: - name: Build cmux-tui (cross) if: matrix.cross == true env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} PACKAGE_NPM: ${{ inputs.package_npm }} + VERSION: ${{ inputs.version }} working-directory: cmux-tui shell: bash run: | @@ -165,9 +166,10 @@ jobs: unset CMUX_GHOSTTY_SRC CMUX_TUI_BUILD_COMMIT="$(git -C .. rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ../ghostty rev-parse HEAD)" + CMUX_TUI_DISTRIBUTION_VERSION="$VERSION" export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT CMUX_TUI_DISTRIBUTION_VERSION if [[ "$PACKAGE_NPM" == "true" ]]; then - CMUX_TUI_NPM_BOOTSTRAP_VERSION="$CMUX_TUI_DISTRIBUTION_VERSION" + CMUX_TUI_NPM_BOOTSTRAP_VERSION="$VERSION" export CMUX_TUI_NPM_BOOTSTRAP_VERSION fi if [[ "$RUNNER_OS" == macOS ]]; then @@ -205,14 +207,8 @@ jobs: if: matrix.cross == false shell: bash env: - EXPECTED_VERSION: ${{ (inputs.package_npm || inputs.package_pypi) && inputs.version || '' }} + EXPECTED_VERSION: ${{ inputs.version }} run: | - if [[ -z "$EXPECTED_VERSION" ]]; then - EXPECTED_VERSION="$( - cargo metadata --manifest-path cmux-tui/Cargo.toml --no-deps --format-version 1 | - python3 -c 'import json, sys; print(next(package["version"] for package in json.load(sys.stdin)["packages"] if package["name"] == "cmux-tui"))' - )" - fi python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ --binary "dist/cmux-tui-${{ matrix.target }}${{ matrix.ext }}" \ --version "$EXPECTED_VERSION" \ @@ -377,8 +373,8 @@ jobs: - name: Build libghostty-vt + cmux-tui (Windows GNU) env: - CMUX_TUI_DISTRIBUTION_VERSION: ${{ inputs.version }} PACKAGE_NPM: ${{ inputs.package_npm }} + VERSION: ${{ inputs.version }} shell: bash run: | # Keep the manual Zig build and Cargo's build script on the same @@ -386,9 +382,10 @@ jobs: unset CMUX_GHOSTTY_SRC CMUX_TUI_BUILD_COMMIT="$(git rev-parse HEAD)" CMUX_TUI_GHOSTTY_COMMIT="$(git -C ghostty rev-parse HEAD)" + CMUX_TUI_DISTRIBUTION_VERSION="$VERSION" export CMUX_TUI_BUILD_COMMIT CMUX_TUI_GHOSTTY_COMMIT CMUX_TUI_DISTRIBUTION_VERSION if [[ "$PACKAGE_NPM" == "true" ]]; then - CMUX_TUI_NPM_BOOTSTRAP_VERSION="$CMUX_TUI_DISTRIBUTION_VERSION" + CMUX_TUI_NPM_BOOTSTRAP_VERSION="$VERSION" export CMUX_TUI_NPM_BOOTSTRAP_VERSION fi cd ghostty diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fe25701cb256..5c845ecb38ac 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8423,9 +8423,14 @@ impl Mux { /// absolute deadline guarantees an accepted daemon handoff cannot wedge /// the old process or its socket forever. pub fn shutdown(&self) -> anyhow::Result<()> { + self.shutdown_until(Instant::now() + self.shutdown_total_timeout()) + } + + /// Finish cleanup within a caller-owned absolute deadline so surrounding + /// process owners and mux resources share one bounded handoff window. + pub fn shutdown_until(&self, overall_deadline: Instant) -> anyhow::Result<()> { let cleanup_lifecycle = self.shutdown_cleanup_lifecycle.begin(); self.shutdown_owner_reconciler.stop(); - let overall_deadline = Instant::now() + self.shutdown_total_timeout(); let mut retry_delay = SERVER_EXIT_RETRY_INITIAL_DELAY; let mut last_error = None; loop { diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 2c0773aa6f5c..e03e313e8a3e 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1312,18 +1312,31 @@ struct ServerProcessOwners { } impl ServerProcessOwners { - fn shutdown(mut self) -> anyhow::Result<()> { + fn shutdown_until(&mut self, deadline: std::time::Instant) -> ServerProcessOwnersShutdown { #[cfg(target_os = "linux")] drop(self.provider_management.take()); drop(self.websocket.take()); #[cfg(unix)] - if let Some(remote) = self.remote.take() { - remote.shutdown()?; + if let Some(remote) = self.remote.as_mut() { + match remote.shutdown_until(deadline) { + remote_runtime::DaemonRuntimeShutdown::Complete(result) => { + self.remote.take(); + return ServerProcessOwnersShutdown::Complete(result); + } + remote_runtime::DaemonRuntimeShutdown::Pending => { + return ServerProcessOwnersShutdown::Pending; + } + } } - Ok(()) + ServerProcessOwnersShutdown::Complete(Ok(())) } } +enum ServerProcessOwnersShutdown { + Complete(anyhow::Result<()>), + Pending, +} + enum ServerProcessOwnersState { Registering(ServerProcessOwners), Ready(ServerProcessOwners), @@ -1331,6 +1344,11 @@ enum ServerProcessOwnersState { Complete(Option), } +enum ServerProcessOwnersCleanupOutcome { + Complete { error: Option }, + Incomplete { message: String }, +} + struct ServerProcessOwnersCleanup { state: std::sync::Mutex, changed: std::sync::Condvar, @@ -1403,13 +1421,38 @@ impl ServerProcessOwnersCleanup { } } - fn run_once(&self) -> anyhow::Result<()> { + fn run_once_until(&self, deadline: std::time::Instant) -> ServerProcessOwnersCleanupOutcome { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - let owners = loop { + let mut owners = loop { match &*state { ServerProcessOwnersState::Registering(_) | ServerProcessOwnersState::Running => { - state = - self.changed.wait(state).unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(remaining) = + deadline.checked_duration_since(std::time::Instant::now()) + else { + return ServerProcessOwnersCleanupOutcome::Incomplete { + message: + "server process owner registration exceeded the shutdown deadline" + .into(), + }; + }; + let (next, timeout) = self + .changed + .wait_timeout(state, remaining) + .unwrap_or_else(std::sync::PoisonError::into_inner); + state = next; + if timeout.timed_out() + && matches!( + &*state, + ServerProcessOwnersState::Registering(_) + | ServerProcessOwnersState::Running + ) + { + return ServerProcessOwnersCleanupOutcome::Incomplete { + message: + "server process owner registration exceeded the shutdown deadline" + .into(), + }; + } } ServerProcessOwnersState::Ready(_) => { let ServerProcessOwnersState::Ready(owners) = @@ -1420,22 +1463,37 @@ impl ServerProcessOwnersCleanup { break owners; } ServerProcessOwnersState::Complete(error) => { - return error - .as_ref() - .map(|message| Err(anyhow::anyhow!(message.clone()))) - .unwrap_or(Ok(())); + return ServerProcessOwnersCleanupOutcome::Complete { error: error.clone() }; } } }; drop(state); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| owners.shutdown())) - .unwrap_or_else(|_| Err(anyhow::anyhow!("server process owner cleanup panicked"))); - let error = result.as_ref().err().map(|error| format!("{error:#}")); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + owners.shutdown_until(deadline) + })); state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - *state = ServerProcessOwnersState::Complete(error); + let outcome = match result { + Ok(ServerProcessOwnersShutdown::Complete(result)) => { + let error = result.as_ref().err().map(|error| format!("{error:#}")); + *state = ServerProcessOwnersState::Complete(error.clone()); + ServerProcessOwnersCleanupOutcome::Complete { error } + } + Ok(ServerProcessOwnersShutdown::Pending) => { + *state = ServerProcessOwnersState::Ready(owners); + ServerProcessOwnersCleanupOutcome::Incomplete { + message: "remote daemon cleanup exceeded the shutdown deadline".into(), + } + } + Err(_) => { + *state = ServerProcessOwnersState::Ready(owners); + ServerProcessOwnersCleanupOutcome::Incomplete { + message: "server process owner cleanup panicked".into(), + } + } + }; self.changed.notify_all(); - result + outcome } } @@ -1507,14 +1565,14 @@ impl ServerShutdownCleanup { } } - fn attempt_mux_shutdown(&self) -> anyhow::Result<()> { + fn attempt_mux_shutdown_until(&self, deadline: std::time::Instant) -> anyhow::Result<()> { #[cfg(test)] if let Some(attempt) = self.shutdown_attempt.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone() { return attempt(); } - self.mux.shutdown() + self.mux.shutdown_until(deadline) } #[cfg(test)] @@ -1532,6 +1590,16 @@ impl ServerShutdownCleanup { Some(timeout); } + fn cleanup_deadline(&self) -> std::time::Instant { + #[cfg(test)] + if let Some(timeout) = + *self.process_owner_timeout.lock().unwrap_or_else(std::sync::PoisonError::into_inner) + { + return std::time::Instant::now() + timeout; + } + std::time::Instant::now() + cmux_tui_core::server::SERVER_SHUTDOWN_TIMEOUT + } + fn run_once(&self) -> ServerShutdownCleanupOutcome { let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let (attempt, retry) = loop { @@ -1563,39 +1631,56 @@ impl ServerShutdownCleanup { }; drop(state); - // Runtime and listener owners join exactly once before Mux::shutdown. - // A mux failure retains the socket until another explicit request, - // while a joined runtime error is terminal and remains reportable. - let process_owner_error = - self.process_owners.run_once().err().map(|error| format!("{error:#}")); - let result = - std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.attempt_mux_shutdown())); - let outcome = match result { - Ok(Ok(())) => ServerShutdownCleanupOutcome::Complete { error: process_owner_error }, - Ok(Err(error)) => ServerShutdownCleanupOutcome::Degraded { - attempt, - message: format!( - "{}: {error:#}{}", - localization::catalog().server.shutdown_cleanup_incomplete, - process_owner_error - .as_deref() - .map(|owner| format!("; process owner cleanup: {owner}")) - .unwrap_or_default() - ), - retry, - }, - Err(_) => ServerShutdownCleanupOutcome::Degraded { - attempt, - message: format!( - "{}{}", - localization::catalog().server.shutdown_cleanup_incomplete, - process_owner_error - .as_deref() - .map(|owner| format!("; process owner cleanup: {owner}")) - .unwrap_or_default() - ), - retry, - }, + // Runtime owners and mux resources share one absolute deadline. If an + // owner is still live, retain it and the published socket until an + // explicit retry instead of starting a second cleanup budget. + let deadline = self.cleanup_deadline(); + let process_owner_outcome = self.process_owners.run_once_until(deadline); + let outcome = match process_owner_outcome { + ServerProcessOwnersCleanupOutcome::Incomplete { message } => { + ServerShutdownCleanupOutcome::Degraded { + attempt, + message: format!( + "{}: {message}", + localization::catalog().server.shutdown_cleanup_incomplete + ), + retry, + } + } + ServerProcessOwnersCleanupOutcome::Complete { error: process_owner_error } => { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.attempt_mux_shutdown_until(deadline) + })); + match result { + Ok(Ok(())) => { + ServerShutdownCleanupOutcome::Complete { error: process_owner_error } + } + Ok(Err(error)) => ServerShutdownCleanupOutcome::Degraded { + attempt, + message: format!( + "{}: {error:#}{}", + localization::catalog().server.shutdown_cleanup_incomplete, + process_owner_error + .as_deref() + .map(|owner| format!("; process owner cleanup: {owner}")) + .unwrap_or_default() + ), + retry, + }, + Err(_) => ServerShutdownCleanupOutcome::Degraded { + attempt, + message: format!( + "{}{}", + localization::catalog().server.shutdown_cleanup_incomplete, + process_owner_error + .as_deref() + .map(|owner| format!("; process owner cleanup: {owner}")) + .unwrap_or_default() + ), + retry, + }, + } + } }; let next_state = match &outcome { ServerShutdownCleanupOutcome::Complete { error } => { @@ -2640,13 +2725,18 @@ mod tests { }); let bounded = observed.recv_timeout(std::time::Duration::from_millis(350)); + let returned_in_time = bounded.is_ok(); + let returned_incomplete = matches!(&bounded, Ok(Err(_))); + let retained_remote_owner = !cleanup.process_owners.remote_is_finished(); let _ = release.send(()); - if bounded.is_ok() { + if returned_in_time { cleanup.prepare_retry(0); cleanup.run_until_complete().unwrap(); } worker.join().unwrap(); - assert!(bounded.is_ok(), "server process-owner shutdown ignored the cleanup deadline"); + assert!(returned_in_time, "server process-owner shutdown ignored the cleanup deadline"); + assert!(returned_incomplete, "bounded owner cleanup did not request an explicit retry"); + assert!(retained_remote_owner, "bounded owner cleanup discarded the live remote runtime"); } #[test] diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index 3862264d067b..a22bf1baf7dd 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -339,6 +339,11 @@ pub struct DaemonRuntimeHandle { thread: Option>>, } +pub(crate) enum DaemonRuntimeShutdown { + Complete(anyhow::Result<()>), + Pending, +} + impl DaemonRuntimeHandle { pub fn info(&self) -> &DaemonRuntimeInfo { &self.info @@ -348,6 +353,40 @@ impl DaemonRuntimeHandle { self.thread.as_ref().is_some_and(thread::JoinHandle::is_finished) } + fn request_shutdown(&self) { + let _ = self.shutdown.send(true); + } + + pub(crate) fn shutdown_until(&mut self, deadline: std::time::Instant) -> DaemonRuntimeShutdown { + self.request_shutdown(); + loop { + let Some(worker) = self.thread.as_ref() else { + return DaemonRuntimeShutdown::Complete(Err(anyhow!( + "remote daemon runtime thread is absent" + ))); + }; + if worker.is_finished() { + return DaemonRuntimeShutdown::Complete(self.join_runtime_thread()); + } + let Some(remaining) = deadline.checked_duration_since(std::time::Instant::now()) else { + return DaemonRuntimeShutdown::Pending; + }; + thread::sleep(remaining.min(Duration::from_millis(10))); + } + } + + fn join_runtime_thread(&mut self) -> anyhow::Result<()> { + match self + .thread + .take() + .ok_or_else(|| anyhow!("remote daemon runtime thread is absent"))? + .join() + { + Ok(result) => result, + Err(_) => Err(anyhow!("remote daemon runtime thread panicked")), + } + } + #[cfg(test)] pub(crate) fn from_test_thread(thread: thread::JoinHandle>) -> Self { let (shutdown, _) = watch::channel(false); @@ -383,11 +422,8 @@ impl DaemonRuntimeHandle { { thread::sleep(Duration::from_millis(milliseconds)); } - let _ = self.shutdown.send(true); - let result = match self.thread.take().expect("daemon runtime thread is present").join() { - Ok(result) => result, - Err(_) => Err(anyhow!("remote daemon runtime thread panicked")), - }; + self.request_shutdown(); + let result = self.join_runtime_thread(); #[cfg(debug_assertions)] if let Some(marker) = shutdown_marker { let _ = fs::write(marker, b"complete"); From 4251bed9bf104a76e28658ff81061491db173c02 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 21:54:45 -0700 Subject: [PATCH 332/521] test: expose remaining lifecycle degradation --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 89 +++++++++++++++++++ cmux-tui/crates/cmux-tui/src/main.rs | 28 ++++++ 2 files changed, 117 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 2395834b6549..cef8546cde5e 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -173,6 +173,95 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await firstTicket.wait(timeout: .seconds(1))) } + @Test func twoStuckClosesBoundAdmissionUntilAWorkerRecovers() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + closeTeardownTimeout: .milliseconds(50), + maximumRuntimeSurfaceOwnerCount: 4 + ) + let surfaces = (0..<3).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + defer { for surface in surfaces { surface.deallocate() } } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = DispatchSemaphore(value: 0) + defer { + releaseFrees.signal() + releaseFrees.signal() + freeStarted.continuation.finish() + } + + let firstTicket = try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.firstStuckClose", + surface: surfaces[0], + callbackContext: nil, + freeSurface: { _ in + freeStarted.continuation.yield() + releaseFrees.wait() + } + ) + ) + let secondTicket = try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondStuckClose", + surface: surfaces[1], + callbackContext: nil, + freeSurface: { _ in + freeStarted.continuation.yield() + releaseFrees.wait() + } + ) + ) + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + _ = await freeStartedIterator.next() + + let degradationDeadline = ContinuousClock.now + .seconds(1) + while await !coordinator.debugCloseTeardownDegraded, + ContinuousClock.now < degradationDeadline { + await Task.yield() + } + #expect(await coordinator.debugCloseTeardownDegraded) + #expect(await coordinator.debugPendingTeardownCount == 2) + + let rejectedTicket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.closeAfterBothWorkersStalled", + surface: surfaces[2], + callbackContext: nil, + freeSurface: { _ in } + ) + #expect(rejectedTicket == nil) + + releaseFrees.signal() + releaseFrees.signal() + #expect(await firstTicket.wait(timeout: .seconds(1))) + #expect(await secondTicket.wait(timeout: .seconds(1))) + + let recoveryDeadline = ContinuousClock.now + .seconds(1) + while await coordinator.debugCloseTeardownDegraded, + ContinuousClock.now < recoveryDeadline { + await Task.yield() + } + #expect(await !coordinator.debugCloseTeardownDegraded) + let recoveredTicket = try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.closeAfterWorkerRecovery", + surface: surfaces[2], + callbackContext: nil, + freeSurface: { _ in } + ) + ) + #expect(await recoveredTicket.wait(timeout: .seconds(1))) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index e03e313e8a3e..131a34421708 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -2704,6 +2704,9 @@ mod tests { #[cfg(unix)] #[test] fn server_process_owner_shutdown_obeys_cleanup_deadline() { + use std::io::{BufRead as _, Write as _}; + use std::os::unix::net::UnixStream; + let process_owners = Arc::new(ServerProcessOwnersCleanup::new()); let (release, blocked) = std::sync::mpsc::channel(); let remote_thread = std::thread::spawn(move || { @@ -2715,6 +2718,14 @@ mod tests { process_owners.seal(); let mux = Mux::new("bounded-process-owner-test", SurfaceOptions::default()); + let suffix = + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos(); + let socket_dir = std::env::temp_dir() + .join(format!("cmux-owner-health-{}-{suffix:x}", std::process::id())); + std::fs::create_dir(&socket_dir).unwrap(); + let socket_path = socket_dir.join("server.sock"); + let published = + cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())).unwrap(); let cleanup = Arc::new(ServerShutdownCleanup::new_with_process_owners(mux, process_owners)); cleanup.set_shutdown_attempt_for_test(|| Ok(())); cleanup.set_process_owner_timeout_for_test(std::time::Duration::from_millis(50)); @@ -2728,15 +2739,32 @@ mod tests { let returned_in_time = bounded.is_ok(); let returned_incomplete = matches!(&bounded, Ok(Err(_))); let retained_remote_owner = !cleanup.process_owners.remote_is_finished(); + let mut stream = UnixStream::connect(&socket_path).unwrap(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(1))).unwrap(); + stream.write_all(b"{\"id\":1,\"cmd\":\"identify\"}\n").unwrap(); + stream.flush().unwrap(); + let mut response = String::new(); + BufReader::new(stream).read_line(&mut response).unwrap(); + let identity: serde_json::Value = serde_json::from_str(&response).unwrap(); + let health = &identity["data"]["shutdown_cleanup"]; + let reported_process_owner_degradation = health["pending"].as_u64() == Some(1) + && health["retrying"].as_bool() == Some(false) + && health["degraded"].as_bool() == Some(true); let _ = release.send(()); if returned_in_time { cleanup.prepare_retry(0); cleanup.run_until_complete().unwrap(); } worker.join().unwrap(); + published.cleanup(); + std::fs::remove_dir_all(socket_dir).unwrap(); assert!(returned_in_time, "server process-owner shutdown ignored the cleanup deadline"); assert!(returned_incomplete, "bounded owner cleanup did not request an explicit retry"); assert!(retained_remote_owner, "bounded owner cleanup discarded the live remote runtime"); + assert!( + reported_process_owner_degradation, + "identify hid process-owner cleanup that required an explicit retry" + ); } #[test] From 7ceb92a84aa1d1bcae37a9afba5a1da876778c85 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:14:53 -0700 Subject: [PATCH 333/521] fix: bound remaining lifecycle degradation --- ...minalSurfaceRuntimeTeardownAdmission.swift | 63 ++++++ ...faceRuntimeTeardownCoordinator+Debug.swift | 13 +- ...nalSurfaceRuntimeTeardownCoordinator.swift | 141 ++++++++++-- ...erminalSurfaceRuntimeTeardownRequest.swift | 6 + ...nalSurfaceRuntimeTeardownReservation.swift | 13 ++ .../Surface/TerminalSurface+Debug.swift | 36 ++++ .../TerminalSurface+RuntimeLifecycle.swift | 85 +++++++- .../Surface/TerminalSurface.swift | 40 +++- ...rfaceRuntimeTeardownCoordinatorTests.swift | 201 ++++++++++-------- cmux-tui/crates/cmux-tui-core/src/mux.rs | 18 ++ cmux-tui/crates/cmux-tui/src/main.rs | 11 + 11 files changed, 513 insertions(+), 114 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift index 403ba88a3186..996585a1809a 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -1,4 +1,67 @@ import Foundation +internal import os + +/// Synchronous, bounded admission for native-surface ownership. +/// +/// Creation and deinit cannot suspend while transferring a native pointer, so +/// this ledger uses one short unfair-lock critical section. Native work never +/// runs under the lock. A fully stalled close pool degrades admission until a +/// worker returns, preventing repeated create/close cycles from growing an +/// unbounded retained teardown backlog. +final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { + private struct State { + var reservationIDs: Set = [] + var closeTeardownDegraded = false + } + + private let maximumOwnerCount: Int + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(maximumOwnerCount: Int) { + precondition( + maximumOwnerCount >= TerminalSurfaceRuntimeTeardownCoordinator + .maximumConcurrentCloseTeardownCount, + "native surface ownership must cover every close worker" + ) + self.maximumOwnerCount = maximumOwnerCount + } + + func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { + state.withLock { state in + guard !state.closeTeardownDegraded, + state.reservationIDs.count < maximumOwnerCount else { + return nil + } + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + return reservation + } + } + + func contains(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) -> Bool { + state.withLock { $0.reservationIDs.contains(reservation.id) } + } + + func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { + state.withLock { state in + _ = state.reservationIDs.remove(reservation.id) + } + } + + func setCloseTeardownDegraded(_ degraded: Bool) { + state.withLock { $0.closeTeardownDegraded = degraded } + } + +#if DEBUG + var debugCloseTeardownDegraded: Bool { + state.withLock { $0.closeTeardownDegraded } + } + + var debugOwnerCount: Int { + state.withLock { $0.reservationIDs.count } + } +#endif +} /// Main-actor admission state for one coordinator's bounded native-free slots. @MainActor diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift index 7aef9d9a5349..2b25229eb7cb 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift @@ -10,7 +10,18 @@ extension TerminalSurfaceRuntimeTeardownCoordinator { /// host; suites that create surfaces assert this drained back to their /// baseline after teardown. public var debugPendingTeardownCount: Int { - pendingReasonsById.count + pendingRequestsById.count + } + + /// Test support: whether every close worker has exceeded its deadline and + /// new native-surface ownership is fenced until one worker returns. + public var debugCloseTeardownDegraded: Bool { + runtimeOwnershipAdmission.debugCloseTeardownDegraded + } + + /// Test support: live plus retained native surfaces charged to admission. + public nonisolated var debugRuntimeSurfaceOwnerCount: Int { + runtimeOwnershipAdmission.debugOwnerCount } } #endif diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index ec4847448c11..5a2a905c0b79 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -11,6 +11,10 @@ internal import CMUXDebugLog /// Close/deinit frees use two independently startable utility slots. Each slot /// remains serial, so teardown cannot recursively reuse one worker, while one /// stuck native join leaves the other slot available to drain later closes. +/// Every native surface holds a bounded ownership reservation from creation +/// through completed free. If both close workers time out, new ownership is +/// rejected until a worker recovers, so retained frees cannot grow without +/// bound. /// Each admitted hibernation also owns one independently startable utility slot. /// Deadline observers report, but never block on, stuck frees. The app constructs /// exactly one instance and injects it through @@ -19,31 +23,70 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Maximum native close frees that can run concurrently. public static let maximumConcurrentCloseTeardownCount = 2 + /// Maximum live plus retained native surfaces owned by one coordinator. + public static let maximumRuntimeSurfaceOwnerCount = 4_096 + /// Largest batch that can own independently startable native-free slots. public static let maximumIsolatedHibernationTeardownCount = 2 - private let timeout: Duration = .seconds(5) + private let closeTeardownTimeout: Duration #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in // TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift can report the // pending count; private in release builds. - var pendingReasonsById: [UUID: String] = [:] + var pendingRequestsById: [UUID: (surfaceID: UUID, reason: String)] = [:] #else - private var pendingReasonsById: [UUID: String] = [:] + private var pendingRequestsById: [UUID: (surfaceID: UUID, reason: String)] = [:] #endif private var queuedCloseRequests: [TerminalSurfaceRuntimeTeardownRequest] = [] private var availableCloseExecutionSlots: Set + private var timedOutCloseExecutionSlots: Set = [] private let closeTeardownQueues: [DispatchQueue] private let isolatedHibernationQueues: [DispatchQueue] +#if DEBUG + // Readable at internal scope in DEBUG so the debug-only extension in + // TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift can report + // admission health; private in release builds. + nonisolated let runtimeOwnershipAdmission: + TerminalSurfaceRuntimeOwnershipAdmission +#else + private nonisolated let runtimeOwnershipAdmission: + TerminalSurfaceRuntimeOwnershipAdmission +#endif private nonisolated let isolatedHibernationAdmission = TerminalSurfaceRuntimeTeardownAdmission() /// Creates the process's teardown coordinator. public init() { + closeTeardownTimeout = .seconds(5) + runtimeOwnershipAdmission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: Self.maximumRuntimeSurfaceOwnerCount + ) availableCloseExecutionSlots = Set( 0.. .zero) + self.closeTeardownTimeout = closeTeardownTimeout + runtimeOwnershipAdmission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: maximumRuntimeSurfaceOwnerCount + ) + availableCloseExecutionSlots = Set( + 0.. [DispatchQueue] { + ( 0.. [DispatchQueue] { + ( 0.. TerminalSurfaceRuntimeOwnershipReservation? { + runtimeOwnershipAdmission.reserve() + } + + nonisolated func cancelRuntimeSurfaceOwnership( + _ reservation: TerminalSurfaceRuntimeOwnershipReservation + ) { + runtimeOwnershipAdmission.release(reservation) + } + @MainActor func reserveIsolatedHibernationTeardown() -> TerminalSurfaceRuntimeTeardownReservation? { @@ -97,7 +155,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// main actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. - /// - Returns: A ticket that completes after the native free and userdata releases. + /// - Returns: A ticket that completes after the native free and userdata + /// releases, or `nil` when bounded ownership admission is unavailable. + /// On `nil`, the caller retains the native pointer and callback userdata. @discardableResult public nonisolated func enqueueRuntimeTeardown( id: UUID, @@ -108,7 +168,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) -> TerminalSurfaceRuntimeTeardownTicket { + ) -> TerminalSurfaceRuntimeTeardownTicket? { enqueueRuntimeTeardown( id: id, workspaceId: workspaceId, @@ -146,7 +206,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// actor after the free completes. /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. - /// - Returns: A ticket that completes after the native free and userdata releases. + /// - Returns: A ticket that completes after the native free and userdata + /// releases, or `nil` when bounded ownership admission is degraded or + /// exhausted. On `nil`, the caller retains the native pointer and its + /// callback userdata. @discardableResult nonisolated func enqueueRuntimeTeardown( id: UUID, @@ -156,16 +219,27 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, + runtimeOwnershipReservation: + TerminalSurfaceRuntimeOwnershipReservation? = nil, executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .boundedClose, isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? = nil, freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in ghostty_surface_free(surface) } - ) -> TerminalSurfaceRuntimeTeardownTicket { + ) -> TerminalSurfaceRuntimeTeardownTicket? { + let admittedRuntimeOwnership: TerminalSurfaceRuntimeOwnershipReservation + if let existing = runtimeOwnershipReservation { + guard runtimeOwnershipAdmission.contains(existing) else { return nil } + admittedRuntimeOwnership = existing + } else { + guard let admitted = runtimeOwnershipAdmission.reserve() else { return nil } + admittedRuntimeOwnership = admitted + } let completion = TerminalSurfaceRuntimeTeardownCompletion() let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) let request = TerminalSurfaceRuntimeTeardownRequest( + ticketID: ticket.id, id: id, workspaceId: workspaceId, reason: reason, @@ -173,6 +247,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: byteTeeLease, + runtimeOwnershipReservation: admittedRuntimeOwnership, freeSurface: freeSurface, completion: completion ) @@ -192,7 +267,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? = nil ) async { - pendingReasonsById[request.id] = request.reason + pendingRequestsById[request.ticketID] = ( + surfaceID: request.id, + reason: request.reason + ) switch executionLane { case .isolatedHibernation: if let isolatedHibernationReservation, @@ -206,7 +284,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { // concurrently. This bounds blocked native workers at two without // letting one stuck pane strand another admitted pane. Task { - await self.observeTimeout(id: request.id) + await self.observeTimeout(requestID: request.ticketID) } isolatedHibernationQueues[executionSlot].async { self.freeNativeSurface(request) @@ -215,7 +293,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { isolatedHibernationReservation ) await self.finishFree(request) - await self.complete(id: request.id) + await self.complete(requestID: request.ticketID) } } return @@ -238,13 +316,16 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { availableCloseExecutionSlots.remove(executionSlot) let request = queuedCloseRequests.removeFirst() Task { - await self.observeTimeout(id: request.id) + await self.observeTimeout( + requestID: request.ticketID, + closeExecutionSlot: executionSlot + ) } closeTeardownQueues[executionSlot].async { self.freeNativeSurface(request) Task { await self.finishFree(request) - await self.complete(id: request.id) + await self.complete(requestID: request.ticketID) await self.closeTeardownFinished(executionSlot: executionSlot) } } @@ -252,6 +333,11 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { } private func closeTeardownFinished(executionSlot: Int) { + timedOutCloseExecutionSlots.remove(executionSlot) + if timedOutCloseExecutionSlots.count + < Self.maximumConcurrentCloseTeardownCount { + runtimeOwnershipAdmission.setCloseTeardownDegraded(false) + } availableCloseExecutionSlots.insert(executionSlot) scheduleCloseTeardowns() } @@ -285,6 +371,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { request.byteTeeLease?.release() } } + runtimeOwnershipAdmission.release( + request.runtimeOwnershipReservation + ) await request.completion.finish() #if DEBUG logDebugEvent( @@ -294,22 +383,32 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { #endif } - private func complete(id: UUID) { - pendingReasonsById.removeValue(forKey: id) + private func complete(requestID: UUID) { + pendingRequestsById.removeValue(forKey: requestID) } - private func observeTimeout(id: UUID) async { + private func observeTimeout( + requestID: UUID, + closeExecutionSlot: Int? = nil + ) async { do { // Genuine teardown deadline: report a stuck native free without blocking close. - try await Task.sleep(for: timeout) + try await Task.sleep(for: closeTeardownTimeout) } catch { return } - guard let reason = pendingReasonsById[id] else { return } + guard let pending = pendingRequestsById[requestID] else { return } + if let closeExecutionSlot { + timedOutCloseExecutionSlots.insert(closeExecutionSlot) + if timedOutCloseExecutionSlots.count + == Self.maximumConcurrentCloseTeardownCount { + runtimeOwnershipAdmission.setCloseTeardownDegraded(true) + } + } #if DEBUG logDebugEvent( - "surface.lifecycle.nativeFree.timeout surface=\(id.uuidString.prefix(5)) " + - "reason=\(reason)" + "surface.lifecycle.nativeFree.timeout surface=\(pending.surfaceID.uuidString.prefix(5)) " + + "reason=\(pending.reason)" ) #endif } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift index 18d82e8f78b1..576421adfbce 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownRequest.swift @@ -17,6 +17,7 @@ public import CmuxTerminalCore /// tee callback and the io thread that fires the MANUAL-mode `io_write_cb`), /// so a release ordered after the free can never race an in-flight callback. struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { + let ticketID: UUID let id: UUID let workspaceId: UUID let reason: String @@ -24,6 +25,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { let callbackContext: Unmanaged? let manualIOContext: Unmanaged? let byteTeeLease: (any TerminalByteTeeLease)? + let runtimeOwnershipReservation: TerminalSurfaceRuntimeOwnershipReservation let freeSurface: @Sendable (ghostty_surface_t) -> Void let completion: TerminalSurfaceRuntimeTeardownCompletion #if DEBUG @@ -32,6 +34,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { #endif init( + ticketID: UUID, id: UUID, workspaceId: UUID, reason: String, @@ -39,9 +42,11 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { callbackContext: Unmanaged?, manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, + runtimeOwnershipReservation: TerminalSurfaceRuntimeOwnershipReservation, freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void, completion: TerminalSurfaceRuntimeTeardownCompletion ) { + self.ticketID = ticketID self.id = id self.workspaceId = workspaceId self.reason = reason @@ -49,6 +54,7 @@ struct TerminalSurfaceRuntimeTeardownRequest: @unchecked Sendable { self.callbackContext = callbackContext self.manualIOContext = manualIOContext self.byteTeeLease = byteTeeLease + self.runtimeOwnershipReservation = runtimeOwnershipReservation self.freeSurface = freeSurface self.completion = completion #if DEBUG diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift index c79fd74320ee..5b4059a18999 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownReservation.swift @@ -8,3 +8,16 @@ struct TerminalSurfaceRuntimeTeardownReservation: Equatable, Sendable { self.id = id } } + +/// Ownership of one live or retained native Ghostty surface. +/// +/// The reservation begins before `ghostty_surface_new` and ends only after +/// `ghostty_surface_free` returns. This bounds the combined live-surface and +/// teardown backlog even when every native-free worker is stalled. +struct TerminalSurfaceRuntimeOwnershipReservation: Equatable, Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift index 98ecc8a1e5e6..ad5548b77c64 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift @@ -181,9 +181,17 @@ extension TerminalSurface { public func releaseSurfaceForTesting() { let callbackContext = surfaceCallbackContext surfaceCallbackContext = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + self.runtimeSurfaceOwnershipReservation = nil guard let surfaceToFree = surface else { callbackContext?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return } @@ -191,6 +199,11 @@ extension TerminalSurface { surface = nil ghostty_surface_free(surfaceToFree) callbackContext?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } } /// Test-only helper to simulate a stale Swift wrapper whose native surface @@ -201,9 +214,17 @@ extension TerminalSurface { let callbackContext = surfaceCallbackContext surfaceCallbackContext = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + self.runtimeSurfaceOwnershipReservation = nil guard let surfaceToFree = surface else { callbackContext?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return } @@ -211,11 +232,26 @@ extension TerminalSurface { ghostty_surface_free(surfaceToFree) runtimeSurfaceFreedOutOfBandForTesting = true callbackContext?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } } /// Test-only helper to install a runtime surface pointer directly. @MainActor public func installRuntimeSurfaceForTesting(_ runtimeSurface: ghostty_surface_t) { + if runtimeSurfaceOwnershipReservation == nil { + guard let runtimeOwnershipReservation = + runtimeTeardown.reserveRuntimeSurfaceOwnership() else { + preconditionFailure( + "Test runtime surface requires bounded runtime ownership" + ) + } + self.runtimeSurfaceOwnershipReservation = + runtimeOwnershipReservation + } let callbackContext: Unmanaged< GhosttySurfaceCallbackContext > diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 9662ffccf100..e0840b200f37 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -134,6 +134,9 @@ extension TerminalSurface { surfaceCallbackContext = nil let teeLease = mobileByteTeeLease mobileByteTeeLease = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + self.runtimeSurfaceOwnershipReservation = nil registry.unregisterRuntimeSurface(surface, ownerId: id) self.surface = nil activePortalHostLease = nil @@ -150,6 +153,11 @@ extension TerminalSurface { #endif callbackContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return nil } return surface @@ -247,6 +255,9 @@ extension TerminalSurface { self.manualIOContext = nil let teeLease = mobileByteTeeLease mobileByteTeeLease = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + self.runtimeSurfaceOwnershipReservation = nil byteTee.dropSurface(surfaceID: id) let surfaceToFree = surface @@ -259,6 +270,11 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } if closeRuntimeTeardownTicket == nil { closeRuntimeTeardownTicket = agentHibernationRuntimeTeardownTicket @@ -272,10 +288,21 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return closeRuntimeTeardownTicket } #endif + guard let runtimeOwnershipReservation else { + preconditionFailure( + "A native terminal surface requires bounded runtime ownership" + ) + } + #if DEBUG if let freeSurface = Self.runtimeSurfaceFreeOverrideForTesting { // Transport manualIOContext and teeLease through the request too: @@ -289,8 +316,14 @@ extension TerminalSurface { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation, freeSurface: freeSurface ) + guard let ticket else { + preconditionFailure( + "Transferred runtime ownership must remain admitted" + ) + } closeRuntimeTeardownTicket = ticket return ticket } @@ -303,8 +336,14 @@ extension TerminalSurface { surface: surfaceToFree, callbackContext: callbackContext, manualIOContext: manualIOContext, - byteTeeLease: teeLease + byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation ) + guard let ticket else { + preconditionFailure( + "Transferred runtime ownership must remain admitted" + ) + } closeRuntimeTeardownTicket = ticket return ticket } @@ -353,6 +392,9 @@ extension TerminalSurface { self.manualIOContext = nil let teeLease = mobileByteTeeLease mobileByteTeeLease = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + self.runtimeSurfaceOwnershipReservation = nil byteTee.dropSurface(surfaceID: id) let surfaceToFree = surface @@ -375,9 +417,23 @@ extension TerminalSurface { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return true } + guard let runtimeOwnershipReservation else { + runtimeTeardown.cancelIsolatedHibernationTeardown( + teardownReservation + ) + preconditionFailure( + "A native terminal surface requires bounded runtime ownership" + ) + } + #if DEBUG logDebugEvent( "surface.lifecycle.hibernate surface=\(id.uuidString.prefix(5)) " + @@ -398,10 +454,15 @@ extension TerminalSurface { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation, executionLane: .isolatedHibernation, isolatedHibernationReservation: teardownReservation, freeSurface: freeSurface ) + precondition( + agentHibernationRuntimeTeardownTicket != nil, + "Transferred runtime ownership must remain admitted" + ) return true } #endif @@ -414,9 +475,14 @@ extension TerminalSurface { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation, executionLane: .isolatedHibernation, isolatedHibernationReservation: teardownReservation ) + precondition( + agentHibernationRuntimeTeardownTicket != nil, + "Transferred runtime ownership must remain admitted" + ) return true } @@ -643,6 +709,7 @@ extension TerminalSurface { #endif return } + guard surface == nil else { return } if deferRuntimeSurfaceCreationForConfigurationReload( view: view, source: source @@ -677,6 +744,17 @@ extension TerminalSurface { return } + guard let runtimeOwnershipReservation = + runtimeTeardown.reserveRuntimeSurfaceOwnership() else { +#if DEBUG + logDebugEvent( + "ghostty.surface.create.failed reason=teardownAdmissionDegraded " + + "surface=\(id.uuidString)" + ) +#endif + return + } + let scaleFactors = scaleFactors(for: view) let runtimeSurfaceCreation = createNativeRuntimeSurface( @@ -689,6 +767,9 @@ extension TerminalSurface { let runtimeInitialInput = runtimeSurfaceCreation.runtimeInitialInput if surface == nil { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) surfaceCallbackContext?.release() surfaceCallbackContext = nil manualIOContext?.release() @@ -712,6 +793,8 @@ extension TerminalSurface { #endif return } + self.runtimeSurfaceOwnershipReservation = + runtimeOwnershipReservation guard let createdSurface = surface else { return } guard let surfaceCallbackContext else { preconditionFailure( diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index 9a8aed10b317..dadd6de09e44 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -280,6 +280,11 @@ public final class TerminalSurface: Identifiable, ObservableObject { (any TerminalSurfaceNativeViewing)? var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false + /// Bounded ownership of the live native surface. Transferred to the + /// teardown coordinator with the pointer and released only after native + /// free returns. + var runtimeSurfaceOwnershipReservation: + TerminalSurfaceRuntimeOwnershipReservation? var closeRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? var agentHibernationRuntimeTeardownTicket: TerminalSurfaceRuntimeTeardownTicket? var agentHibernationRuntimeTeardownReservation: @@ -643,6 +648,9 @@ public final class TerminalSurface: Identifiable, ObservableObject { // mobileByteTeeLease, so teeLease is nil here and ?.release() no-ops. let teeLease = mobileByteTeeLease mobileByteTeeLease = nil + let runtimeOwnershipReservation = + runtimeSurfaceOwnershipReservation + runtimeSurfaceOwnershipReservation = nil // `dropSurface` is @MainActor but `deinit` is nonisolated, so hop to the // main actor with the surface id captured by value (no self capture). // Dropping by id only clears the registry/replay state; releasing @@ -671,6 +679,11 @@ public final class TerminalSurface: Identifiable, ObservableObject { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return } @@ -680,10 +693,21 @@ public final class TerminalSurface: Identifiable, ObservableObject { callbackContext?.release() manualIOContext?.release() teeLease?.release() + if let runtimeOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + } return } #endif + guard let runtimeOwnershipReservation else { + preconditionFailure( + "A native terminal surface requires bounded runtime ownership" + ) + } + #if DEBUG logDebugEvent( "surface.lifecycle.deinit.begin surface=\(id.uuidString.prefix(5)) " + @@ -699,7 +723,7 @@ public final class TerminalSurface: Identifiable, ObservableObject { // the coordinator's deferred free runs. #if DEBUG if let freeSurface = Self.runtimeSurfaceFreeOverrideForTesting { - runtimeTeardown.enqueueRuntimeTeardown( + let ticket = runtimeTeardown.enqueueRuntimeTeardown( id: id, workspaceId: tabId, reason: "deinit", @@ -707,19 +731,29 @@ public final class TerminalSurface: Identifiable, ObservableObject { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation, freeSurface: freeSurface ) + precondition( + ticket != nil, + "Transferred runtime ownership must remain admitted" + ) return } #endif - runtimeTeardown.enqueueRuntimeTeardown( + let ticket = runtimeTeardown.enqueueRuntimeTeardown( id: id, workspaceId: tabId, reason: "deinit", surface: surfaceToFree, callbackContext: callbackContext, manualIOContext: manualIOContext, - byteTeeLease: teeLease + byteTeeLease: teeLease, + runtimeOwnershipReservation: runtimeOwnershipReservation + ) + precondition( + ticket != nil, + "Transferred runtime ownership must remain admitted" ) } } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index cef8546cde5e..0cdfe3001281 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -59,6 +59,12 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } } +private func requireTeardownTicket( + _ ticket: TerminalSurfaceRuntimeTeardownTicket? +) throws -> TerminalSurfaceRuntimeTeardownTicket { + try #require(ticket) +} + @Suite struct TerminalSurfaceRuntimeTeardownCoordinatorTests { @Test func ticketDistinguishesDeadlineFromEventualCompletion() async { let completion = TerminalSurfaceRuntimeTeardownCompletion() @@ -71,22 +77,24 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await ticket.wait(timeout: nil)) } - @Test func enqueuedTeardownInvokesInjectedFreeWithTheSamePointer() async { + @Test func enqueuedTeardownInvokesInjectedFreeWithTheSamePointer() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = FreedSurfaceRecorder() let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { surface.deallocate() } - let ticket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test", - surface: surface, - callbackContext: nil, - freeSurface: { pointer in - let bits = UInt(bitPattern: pointer) - Task { await recorder.record(bits) } - } + let ticket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test", + surface: surface, + callbackContext: nil, + freeSurface: { pointer in + let bits = UInt(bitPattern: pointer) + Task { await recorder.record(bits) } + } + ) ) await recorder.waitForFreeCount(1) @@ -120,7 +128,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec #expect(await Set(recorder.freed) == Set(surfaces.map { UInt(bitPattern: $0) })) } - @Test func oneStuckCloseDoesNotStrandLaterCloses() async { + @Test func oneStuckCloseDoesNotStrandLaterCloses() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let surfaces = (0..<3).map { _ in UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) @@ -133,35 +141,41 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec firstFreeStarted.continuation.finish() } - let firstTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.stuckClose", - surface: surfaces[0], - callbackContext: nil, - freeSurface: { _ in - firstFreeStarted.continuation.yield() - releaseFirstFree.wait() - } + let firstTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.stuckClose", + surface: surfaces[0], + callbackContext: nil, + freeSurface: { _ in + firstFreeStarted.continuation.yield() + releaseFirstFree.wait() + } + ) ) var firstFreeIterator = firstFreeStarted.stream.makeAsyncIterator() _ = await firstFreeIterator.next() - let secondTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.closeAfterStuckClose", - surface: surfaces[1], - callbackContext: nil, - freeSurface: { _ in } + let secondTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.closeAfterStuckClose", + surface: surfaces[1], + callbackContext: nil, + freeSurface: { _ in } + ) ) - let thirdTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.secondCloseAfterStuckClose", - surface: surfaces[2], - callbackContext: nil, - freeSurface: { _ in } + let thirdTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondCloseAfterStuckClose", + surface: surfaces[2], + callbackContext: nil, + freeSurface: { _ in } + ) ) let secondCompleted = await secondTicket.wait(timeout: .seconds(1)) @@ -190,7 +204,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec freeStarted.continuation.finish() } - let firstTicket = try #require( + let firstTicket = try requireTeardownTicket( coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), @@ -203,7 +217,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } ) ) - let secondTicket = try #require( + let secondTicket = try requireTeardownTicket( coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), @@ -227,6 +241,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } #expect(await coordinator.debugCloseTeardownDegraded) #expect(await coordinator.debugPendingTeardownCount == 2) + #expect(coordinator.debugRuntimeSurfaceOwnerCount == 2) let rejectedTicket = coordinator.enqueueRuntimeTeardown( id: UUID(), @@ -237,11 +252,13 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec freeSurface: { _ in } ) #expect(rejectedTicket == nil) + #expect(coordinator.debugRuntimeSurfaceOwnerCount == 2) releaseFrees.signal() releaseFrees.signal() #expect(await firstTicket.wait(timeout: .seconds(1))) #expect(await secondTicket.wait(timeout: .seconds(1))) + #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) let recoveryDeadline = ContinuousClock.now + .seconds(1) while await coordinator.debugCloseTeardownDegraded, @@ -249,7 +266,7 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec await Task.yield() } #expect(await !coordinator.debugCloseTeardownDegraded) - let recoveredTicket = try #require( + let recoveredTicket = try requireTeardownTicket( coordinator.enqueueRuntimeTeardown( id: UUID(), workspaceId: UUID(), @@ -287,20 +304,22 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec let isolatedReservation = try #require( await coordinator.reserveIsolatedHibernationTeardown() ) - let isolatedTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.isolatedHibernation", - surface: isolatedSurface, - callbackContext: nil, - manualIOContext: nil, - byteTeeLease: nil, - executionLane: .isolatedHibernation, - isolatedHibernationReservation: isolatedReservation, - freeSurface: { _ in - isolatedFreeStarted.continuation.yield() - _ = releaseIsolatedFree.wait(timeout: .distantFuture) - } + let isolatedTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.isolatedHibernation", + surface: isolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: isolatedReservation, + freeSurface: { _ in + isolatedFreeStarted.continuation.yield() + _ = releaseIsolatedFree.wait(timeout: .distantFuture) + } + ) ) var isolatedFreeIterator = isolatedFreeStarted.stream.makeAsyncIterator() _ = await isolatedFreeIterator.next() @@ -309,29 +328,33 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec await coordinator.reserveIsolatedHibernationTeardown() ) #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) - let secondIsolatedTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.secondIsolatedHibernation", - surface: queuedIsolatedSurface, - callbackContext: nil, - manualIOContext: nil, - byteTeeLease: nil, - executionLane: .isolatedHibernation, - isolatedHibernationReservation: secondReservation, - freeSurface: { _ in - secondIsolatedFreeCount.withLock { $0 += 1 } - } + let secondIsolatedTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.secondIsolatedHibernation", + surface: queuedIsolatedSurface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: secondReservation, + freeSurface: { _ in + secondIsolatedFreeCount.withLock { $0 += 1 } + } + ) ) - let closeTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.boundedClose", - surface: closeSurface, - callbackContext: nil, - freeSurface: { _ in - closeFreeCount.withLock { $0 += 1 } - } + let closeTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.boundedClose", + surface: closeSurface, + callbackContext: nil, + freeSurface: { _ in + closeFreeCount.withLock { $0 += 1 } + } + ) ) #expect(await closeTicket.wait(timeout: .seconds(1))) @@ -363,19 +386,21 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec ) await coordinator.cancelIsolatedHibernationTeardown(staleReservation) - let ticket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.staleIsolatedReservation", - surface: surface, - callbackContext: nil, - manualIOContext: nil, - byteTeeLease: nil, - executionLane: .isolatedHibernation, - isolatedHibernationReservation: staleReservation, - freeSurface: { _ in - freeCount.withLock { $0 += 1 } - } + let ticket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.staleIsolatedReservation", + surface: surface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + executionLane: .isolatedHibernation, + isolatedHibernationReservation: staleReservation, + freeSurface: { _ in + freeCount.withLock { $0 += 1 } + } + ) ) #expect(await ticket.wait(timeout: .seconds(1))) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5c845ecb38ac..5da1d0234504 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8518,6 +8518,24 @@ impl Mux { watch } + /// Publish an outer server cleanup attempt through the same health state + /// used by mux-owned shutdown work. Server process owners live outside the + /// mux, but clients must observe one authoritative cleanup lifecycle. + pub fn schedule_server_shutdown_cleanup(&self) { + self.shutdown_cleanup_lifecycle.schedule(); + } + + /// Clear the shared cleanup health after every mux and outer server owner + /// has completed shutdown. + pub fn complete_server_shutdown_cleanup(&self) { + self.shutdown_cleanup_lifecycle.complete(); + } + + /// Publish an outer server cleanup failure that requires explicit retry. + pub fn degrade_server_shutdown_cleanup(&self) { + self.shutdown_cleanup_lifecycle.degrade(); + } + pub(crate) fn shutdown_cleanup_health(&self) -> ShutdownCleanupHealth { let owner_pending = self.shutdown_owners.len(); #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 131a34421708..8fbac3ff0530 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1631,6 +1631,8 @@ impl ServerShutdownCleanup { }; drop(state); + self.mux.schedule_server_shutdown_cleanup(); + // Runtime owners and mux resources share one absolute deadline. If an // owner is still live, retain it and the published socket until an // explicit retry instead of starting a second cleanup budget. @@ -1682,6 +1684,15 @@ impl ServerShutdownCleanup { } } }; + match &outcome { + ServerShutdownCleanupOutcome::Complete { .. } => { + self.mux.complete_server_shutdown_cleanup(); + } + ServerShutdownCleanupOutcome::Degraded { .. } + | ServerShutdownCleanupOutcome::Abandoned => { + self.mux.degrade_server_shutdown_cleanup(); + } + } let next_state = match &outcome { ServerShutdownCleanupOutcome::Complete { error } => { ServerShutdownCleanupState::Complete { error: error.clone() } From 04bf0113375bcca8aa21081940116e1ef06f29b6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:42:11 -0700 Subject: [PATCH 334/521] test(tui): preserve unreadable host ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 89 ++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5da1d0234504..2cfa89694b75 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26265,6 +26265,95 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[cfg(unix)] + #[test] + fn restart_rejects_unreadable_canonical_record_without_exiting_live_terminal() { + use std::os::unix::fs::PermissionsExt as _; + + const TERMINAL: &str = "00000000000040008000000000000021"; + const INCARNATION: &str = "10000000000040008000000000000021"; + const SESSION: &str = "recover-unreadable-terminal"; + let root = std::env::temp_dir().join(format!( + "cmux-mux-terminal-unreadable-{}", + crate::workspace_registry::new_uuid_v4() + )); + { + let mut registry = WorkspaceRegistry::open(&root, SESSION).unwrap(); + registry + .commit( + &WorkspaceMutation::new("workspace", "test").unwrap(), + &serde_json::json!({"op":"create-workspace"}), + None, + Some(0), + "workspace-added", + "workspace-one", + &[RegistryWorkspace { + id: 1, + public_id: WorkspacePublicId::random().unwrap(), + key: "workspace-one".into(), + name: "One".into(), + group_key: SESSION.into(), + }], + &serde_json::json!({"workspace":1,"key":"workspace-one"}), + ) + .unwrap(); + let mut terminal = RegistryTerminal { + terminal_id: TERMINAL.into(), + workspace_key: "workspace-one".into(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({"command_present":true}), + exit: None, + }; + commit_terminal_transition( + &mut registry, + "terminal-reserved", + "reserve-unreadable-terminal", + &terminal, + ) + .unwrap(); + terminal.incarnation = Some(INCARNATION.into()); + terminal.lifecycle = TerminalLifecycle::Running; + commit_terminal_transition( + &mut registry, + "terminal-running", + "run-unreadable-terminal", + &terminal, + ) + .unwrap(); + } + + let host_root = crate::terminal_host_runtime::terminal_host_root(&root, SESSION); + std::fs::create_dir_all(&host_root).unwrap(); + let record_path = host_root.join(format!("{TERMINAL}.json")); + std::fs::write(&record_path, b"{").unwrap(); + let mut permissions = std::fs::metadata(&record_path).unwrap().permissions(); + permissions.set_mode(0o600); + std::fs::set_permissions(&record_path, permissions).unwrap(); + let options = + SurfaceOptions { terminal_host_root: Some(host_root), ..SurfaceOptions::default() }; + + let startup_rejected = match Mux::open_persistent(SESSION, options, &root) { + Ok(mux) => { + drop(mux); + false + } + Err(_) => true, + }; + let registry = WorkspaceRegistry::open(&root, SESSION).unwrap(); + let terminal = registry.terminal_record(TERMINAL).unwrap().unwrap(); + drop(registry); + std::fs::remove_dir_all(root).unwrap(); + + assert!( + startup_rejected, + "restart treated an unreadable canonical record as a missing host" + ); + assert_eq!(terminal.lifecycle, TerminalLifecycle::Running); + assert_eq!(terminal.incarnation.as_deref(), Some(INCARNATION)); + assert_eq!(terminal.exit, None); + } + #[cfg(unix)] #[test] fn dead_public_terminal_adoption_wakes_wait_exit() { From db64c81eff38b1d06ade4752ba9f242d09445953 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 22:51:56 -0700 Subject: [PATCH 335/521] fix(tui): fail closed on canonical host records --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2cfa89694b75..8c84159721ef 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3270,6 +3270,28 @@ impl Mux { self.materialize_exited_terminal(&terminal.terminal_id, &options)?; continue; } + if let Some(root) = options.terminal_host_root.as_deref() { + // The directory scan deliberately tolerates malformed debris. + // Before converting a durable live row into record absence, + // probe its exact canonical path so unreadable ownership or a + // record published during the scan fails startup without + // changing the terminal lifecycle. + let exact_record = crate::terminal_host_runtime::load_terminal_host_record( + root, + &terminal.terminal_id, + ) + .with_context(|| { + format!( + "load exact terminal-host record for durable terminal {}", + terminal.terminal_id + ) + })?; + anyhow::ensure!( + exact_record.is_none(), + "terminal-host record appeared after adoption discovery for {}", + terminal.terminal_id + ); + } self.mark_terminal_exited_and_materialize( &terminal.terminal_id, "terminal-record-missing", From 19e63b7a64e5f75420c70b6759b164fd4bfd25f8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:47:21 -0700 Subject: [PATCH 336/521] test(tui): expose remaining ownership gaps --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 103 ++++++++++++++++++ .../crates/cmux-tui/src/server_lifecycle.rs | 38 +++++++ 2 files changed, 141 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8c84159721ef..9e16f4117193 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24810,6 +24810,21 @@ mod tests { ); } + #[test] + fn active_synchronous_cleanup_reports_staged_owners_as_retrying() { + let mux = test_mux(); + let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); + let owned = mux.surface(surface.id).unwrap(); + + mux.schedule_server_shutdown_cleanup(); + assert!(mux.shutdown_owners.stage_surface(&owned).is_some()); + let health = mux.shutdown_cleanup_health(); + mux.complete_server_shutdown_cleanup(); + assert!(mux.close_surface(surface.id).unwrap()); + + assert_eq!(health, ShutdownCleanupHealth { pending: 1, retrying: true, degraded: false }); + } + #[cfg(unix)] #[test] fn server_shutdown_rejects_malformed_host_records_before_removing_topology() { @@ -26287,6 +26302,94 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[cfg(unix)] + #[test] + fn restart_waits_for_inflight_terminal_host_publication_before_marking_missing() { + const TERMINAL: &str = "00000000000040008000000000000041"; + const SESSION: &str = "recover-publishing-terminal"; + let root = std::env::temp_dir().join(format!( + "cmux-mux-terminal-publication-{}", + crate::workspace_registry::new_uuid_v4() + )); + { + let mut registry = WorkspaceRegistry::open(&root, SESSION).unwrap(); + registry + .commit( + &WorkspaceMutation::new("workspace", "test").unwrap(), + &serde_json::json!({"op":"create-workspace"}), + None, + Some(0), + "workspace-added", + "workspace-one", + &[RegistryWorkspace { + id: 1, + public_id: WorkspacePublicId::random().unwrap(), + key: "workspace-one".into(), + name: "One".into(), + group_key: SESSION.into(), + }], + &serde_json::json!({"workspace":1,"key":"workspace-one"}), + ) + .unwrap(); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + "reserve-publishing-terminal", + &RegistryTerminal { + terminal_id: TERMINAL.into(), + workspace_key: "workspace-one".into(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({"command_present":true}), + exit: None, + }, + ) + .unwrap(); + } + + let host_root = crate::terminal_host_runtime::terminal_host_root(&root, SESSION); + std::fs::create_dir_all(&host_root).unwrap(); + let record_path = host_root.join(format!("{TERMINAL}.json")); + let publication = crate::server::SocketPublicationGuard::acquire( + &record_path, + Instant::now() + Duration::from_secs(1), + ) + .unwrap(); + let options = + SurfaceOptions { terminal_host_root: Some(host_root), ..SurfaceOptions::default() }; + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let open_root = root.clone(); + let worker = std::thread::spawn(move || { + result_tx.send(Mux::open_persistent(SESSION, options, &open_root)).unwrap(); + }); + + let early = result_rx.recv_timeout(Duration::from_millis(250)); + let waited_for_publication = + matches!(&early, Err(std::sync::mpsc::RecvTimeoutError::Timeout)); + drop(publication); + let mux = match early { + Ok(result) => result.unwrap(), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => result_rx + .recv_timeout(Duration::from_secs(2)) + .expect("restart did not resume after publication ownership ended") + .unwrap(), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + panic!("restart worker stopped before reporting its result") + } + }; + worker.join().unwrap(); + let terminal = mux.resolve_terminal(TERMINAL).unwrap().unwrap().terminal; + mux.shutdown().unwrap(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + + assert!( + waited_for_publication, + "restart finalized terminal-host absence while publication ownership was live" + ); + assert_eq!(terminal.lifecycle, TerminalLifecycle::Exited); + } + #[cfg(unix)] #[test] fn restart_rejects_unreadable_canonical_record_without_exiting_live_terminal() { diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 120f2ee06b18..a698a3ef0722 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -1896,6 +1896,44 @@ mod tests { assert!(message.contains("Stopping exits pane processes.")); } + #[test] + fn legacy_identity_may_omit_shutdown_cleanup_as_a_whole() { + let identity = ServerIdentity::from_protocol_data(&json!({ + "app": "cmux-tui", + "pid": 42, + "protocol": PROTOCOL_VERSION, + })) + .unwrap(); + + assert_eq!(identity.shutdown_cleanup, ShutdownCleanupStatus::default()); + } + + #[test] + fn present_shutdown_cleanup_requires_every_typed_member() { + let malformed = [ + json!(null), + json!({}), + json!({"pending": 1, "retrying": true}), + json!({"pending": "1", "retrying": true, "degraded": false}), + json!({"pending": 1, "retrying": "true", "degraded": false}), + json!({"pending": 1, "retrying": true, "degraded": 0}), + ]; + + for shutdown_cleanup in malformed { + let identity = json!({ + "app": "cmux-tui", + "pid": 42, + "protocol": PROTOCOL_VERSION, + "shutdown_cleanup": shutdown_cleanup, + }); + assert!( + ServerIdentity::from_protocol_data(&identity).is_err(), + "accepted malformed shutdown cleanup: {}", + identity["shutdown_cleanup"] + ); + } + } + #[cfg(unix)] #[test] fn shell_quote_preserves_non_utf8_socket_paths() { From a31f84d6caeed50b63484e5338f6534207afb705 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:39:45 -0700 Subject: [PATCH 337/521] fix(tui): fence host publication ownership --- cmux-tui/crates/cmux-tui-core/src/lib.rs | 2 + cmux-tui/crates/cmux-tui-core/src/mux.rs | 39 ++- .../crates/cmux-tui-core/src/publication.rs | 258 ++++++++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 277 +----------------- cmux-tui/crates/cmux-tui-core/src/surface.rs | 21 +- .../src/terminal_host_runtime.rs | 93 ++++-- cmux-tui/crates/cmux-tui/src/main.rs | 2 +- .../crates/cmux-tui/src/server_lifecycle.rs | 40 +-- 8 files changed, 411 insertions(+), 321 deletions(-) create mode 100644 cmux-tui/crates/cmux-tui-core/src/publication.rs diff --git a/cmux-tui/crates/cmux-tui-core/src/lib.rs b/cmux-tui/crates/cmux-tui-core/src/lib.rs index 4b7722427de7..700d21963722 100644 --- a/cmux-tui/crates/cmux-tui-core/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-core/src/lib.rs @@ -16,6 +16,7 @@ mod pairing; #[cfg(unix)] pub mod process_session; pub mod provider_management; +mod publication; pub mod resource; mod resource_api; mod resource_mutation; @@ -56,6 +57,7 @@ pub use mux::{ WorkspacePlacement, ZoomMode, ZoomState, }; pub use pairing::{PairingChallenge, PairingDecision, PairingError}; +pub use publication::PublicationGuard; pub use resource_api::{ResourceMachineRequest, ResourceMachineService}; pub use resource_selector::{ResolvedResourcePath, ResourceSelectors, ResourceTarget}; pub use short_id::assign_short_ids; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 9e16f4117193..d3afd396cd09 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -3273,9 +3273,14 @@ impl Mux { if let Some(root) = options.terminal_host_root.as_deref() { // The directory scan deliberately tolerates malformed debris. // Before converting a durable live row into record absence, - // probe its exact canonical path so unreadable ownership or a - // record published during the scan fails startup without - // changing the terminal lifecycle. + // exclude its launcher through the same transferable guard + // held until the child publishes. The exact canonical probe + // then either observes that publication, rejects unreadable + // ownership, or proves no child can publish after absence. + let _publication = crate::terminal_host_runtime::acquire_terminal_host_publication( + root, + &terminal.terminal_id, + )?; let exact_record = crate::terminal_host_runtime::load_terminal_host_record( root, &terminal.terminal_id, @@ -6178,7 +6183,7 @@ impl Mux { #[cfg(all(not(test), unix))] let use_host_runtime = true; #[cfg(unix)] - if let (Some(_), Some(workspace_key), true) = + if let (Some(host_root), Some(workspace_key), true) = (opts.terminal_host_root.as_ref(), workspace_key, use_host_runtime) { let terminal_id = reservation @@ -6187,6 +6192,14 @@ impl Mux { .map(Ok) .unwrap_or_else(TerminalId::random)?; let terminal_hex = terminal_id.to_hex(); + // Acquire before the durable Launching commit. A replacement can + // therefore finalize record absence only after this parent dies + // before spawn or the inherited child finishes publication. + let publication_guard = + crate::terminal_host_runtime::acquire_terminal_host_publication( + host_root, + &terminal_hex, + )?; let launch_spec = terminal_launch_spec(&opts); let terminal = RegistryTerminal { terminal_id: terminal_hex.clone(), @@ -6234,7 +6247,8 @@ impl Mux { id, opts, Arc::downgrade(self), - Some(terminal_id), + terminal_id, + publication_guard, cell_pixels, ) { Ok(surface) => surface, @@ -8569,8 +8583,11 @@ impl Mux { let (unpublished_pending, unpublished_degraded) = (0, false); let lifecycle = self.shutdown_cleanup_lifecycle.snapshot(); let state = self.shutdown_owner_reconciler.state.lock().unwrap(); - let owner_retrying = owner_pending != 0 && state.worker_started && !state.degraded; - let owner_degraded = owner_pending != 0 && (state.degraded || !state.worker_started); + let synchronous_retry = lifecycle.pending && lifecycle.retrying && !lifecycle.degraded; + let owner_retrying = + owner_pending != 0 && (synchronous_retry || (state.worker_started && !state.degraded)); + let owner_degraded = + owner_pending != 0 && !synchronous_retry && (state.degraded || !state.worker_started); let degraded = lifecycle.degraded || owner_degraded || unpublished_degraded; ShutdownCleanupHealth { pending: owner_pending @@ -26350,11 +26367,9 @@ mod tests { let host_root = crate::terminal_host_runtime::terminal_host_root(&root, SESSION); std::fs::create_dir_all(&host_root).unwrap(); let record_path = host_root.join(format!("{TERMINAL}.json")); - let publication = crate::server::SocketPublicationGuard::acquire( - &record_path, - Instant::now() + Duration::from_secs(1), - ) - .unwrap(); + let publication = + crate::PublicationGuard::acquire(&record_path, Instant::now() + Duration::from_secs(1)) + .unwrap(); let options = SurfaceOptions { terminal_host_root: Some(host_root), ..SurfaceOptions::default() }; let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); diff --git a/cmux-tui/crates/cmux-tui-core/src/publication.rs b/cmux-tui/crates/cmux-tui-core/src/publication.rs new file mode 100644 index 000000000000..5b003f017e0b --- /dev/null +++ b/cmux-tui/crates/cmux-tui-core/src/publication.rs @@ -0,0 +1,258 @@ +//! Cross-process ownership for publishing one durable filesystem identity. + +#[cfg(unix)] +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +#[cfg(unix)] +use std::sync::{Condvar, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use fs4::FileExt; + +pub(crate) fn publication_lock_path(path: &Path) -> std::io::Result { + let parent = path.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "publication target has no parent directory", + ) + })?; + let file_name = path.file_name().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "publication target has no file name") + })?; + let mut lock_name = std::ffi::OsString::from("."); + lock_name.push(file_name); + lock_name.push(".publish.lock"); + Ok(parent.join(lock_name)) +} + +#[cfg(unix)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct PublicationLockIdentity { + device: u64, + inode: u64, +} + +#[cfg(unix)] +static PUBLICATION_LOCAL_STATE: OnceLock<(Mutex>, Condvar)> = + OnceLock::new(); + +#[cfg(unix)] +struct PublicationLocalGuard { + identity: PublicationLockIdentity, +} + +#[cfg(unix)] +impl PublicationLocalGuard { + fn acquire( + identity: PublicationLockIdentity, + path: &Path, + deadline: Instant, + ) -> std::io::Result { + let (held, changed) = PUBLICATION_LOCAL_STATE.get_or_init(Default::default); + let mut held = held.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + loop { + if held.insert(identity) { + return Ok(Self { identity }); + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .filter(|remaining| !remaining.is_zero()) + .ok_or_else(|| publication_timeout(path))?; + held = match changed.wait_timeout(held, remaining) { + Ok((held, _)) => held, + Err(error) => error.into_inner().0, + }; + } + } +} + +#[cfg(unix)] +impl Drop for PublicationLocalGuard { + fn drop(&mut self) { + let (held, changed) = PUBLICATION_LOCAL_STATE.get_or_init(Default::default); + held.lock().unwrap_or_else(std::sync::PoisonError::into_inner).remove(&self.identity); + changed.notify_all(); + } +} + +fn publication_timeout(path: &Path) -> std::io::Error { + std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("timed out waiting to publish {}", path.display()), + ) +} + +/// Exclusive authority to mutate one durable filesystem publication. +/// +/// The lock is released by closing its file descriptor. On Unix, callers can +/// deliberately inherit that descriptor into a helper process so ownership +/// survives a parent-process exit without an unlocked interval. +#[must_use = "dropping the guard releases publication ownership"] +pub struct PublicationGuard { + // Rust drops fields in declaration order. Close this process's file-lock + // reference before reopening local admission for another thread. + file: std::fs::File, + target: PathBuf, + #[cfg(unix)] + _local: PublicationLocalGuard, +} + +impl PublicationGuard { + pub fn acquire(path: &Path, deadline: Instant) -> std::io::Result { + let lock_path = publication_lock_path(path)?; + let mut options = std::fs::OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + + options.mode(0o600).custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + } + let file = options.open(&lock_path)?; + #[cfg(unix)] + let local = { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let metadata = file.metadata()?; + if !metadata.file_type().is_file() || metadata.nlink() != 1 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "publication lock is not a single regular file", + )); + } + // SAFETY: geteuid has no preconditions and returns the effective + // user that owns files created by this process. + let current_user = unsafe { libc::geteuid() }; + if metadata.uid() != current_user { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "publication lock is owned by another user", + )); + } + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + PublicationLocalGuard::acquire( + PublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, + path, + deadline, + )? + }; + #[cfg(not(unix))] + crate::platform::restrict_file(&lock_path)?; + loop { + match FileExt::try_lock(&file) { + Ok(()) => { + return Ok(Self { + file, + target: path.to_path_buf(), + #[cfg(unix)] + _local: local, + }); + } + Err(fs4::TryLockError::WouldBlock) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(fs4::TryLockError::WouldBlock) => return Err(publication_timeout(path)), + Err(fs4::TryLockError::Error(error)) => return Err(error), + } + } + } + + pub fn target(&self) -> &Path { + &self.target + } + + /// Arrange for this guard's lock descriptor to survive `exec` in `command`. + /// + /// The returned descriptor number must be passed to the child, which + /// should validate and adopt it with [`Self::adopt_inherited`]. The parent + /// descriptor remains close-on-exec and continues to own the same lock. + #[cfg(unix)] + pub fn inherit_into(&self, command: &mut std::process::Command) -> std::os::fd::RawFd { + use std::os::fd::AsRawFd as _; + use std::os::unix::process::CommandExt as _; + + let descriptor = self.file.as_raw_fd(); + // SAFETY: fcntl is async-signal-safe, and this only clears + // close-on-exec on the guard-owned descriptor in the child between + // fork and exec. The parent descriptor remains close-on-exec. + unsafe { + command.pre_exec(move || { + let flags = libc::fcntl(descriptor, libc::F_GETFD); + if flags == -1 { + return Err(std::io::Error::last_os_error()); + } + if libc::fcntl(descriptor, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + descriptor + } + + /// Validate and adopt a publication descriptor inherited across `exec`. + /// + /// # Safety + /// + /// `descriptor` must be a valid, uniquely owned child-process descriptor + /// produced by [`Self::inherit_into`]. The caller must not use or close it + /// after this function takes ownership. + #[cfg(unix)] + pub unsafe fn adopt_inherited( + path: &Path, + descriptor: std::os::fd::RawFd, + deadline: Instant, + ) -> std::io::Result { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; + use std::os::unix::fs::MetadataExt as _; + + if descriptor <= libc::STDERR_FILENO { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "publication descriptor is invalid", + )); + } + // SAFETY: F_GETFD only validates the caller-owned descriptor. + let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFD) }; + if flags == -1 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: the function contract transfers unique ownership of this + // inherited child descriptor into the returned guard. + let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; + // Do not leak publication ownership into any subprocess the helper + // might launch after adoption. + if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, flags | libc::FD_CLOEXEC) } == -1 { + return Err(std::io::Error::last_os_error()); + } + let metadata = file.metadata()?; + let lock_path = publication_lock_path(path)?; + let published_metadata = std::fs::symlink_metadata(&lock_path)?; + if !metadata.file_type().is_file() + || metadata.nlink() != 1 + || metadata.dev() != published_metadata.dev() + || metadata.ino() != published_metadata.ino() + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "inherited publication descriptor has the wrong identity", + )); + } + let local = PublicationLocalGuard::acquire( + PublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, + path, + deadline, + )?; + // An inherited duplicate shares the already-locked open file + // description. A separately opened descriptor conflicts while the + // legitimate owner is alive, so this also proves lock ownership. + match FileExt::try_lock(&file) { + Ok(()) => Ok(Self { file, target: path.to_path_buf(), _local: local }), + Err(fs4::TryLockError::WouldBlock) => Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "inherited publication descriptor does not own the lock", + )), + Err(fs4::TryLockError::Error(error)) => Err(error), + } + } +} diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index becf8c60cbd2..fa8511747d7a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -33,7 +33,6 @@ use std::thread::JoinHandle; use std::time::{Duration, Instant}; use base64::Engine; -use fs4::FileExt; use ghostty_vt::{ Dirty, KeyAction, KeyEncoder, KeyInput, KittyReplayState, Mods, StyledRun, UnderlineStyle, key_input_from_chord, rows_to_runs, sys, @@ -48,6 +47,7 @@ use tungstenite::protocol::frame::coding::CloseCode; use tungstenite::{Message, WebSocket, accept_with_config}; use zeroize::Zeroize; +use crate::PublicationGuard; use crate::browser::{ BrowserAttachUpdate, BrowserFrameUpdate, BrowserMouseDispatch, BrowserPointerOwner, }; @@ -3924,253 +3924,6 @@ fn socket_publication_hook(path: &Path, stage: SocketPublicationStage) { } } -fn socket_publication_lock_path(path: &Path) -> std::io::Result { - let parent = path.parent().ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "server socket has no parent directory", - ) - })?; - let file_name = path.file_name().ok_or_else(|| { - std::io::Error::new(std::io::ErrorKind::InvalidInput, "server socket has no file name") - })?; - let mut lock_name = std::ffi::OsString::from("."); - lock_name.push(file_name); - lock_name.push(".publish.lock"); - Ok(parent.join(lock_name)) -} - -#[cfg(unix)] -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct SocketPublicationLockIdentity { - device: u64, - inode: u64, -} - -#[cfg(unix)] -static SOCKET_PUBLICATION_LOCAL_STATE: std::sync::OnceLock<( - Mutex>, - Condvar, -)> = std::sync::OnceLock::new(); - -#[cfg(unix)] -struct SocketPublicationLocalGuard { - identity: SocketPublicationLockIdentity, -} - -#[cfg(unix)] -impl SocketPublicationLocalGuard { - fn acquire( - identity: SocketPublicationLockIdentity, - path: &Path, - deadline: Instant, - ) -> std::io::Result { - let (held, changed) = SOCKET_PUBLICATION_LOCAL_STATE.get_or_init(Default::default); - let mut held = held.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - loop { - if held.insert(identity) { - return Ok(Self { identity }); - } - let remaining = deadline - .checked_duration_since(Instant::now()) - .filter(|remaining| !remaining.is_zero()) - .ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::TimedOut, - format!("timed out waiting to publish server socket {}", path.display()), - ) - })?; - held = match changed.wait_timeout(held, remaining) { - Ok((held, _)) => held, - Err(error) => error.into_inner().0, - }; - } - } -} - -#[cfg(unix)] -impl Drop for SocketPublicationLocalGuard { - fn drop(&mut self) { - let (held, changed) = SOCKET_PUBLICATION_LOCAL_STATE.get_or_init(Default::default); - held.lock().unwrap_or_else(std::sync::PoisonError::into_inner).remove(&self.identity); - changed.notify_all(); - } -} - -/// Exclusive authority to mutate one local server socket publication. -/// -/// The lock is released by closing its file descriptor. On Unix, callers can -/// deliberately inherit that descriptor into a helper process so publication -/// ownership survives a parent-process exit without an unlocked interval. -#[must_use = "dropping the guard releases socket publication ownership"] -pub struct SocketPublicationGuard { - // Rust drops fields in declaration order. Close this process's file-lock - // reference before reopening local admission for another thread. - file: std::fs::File, - #[cfg(unix)] - _local: SocketPublicationLocalGuard, -} - -impl SocketPublicationGuard { - pub fn acquire(path: &Path, deadline: Instant) -> std::io::Result { - let lock_path = socket_publication_lock_path(path)?; - let mut options = std::fs::OpenOptions::new(); - options.create(true).truncate(false).read(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - - options.mode(0o600).custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); - } - let file = options.open(&lock_path)?; - #[cfg(unix)] - let local = { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let metadata = file.metadata()?; - if !metadata.file_type().is_file() || metadata.nlink() != 1 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "server socket publication lock is not a single regular file", - )); - } - // SAFETY: geteuid has no preconditions and returns the effective - // user that owns files created by this process. - let current_user = unsafe { libc::geteuid() }; - if metadata.uid() != current_user { - return Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "server socket publication lock is owned by another user", - )); - } - file.set_permissions(std::fs::Permissions::from_mode(0o600))?; - SocketPublicationLocalGuard::acquire( - SocketPublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, - path, - deadline, - )? - }; - #[cfg(not(unix))] - platform::restrict_file(&lock_path)?; - loop { - match FileExt::try_lock(&file) { - Ok(()) => { - return Ok(Self { - file, - #[cfg(unix)] - _local: local, - }); - } - Err(fs4::TryLockError::WouldBlock) if Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(10)); - } - Err(fs4::TryLockError::WouldBlock) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - format!("timed out waiting to publish server socket {}", path.display()), - )); - } - Err(fs4::TryLockError::Error(error)) => return Err(error), - } - } - } - - /// Arrange for this guard's lock descriptor to survive `exec` in `command`. - /// - /// The returned descriptor number must be passed to the child, which - /// should validate and adopt it with [`Self::adopt_inherited`]. The parent - /// descriptor remains close-on-exec and continues to own the same lock. - #[cfg(unix)] - pub fn inherit_into(&self, command: &mut std::process::Command) -> std::os::fd::RawFd { - use std::os::fd::AsRawFd as _; - use std::os::unix::process::CommandExt as _; - - let descriptor = self.file.as_raw_fd(); - // SAFETY: fcntl is async-signal-safe, and this only clears - // close-on-exec on the guard-owned descriptor in the child between - // fork and exec. The parent descriptor remains close-on-exec. - unsafe { - command.pre_exec(move || { - let flags = libc::fcntl(descriptor, libc::F_GETFD); - if flags == -1 { - return Err(std::io::Error::last_os_error()); - } - if libc::fcntl(descriptor, libc::F_SETFD, flags & !libc::FD_CLOEXEC) == -1 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - }); - } - descriptor - } - - /// Validate and adopt a publication descriptor inherited across `exec`. - /// - /// # Safety - /// - /// `descriptor` must be a valid, uniquely owned child-process descriptor - /// produced by [`Self::inherit_into`]. The caller must not use or close it - /// after this function takes ownership. - #[cfg(unix)] - pub unsafe fn adopt_inherited( - path: &Path, - descriptor: std::os::fd::RawFd, - deadline: Instant, - ) -> std::io::Result { - use std::os::fd::{AsRawFd as _, FromRawFd as _}; - use std::os::unix::fs::MetadataExt as _; - - if descriptor <= libc::STDERR_FILENO { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "server socket publication descriptor is invalid", - )); - } - // SAFETY: F_GETFD only validates the caller-owned descriptor. - let flags = unsafe { libc::fcntl(descriptor, libc::F_GETFD) }; - if flags == -1 { - return Err(std::io::Error::last_os_error()); - } - // SAFETY: the function contract transfers unique ownership of this - // inherited child descriptor into the returned guard. - let file = unsafe { std::fs::File::from_raw_fd(descriptor) }; - // Do not leak publication ownership into any subprocess the helper - // might launch after adoption. - if unsafe { libc::fcntl(file.as_raw_fd(), libc::F_SETFD, flags | libc::FD_CLOEXEC) } == -1 { - return Err(std::io::Error::last_os_error()); - } - let metadata = file.metadata()?; - let lock_path = socket_publication_lock_path(path)?; - let published_metadata = std::fs::symlink_metadata(&lock_path)?; - if !metadata.file_type().is_file() - || metadata.nlink() != 1 - || metadata.dev() != published_metadata.dev() - || metadata.ino() != published_metadata.ino() - { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "inherited server socket publication descriptor has the wrong identity", - )); - } - let local = SocketPublicationLocalGuard::acquire( - SocketPublicationLockIdentity { device: metadata.dev(), inode: metadata.ino() }, - path, - deadline, - )?; - // An inherited duplicate shares the already-locked open file - // description. A separately opened descriptor conflicts while the - // legitimate owner is alive, so this also proves lock ownership. - match FileExt::try_lock(&file) { - Ok(()) => Ok(Self { file, _local: local }), - Err(fs4::TryLockError::WouldBlock) => Err(std::io::Error::new( - std::io::ErrorKind::WouldBlock, - "inherited server socket publication descriptor does not own the lock", - )), - Err(fs4::TryLockError::Error(error)) => Err(error), - } - } -} - #[cfg(any(not(unix), test))] fn cleanup_without_stable_identity(_path: &Path) { // Windows does not expose a race-free socket publication identity through @@ -4253,7 +4006,7 @@ impl PublishedSocket { #[cfg(unix)] fn cleanup_unix(&self) -> std::io::Result<()> { - let _publication_lock = SocketPublicationGuard::acquire( + let _publication_lock = PublicationGuard::acquire( &self.path, Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT + LOCAL_SOCKET_CONNECT_TIMEOUT, )?; @@ -4350,7 +4103,7 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result, - terminal_id: Option, + terminal_id: crate::terminal_host::TerminalId, + publication_guard: crate::PublicationGuard, cell_pixels: (u16, u16), ) -> anyhow::Result> { let identity = Some(TabResourceIdentity::terminal(None)?); @@ -2438,7 +2441,8 @@ impl Surface { id, opts, mux, - terminal_id, + Some(terminal_id), + Some(publication_guard), identity, cell_pixels, ) @@ -2458,6 +2462,7 @@ impl Surface { opts, mux, None, + None, resource_identity, cell_pixels, ) @@ -2468,6 +2473,7 @@ impl Surface { opts: SurfaceOptions, mux: Weak, terminal_id: Option, + publication_guard: Option, resource_identity: Option, cell_pixels: (u16, u16), ) -> anyhow::Result> { @@ -2488,19 +2494,19 @@ impl Surface { if let Some(root) = opts.terminal_host_root.clone() { let default_colors = mux.upgrade().map(|mux| mux.default_colors()).unwrap_or_default(); let launch_cancelled = || mux.upgrade().is_none_or(|mux| mux.is_shutting_down()); - let attachment = match terminal_id { - Some(terminal_id) => { + let attachment = match (terminal_id, publication_guard) { + (Some(terminal_id), Some(publication_guard)) => { crate::terminal_host_runtime::launch_terminal_host_with_identity_cancellable( &opts, &root, default_colors, cell_pixels, initial_kitty_limits, - terminal_id, + (terminal_id, publication_guard), &launch_cancelled, )? } - None => crate::terminal_host_runtime::launch_terminal_host_cancellable( + (None, None) => crate::terminal_host_runtime::launch_terminal_host_cancellable( &opts, &root, default_colors, @@ -2508,6 +2514,7 @@ impl Surface { initial_kitty_limits, &launch_cancelled, )?, + _ => anyhow::bail!("terminal-host identity and publication ownership diverged"), }; return Self::spawn_hosted( id, @@ -2519,7 +2526,7 @@ impl Surface { resource_identity, ); } - let _ = terminal_id; + let _ = (terminal_id, publication_guard); let reaper = reserve_local_child_reaper().context("reserve bounded PTY session cleanup")?; let initial_geometry = PtyGeometry { cols: opts.cols, diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 093954aed9af..ea5db2b2d7e2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -1706,11 +1706,29 @@ mod unix { state_root.join(format!("terminal-hosts-{}", stable_token(session))) } - /// Strip every descriptor except the private bootstrap stdio before the - /// hidden host starts any threads or opens its endpoint. This runs inside - /// the freshly exec'd `__terminal-host`, so descriptor enumeration is - /// race-free and cannot affect the daemon's own open files. - pub fn isolate_terminal_host_process_fds() -> anyhow::Result<()> { + fn terminal_host_publication_descriptor(args: &[String]) -> anyhow::Result> { + if args.first().map(String::as_str) != Some("--bootstrap-stdio") { + anyhow::bail!("hidden mode requires --bootstrap-stdio"); + } + match &args[1..] { + [] => Ok(None), + [flag, descriptor] if flag == "--publication-fd" => descriptor + .parse::() + .ok() + .filter(|descriptor| *descriptor > libc::STDERR_FILENO) + .map(Some) + .ok_or_else(|| anyhow::anyhow!("terminal-host publication descriptor is invalid")), + _ => anyhow::bail!("hidden mode received unsupported arguments"), + } + } + + /// Strip every descriptor except the private bootstrap stdio and the + /// explicitly inherited publication guard before the hidden host starts + /// any threads or opens its endpoint. This runs inside the freshly exec'd + /// `__terminal-host`, so descriptor enumeration is race-free and cannot + /// affect the daemon's own open files. + pub fn isolate_terminal_host_process_fds(args: &[String]) -> anyhow::Result<()> { + let publication_descriptor = terminal_host_publication_descriptor(args)?; let mut last_error = None; let mut inherited = None; for directory in ["/proc/self/fd", "/dev/fd"] { @@ -1736,6 +1754,9 @@ mod unix { ) })?; for descriptor in descriptors { + if Some(descriptor) == publication_descriptor { + continue; + } // SAFETY: descriptors came from this single-threaded process's // descriptor filesystem snapshot. stdio 0/1/2 is excluded. if unsafe { libc::close(descriptor) } != 0 { @@ -1779,13 +1800,14 @@ mod unix { cancelled: &dyn Fn() -> bool, ) -> anyhow::Result { let terminal_id = TerminalId::random()?; + let publication_guard = acquire_terminal_host_publication(root, &terminal_id.to_hex())?; launch_terminal_host_with_identity_cancellable( options, root, default_colors, cell_pixels, kitty_graphics_limits, - terminal_id, + (terminal_id, publication_guard), cancelled, ) } @@ -1801,13 +1823,14 @@ mod unix { kitty_graphics_limits: KittyGraphicsLimits, terminal_id: TerminalId, ) -> anyhow::Result { + let publication_guard = acquire_terminal_host_publication(root, &terminal_id.to_hex())?; launch_terminal_host_with_identity_cancellable( options, root, default_colors, cell_pixels, kitty_graphics_limits, - terminal_id, + (terminal_id, publication_guard), &|| false, ) } @@ -1818,9 +1841,10 @@ mod unix { default_colors: DefaultColors, cell_pixels: (u16, u16), kitty_graphics_limits: KittyGraphicsLimits, - terminal_id: TerminalId, + ownership: (TerminalId, crate::PublicationGuard), cancelled: &dyn Fn() -> bool, ) -> anyhow::Result { + let (terminal_id, publication_guard) = ownership; prepare_private_dir(root)?; let owner_token = CapabilityToken::random()?; let terminal_hex = encode_hex(terminal_id.as_bytes()); @@ -1833,6 +1857,10 @@ mod unix { prepare_private_dir(&endpoint_root)?; let endpoint = endpoint_root.join(format!("{terminal_hex}.sock")); let record_path = root.join(format!("{terminal_hex}.json")); + anyhow::ensure!( + publication_guard.target() == record_path, + "terminal-host publication ownership has the wrong target" + ); if record_path.exists() || endpoint.exists() { anyhow::bail!("terminal host identity already exists"); } @@ -1878,6 +1906,8 @@ mod unix { if libc::setsid() < 0 { Err(std::io::Error::last_os_error()) } else { Ok(()) } }); } + let publication_descriptor = publication_guard.inherit_into(&mut command); + command.args(["--publication-fd", &publication_descriptor.to_string()]); let reaper = reserve_host_process_reaper() .context("reserve bounded terminal-host process cleanup")?; let child = cmux_tui_process::spawn(&mut command).context("spawn terminal-host process")?; @@ -2412,6 +2442,25 @@ mod unix { ) } + fn terminal_host_record_path(root: &Path, terminal_id: &str) -> anyhow::Result { + let terminal = TerminalId::from_hex(terminal_id) + .ok_or_else(|| anyhow::anyhow!("terminal-host id is not a canonical UUIDv4"))?; + anyhow::ensure!(terminal.to_hex() == terminal_id, "terminal-host id is not canonical"); + Ok(root.join(format!("{terminal_id}.json"))) + } + + /// Exclude a still-launching host before treating its absent canonical + /// record as proof that no PTY owner can publish later. + pub(crate) fn acquire_terminal_host_publication( + root: &Path, + terminal_id: &str, + ) -> anyhow::Result { + prepare_private_dir(root)?; + let path = terminal_host_record_path(root, terminal_id)?; + crate::PublicationGuard::acquire(&path, Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT) + .with_context(|| format!("wait for terminal-host publication {}", path.display())) + } + /// Load the one canonical discovery record for `terminal_id` without /// scanning unrelated hosts. A missing record means there is no discovery /// owner to terminate; malformed or unreadable records remain errors so a @@ -2420,10 +2469,7 @@ mod unix { root: &Path, terminal_id: &str, ) -> anyhow::Result> { - let terminal = TerminalId::from_hex(terminal_id) - .ok_or_else(|| anyhow::anyhow!("terminal-host id is not a canonical UUIDv4"))?; - anyhow::ensure!(terminal.to_hex() == terminal_id, "terminal-host id is not canonical"); - let path = root.join(format!("{terminal_id}.json")); + let path = terminal_host_record_path(root, terminal_id)?; match read_terminal_host_record_until( &path, Instant::now() + TERMINAL_HOST_RECORD_SCAN_TIMEOUT, @@ -4614,9 +4660,7 @@ mod unix { reader: &mut impl Read, writer: &mut impl Write, ) -> anyhow::Result<()> { - if args.iter().map(String::as_str).ne(["--bootstrap-stdio"]) { - anyhow::bail!("hidden mode requires --bootstrap-stdio"); - } + let publication_descriptor = terminal_host_publication_descriptor(args)?; if let Ok(delay) = std::env::var("CMUX_TUI_TEST_BOOTSTRAP_READY_DELAY_MS") && let Ok(delay) = delay.parse::() && delay > 0 @@ -4640,6 +4684,18 @@ mod unix { anyhow::bail!("expected terminal-host Launch, received {:?}", launch_frame.kind); } let launch = HostLaunch::decode(&launch_frame.payload)?; + let publication_descriptor = publication_descriptor + .ok_or_else(|| anyhow::anyhow!("terminal-host launch omitted publication ownership"))?; + // SAFETY: the private launcher passes one unique child descriptor, + // and fd isolation preserves only that descriptor beyond stdio. + let publication_guard = unsafe { + crate::PublicationGuard::adopt_inherited( + Path::new(&launch.record_path), + publication_descriptor, + Instant::now() + HOST_LAUNCH_TIMEOUT, + ) + } + .context("adopt terminal-host record publication ownership")?; let shared = spawn_host_runtime(&launch, &bootstrapped)?; let endpoint = PathBuf::from(&launch.endpoint); @@ -4687,6 +4743,7 @@ mod unix { // leave behind an undiscoverable terminal process. write_record(Path::new(&launch.record_path), &record)?; guard.published = true; + drop(publication_guard); // Integration failure-injection seam for the narrow record-before- // Ready crash window. It is inherited only by explicitly configured @@ -8717,8 +8774,8 @@ mod unix { #[cfg(unix)] pub(crate) use unix::{ ControlResponses, DecodedHostResize, DeferredCellPixelResolution, - adopt_terminal_host_with_kitty_limits, decode_host_resize_payload_for_version, - load_terminal_host_record, + acquire_terminal_host_publication, adopt_terminal_host_with_kitty_limits, + decode_host_resize_payload_for_version, load_terminal_host_record, }; #[cfg(unix)] pub use unix::{ @@ -8741,7 +8798,7 @@ pub fn terminal_host_root(state_root: &Path, session: &str) -> PathBuf { } #[cfg(not(unix))] -pub fn isolate_terminal_host_process_fds() -> anyhow::Result<()> { +pub fn isolate_terminal_host_process_fds(_args: &[String]) -> anyhow::Result<()> { Ok(()) } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 8fbac3ff0530..c6cf896ca271 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1098,7 +1098,7 @@ fn main() { } fn run_terminal_host_process(args: &[String]) -> anyhow::Result<()> { - cmux_tui_core::terminal_host_runtime::isolate_terminal_host_process_fds()?; + cmux_tui_core::terminal_host_runtime::isolate_terminal_host_process_fds(args)?; let stdin = io::stdin(); let stdout = io::stdout(); let mut reader = stdin.lock(); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index a698a3ef0722..1a3f815742b2 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -12,11 +12,12 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Condvar, Mutex, OnceLock}; use std::time::{Duration, Instant}; +use cmux_tui_core::PublicationGuard; use cmux_tui_core::platform::transport; use cmux_tui_core::release::{LAUNCHER_COMMAND_ENV, ReleaseIdentity}; use cmux_tui_core::server::{ LOCAL_SOCKET_CONNECT_TIMEOUT, PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, - SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, SocketPublicationGuard, + SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, }; use serde_json::{Value, json}; @@ -118,7 +119,7 @@ struct LegacySocketQuarantine { device: u64, inode: u64, armed: bool, - publication_guard: SocketPublicationGuard, + publication_guard: PublicationGuard, } #[cfg(unix)] @@ -133,7 +134,7 @@ impl LegacySocketQuarantine { ) -> std::io::Result { use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; - let publication_guard = SocketPublicationGuard::acquire( + let publication_guard = PublicationGuard::acquire( original, Instant::now() + LOCAL_SOCKET_CONNECT_TIMEOUT + LOCAL_SOCKET_CONNECT_TIMEOUT, )?; @@ -210,7 +211,7 @@ impl LegacySocketQuarantine { quarantined: PathBuf, device: u64, inode: u64, - publication_guard: SocketPublicationGuard, + publication_guard: PublicationGuard, ) -> std::io::Result { let quarantine = Self { original, quarantined, device, inode, armed: true, publication_guard }; @@ -603,20 +604,19 @@ impl ServerIdentity { .filter_map(Value::as_str) .map(str::to_string) .collect(); - let cleanup = data.get("shutdown_cleanup"); - let shutdown_cleanup = ShutdownCleanupStatus { - pending: cleanup - .and_then(|cleanup| cleanup.get("pending")) - .and_then(Value::as_u64) - .unwrap_or(0), - retrying: cleanup - .and_then(|cleanup| cleanup.get("retrying")) - .and_then(Value::as_bool) - .unwrap_or(false), - degraded: cleanup - .and_then(|cleanup| cleanup.get("degraded")) - .and_then(Value::as_bool) - .unwrap_or(false), + let shutdown_cleanup = match data.get("shutdown_cleanup") { + None => ShutdownCleanupStatus::default(), + Some(cleanup) => ShutdownCleanupStatus { + pending: cleanup.get("pending").and_then(Value::as_u64).ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) + })?, + retrying: cleanup.get("retrying").and_then(Value::as_bool).ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) + })?, + degraded: cleanup.get("degraded").and_then(Value::as_bool).ok_or_else(|| { + anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) + })?, + }, }; Ok(Self { release: ReleaseIdentity::from_protocol_data(data), @@ -1222,9 +1222,9 @@ pub(crate) fn run_legacy_stop_helper(args: &[String]) -> anyhow::Result<()> { let deadline = Instant::now() + timeout; ensure_legacy_helper_active()?; // SAFETY: this private exec mode receives the unique child copy created by - // `SocketPublicationGuard::inherit_into` and never aliases that raw fd. + // `PublicationGuard::inherit_into` and never aliases that raw fd. let publication_guard = unsafe { - SocketPublicationGuard::adopt_inherited( + PublicationGuard::adopt_inherited( Path::new(original_path), publication_descriptor, deadline, From 4704fbc3773dd1cada144b9ab2b92e84990c59b1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:12:24 -0700 Subject: [PATCH 338/521] test(mac): expose lost surface creation retry --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 82 ++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index f790eb428832..6bc0e5963102 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -1,5 +1,6 @@ -import Foundation import AppKit +import Dispatch +import Foundation import GhosttyKit import Testing import CmuxTerminalCore @@ -189,6 +190,81 @@ import CmuxTerminalCore #expect(surface.runtimeSurfacePointer == nil) } + @Test func runtimeCreationRetriesAfterTeardownAdmissionRecovers() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + closeTeardownTimeout: .milliseconds(50), + maximumRuntimeSurfaceOwnerCount: 4 + ) + let retainedSurfaces = (0..<2).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + defer { for retainedSurface in retainedSurfaces { retainedSurface.deallocate() } } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = DispatchSemaphore(value: 0) + defer { + releaseFrees.signal() + releaseFrees.signal() + freeStarted.continuation.finish() + } + + let tickets = try retainedSurfaces.enumerated().map { index, retainedSurface in + let ticket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.creationRetry.\(index)", + surface: retainedSurface, + callbackContext: nil, + freeSurface: { _ in + freeStarted.continuation.yield() + releaseFrees.wait() + } + ) + return try #require(ticket) + } + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + _ = await freeStartedIterator.next() + + let degradationDeadline = ContinuousClock.now + .seconds(1) + while await !coordinator.debugCloseTeardownDegraded, + ContinuousClock.now < degradationDeadline { + await Task.yield() + } + #expect(await coordinator.debugCloseTeardownDegraded) + + let nativeView = FakeTerminalSurfaceNativeView( + frame: NSRect(x: 0, y: 0, width: 800, height: 600) + ) + let paneHost = FakeTerminalSurfacePaneHost(surfaceView: nativeView) + let surface = makeSurface( + runtimeSpawnPolicy: .immediate, + scheduler: RecordingRestoreSpawnScheduler(), + nativeView: nativeView, + paneHost: paneHost, + runtimeTeardown: coordinator + ) + surface.claudeCommandShimInstallCompleted = true + + surface.createSurface(for: nativeView) + #expect(surface.debugRuntimeSurfaceCreateAttemptCountForTesting() == 1) + + releaseFrees.signal() + let retryDeadline = ContinuousClock.now + .seconds(1) + while surface.debugRuntimeSurfaceCreateAttemptCountForTesting() < 2, + ContinuousClock.now < retryDeadline { + await Task.yield() + } + + #expect( + surface.debugRuntimeSurfaceCreateAttemptCountForTesting() == 2, + "surface creation was not retried after native ownership admission recovered" + ) + releaseFrees.signal() + for ticket in tickets { + #expect(await ticket.wait(timeout: .seconds(1))) + } + } + @Test func configurationReloadDefersAndPromotesRuntimeCreation() { let nativeView = FakeTerminalSurfaceNativeView( frame: NSRect(x: 0, y: 0, width: 800, height: 600) @@ -402,6 +478,8 @@ import CmuxTerminalCore nativeView: FakeTerminalSurfaceNativeView, paneHost: FakeTerminalSurfacePaneHost, engine: FakeTerminalEngine = FakeTerminalEngine(), + runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator = + TerminalSurfaceRuntimeTeardownCoordinator(), runtimeFilesystem: TerminalSurfaceRuntimeFilesystem = TerminalSurfaceRuntimeFilesystem( claudeCommandShimTemporaryDirectory: URL(fileURLWithPath: "/tmp/cmux-terminal-tests", isDirectory: true), installClaudeCommandShim: { _, _, _ in nil }, @@ -421,7 +499,7 @@ import CmuxTerminalCore byteTee: FakeTerminalByteTee(), rendererRealization: FakeRendererRealizationScheduler(), hibernationRecorder: FakeHibernationRecorder(), - runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator(), + runtimeTeardown: runtimeTeardown, restoreSpawnScheduler: scheduler, runtimeFilesystem: runtimeFilesystem, sessionPortBase: 40_000, From 5137b03a931ce7873a3138d8df4a4c2b52e39265 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:31:04 -0700 Subject: [PATCH 339/521] fix: recover deferred surface creation --- ...minalSurfaceRuntimeTeardownAdmission.swift | 119 +++++++++++++++++- ...nalSurfaceRuntimeTeardownCoordinator.swift | 14 +++ .../TerminalSurface+RuntimeLifecycle.swift | 80 ++++++++++-- .../Surface/TerminalSurface.swift | 4 + cmux-tui/crates/cmux-pty/src/lib.rs | 1 + cmux-tui/crates/cmux-tui-core/src/platform.rs | 2 +- 6 files changed, 205 insertions(+), 15 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift index 996585a1809a..1789a2dd95b9 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -1,6 +1,9 @@ import Foundation internal import os +typealias TerminalSurfaceRuntimeOwnershipRecovery = + @MainActor @Sendable () -> Void + /// Synchronous, bounded admission for native-surface ownership. /// /// Creation and deinit cannot suspend while transferring a native pointer, so @@ -9,9 +12,18 @@ internal import os /// worker returns, preventing repeated create/close cycles from growing an /// unbounded retained teardown backlog. final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { + private struct RecoveryEntry { + var action: TerminalSurfaceRuntimeOwnershipRecovery + var previousID: UUID? + var nextID: UUID? + } + private struct State { var reservationIDs: Set = [] var closeTeardownDegraded = false + var recoveryEntriesByID: [UUID: RecoveryEntry] = [:] + var recoveryHeadID: UUID? + var recoveryTailID: UUID? } private let maximumOwnerCount: Int @@ -26,12 +38,25 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { self.maximumOwnerCount = maximumOwnerCount } - func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { + func reserve( + recoveryID: UUID? = nil, + onRecovery: TerminalSurfaceRuntimeOwnershipRecovery? = nil + ) -> TerminalSurfaceRuntimeOwnershipReservation? { state.withLock { state in guard !state.closeTeardownDegraded, state.reservationIDs.count < maximumOwnerCount else { + if let recoveryID, let onRecovery { + enqueueRecoveryAction( + onRecovery, + id: recoveryID, + in: &state + ) + } return nil } + if let recoveryID { + _ = removeRecoveryAction(recoveryID, from: &state) + } let reservation = TerminalSurfaceRuntimeOwnershipReservation() state.reservationIDs.insert(reservation.id) return reservation @@ -43,13 +68,101 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { - state.withLock { state in + let recoveryActions = state.withLock { state in _ = state.reservationIDs.remove(reservation.id) + return takeAvailableRecoveryActions(from: &state) } + schedule(recoveryActions) } func setCloseTeardownDegraded(_ degraded: Bool) { - state.withLock { $0.closeTeardownDegraded = degraded } + let recoveryActions = state.withLock { state in + state.closeTeardownDegraded = degraded + return takeAvailableRecoveryActions(from: &state) + } + schedule(recoveryActions) + } + + func cancelRecovery(_ recoveryID: UUID) { + state.withLock { state in + _ = removeRecoveryAction(recoveryID, from: &state) + } + } + + private func removeRecoveryAction( + _ recoveryID: UUID, + from state: inout State + ) -> TerminalSurfaceRuntimeOwnershipRecovery? { + guard let entry = state.recoveryEntriesByID.removeValue( + forKey: recoveryID + ) else { + return nil + } + if let previousID = entry.previousID { + state.recoveryEntriesByID[previousID]?.nextID = entry.nextID + } else { + state.recoveryHeadID = entry.nextID + } + if let nextID = entry.nextID { + state.recoveryEntriesByID[nextID]?.previousID = entry.previousID + } else { + state.recoveryTailID = entry.previousID + } + return entry.action + } + + private func enqueueRecoveryAction( + _ action: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + id recoveryID: UUID, + in state: inout State + ) { + if state.recoveryEntriesByID[recoveryID] != nil { + state.recoveryEntriesByID[recoveryID]?.action = action + return + } + let previousID = state.recoveryTailID + state.recoveryEntriesByID[recoveryID] = RecoveryEntry( + action: action, + previousID: previousID, + nextID: nil + ) + if let previousID { + state.recoveryEntriesByID[previousID]?.nextID = recoveryID + } else { + state.recoveryHeadID = recoveryID + } + state.recoveryTailID = recoveryID + } + + private func takeAvailableRecoveryActions( + from state: inout State + ) -> [TerminalSurfaceRuntimeOwnershipRecovery] { + guard !state.closeTeardownDegraded else { return [] } + let availableCount = maximumOwnerCount - state.reservationIDs.count + guard availableCount > 0, state.recoveryHeadID != nil else { + return [] + } + var recoveryActions: [TerminalSurfaceRuntimeOwnershipRecovery] = [] + while recoveryActions.count < availableCount, + let recoveryID = state.recoveryHeadID { + if let recoveryAction = removeRecoveryAction( + recoveryID, + from: &state + ) { + recoveryActions.append(recoveryAction) + } + } + return recoveryActions + } + + private func schedule( + _ recoveryActions: [TerminalSurfaceRuntimeOwnershipRecovery] + ) { + for recoveryAction in recoveryActions { + Task { @MainActor in + recoveryAction() + } + } } #if DEBUG diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 5a2a905c0b79..337e22c89c16 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -113,6 +113,20 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { runtimeOwnershipAdmission.reserve() } + nonisolated func reserveRuntimeSurfaceOwnership( + recoveryID: UUID, + onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery + ) -> TerminalSurfaceRuntimeOwnershipReservation? { + runtimeOwnershipAdmission.reserve( + recoveryID: recoveryID, + onRecovery: onRecovery + ) + } + + nonisolated func cancelRuntimeSurfaceOwnershipRecovery(_ recoveryID: UUID) { + runtimeOwnershipAdmission.cancelRecovery(recoveryID) + } + nonisolated func cancelRuntimeSurfaceOwnership( _ reservation: TerminalSurfaceRuntimeOwnershipReservation ) { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index e0840b200f37..e8ba77083375 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -208,6 +208,7 @@ extension TerminalSurface { guard portalLifecycleState != .closing else { return } recordTeardownRequest(reason: reason) portalLifecycleState = .closing + cancelRuntimeSurfaceCreationAfterAdmissionRecovery() // Parked wake-ups are for re-anchoring live content; a closing surface // has none, and the park guard refuses new entries from here on. clearPortalHostVacancyRetries() @@ -224,6 +225,7 @@ extension TerminalSurface { func markPortalLifecycleClosed(reason: String) { guard portalLifecycleState != .closed else { return } portalLifecycleState = .closed + cancelRuntimeSurfaceCreationAfterAdmissionRecovery() portalLifecycleGeneration &+= 1 clearPortalHostVacancyRetries() #if DEBUG @@ -382,6 +384,7 @@ extension TerminalSurface { _ = fontSizeLineageSnapshot() mobileViewportFontFitState = nil runtimeSurfaceSuspendedForAgentHibernation = true + cancelRuntimeSurfaceCreationAfterAdmissionRecovery() backgroundSurfaceStartQueued = false backgroundSurfaceStartSource = .normal cancelClaudeCommandShimInstallLifecycle() @@ -695,6 +698,55 @@ extension TerminalSurface { createSurface(for: view, source: source) } + @MainActor + private func reserveRuntimeSurfaceOwnershipForCreation( + view: any TerminalSurfaceNativeViewing, + source: RuntimeSurfaceCreationSource + ) -> TerminalSurfaceRuntimeOwnershipReservation? { + runtimeSurfaceAdmissionDeferredCreationSource = + ( + runtimeSurfaceAdmissionDeferredCreationSource + ?? source + ).promoted(with: source) + runtimeSurfaceAdmissionDeferredCreationView = view + guard let reservation = + runtimeTeardown.reserveRuntimeSurfaceOwnership( + recoveryID: id, + onRecovery: { [weak self] in + self? + .resumeRuntimeSurfaceCreationAfterAdmissionRecovery() + } + ) else { + return nil + } + runtimeSurfaceAdmissionDeferredCreationSource = nil + runtimeSurfaceAdmissionDeferredCreationView = nil + return reservation + } + + @MainActor + private func resumeRuntimeSurfaceCreationAfterAdmissionRecovery() { + let source = + runtimeSurfaceAdmissionDeferredCreationSource + ?? .normal + let view = + runtimeSurfaceAdmissionDeferredCreationView + ?? attachedView + ?? surfaceView + runtimeSurfaceAdmissionDeferredCreationSource = nil + runtimeSurfaceAdmissionDeferredCreationView = nil + guard allowsRuntimeSurfaceCreation(), surface == nil else { + return + } + createSurface(for: view, source: source) + } + + private func cancelRuntimeSurfaceCreationAfterAdmissionRecovery() { + runtimeTeardown.cancelRuntimeSurfaceOwnershipRecovery(id) + runtimeSurfaceAdmissionDeferredCreationSource = nil + runtimeSurfaceAdmissionDeferredCreationView = nil + } + @MainActor func createSurface(for view: any TerminalSurfaceNativeViewing, source: RuntimeSurfaceCreationSource) { guard allowsRuntimeSurfaceCreation() else { @@ -734,18 +786,11 @@ extension TerminalSurface { Self.surfaceLog("createSurface start surface=\(id.uuidString) tab=\(tabId.uuidString) bounds=\(view.bounds) inWindow=\(view.window != nil) resources=\(resourcesDir) terminfo=\(terminfo) xdg=\(xdg) manpath=\(manpath)") #endif - guard let app = engine.runtimeApp else { - #if DEBUG - logDebugEvent("ghostty.surface.create.failed reason=appNotInitialized surface=\(id.uuidString)") - #endif - #if DEBUG - Self.surfaceLog("createSurface FAILED surface=\(id.uuidString): ghostty app not initialized") - #endif - return - } - guard let runtimeOwnershipReservation = - runtimeTeardown.reserveRuntimeSurfaceOwnership() else { + reserveRuntimeSurfaceOwnershipForCreation( + view: view, + source: source + ) else { #if DEBUG logDebugEvent( "ghostty.surface.create.failed reason=teardownAdmissionDegraded " + @@ -755,6 +800,19 @@ extension TerminalSurface { return } + guard let app = engine.runtimeApp else { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + runtimeOwnershipReservation + ) + #if DEBUG + logDebugEvent("ghostty.surface.create.failed reason=appNotInitialized surface=\(id.uuidString)") + #endif + #if DEBUG + Self.surfaceLog("createSurface FAILED surface=\(id.uuidString): ghostty app not initialized") + #endif + return + } + let scaleFactors = scaleFactors(for: view) let runtimeSurfaceCreation = createNativeRuntimeSurface( diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index dadd6de09e44..60f2eda69b6d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -278,6 +278,10 @@ public final class TerminalSurface: Identifiable, ObservableObject { RuntimeSurfaceCreationSource? weak var configurationReloadDeferredRuntimeSurfaceView: (any TerminalSurfaceNativeViewing)? + var runtimeSurfaceAdmissionDeferredCreationSource: + RuntimeSurfaceCreationSource? + weak var runtimeSurfaceAdmissionDeferredCreationView: + (any TerminalSurfaceNativeViewing)? var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false /// Bounded ownership of the live native surface. Transferred to the diff --git a/cmux-tui/crates/cmux-pty/src/lib.rs b/cmux-tui/crates/cmux-pty/src/lib.rs index e98cd8d0ff57..f6013b797e02 100644 --- a/cmux-tui/crates/cmux-pty/src/lib.rs +++ b/cmux-tui/crates/cmux-pty/src/lib.rs @@ -195,6 +195,7 @@ mod tests { ); } + #[cfg(target_os = "macos")] #[test] fn pty_spawn_waits_for_process_creation_barrier() { let pair = open(PtySize { rows: 24, cols: 80, pixel_width: 0, pixel_height: 0 }).unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 6a45801a56ff..19b1e94b6c53 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -780,7 +780,7 @@ pub mod transport { ); } - #[cfg(not(target_os = "linux"))] + #[cfg(all(unix, not(target_os = "linux")))] #[test] fn deadline_connect_confines_inheritable_descriptor_to_process_barrier() { use std::os::fd::AsRawFd as _; From e0a903f1f980ff540a666d54d831e166326d9f22 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:00:55 -0700 Subject: [PATCH 340/521] test(mac): expose stale recovery waiter race --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 0cdfe3001281..2525f933ed76 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -59,6 +59,9 @@ private final class LifetimeRecordingByteTeeLease: TerminalByteTeeLease, @unchec } } +@MainActor +private final class RuntimeOwnershipRecoveryProbe {} + private func requireTeardownTicket( _ ticket: TerminalSurfaceRuntimeTeardownTicket? ) throws -> TerminalSurfaceRuntimeTeardownTicket { @@ -279,6 +282,50 @@ private func requireTeardownTicket( #expect(await recoveredTicket.wait(timeout: .seconds(1))) } + @MainActor + @Test func staleRecoveryHeadDoesNotStrandTheNextWaiter() async throws { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + let firstOwner = try #require(admission.reserve()) + let secondOwner = try #require(admission.reserve()) + var staleProbe: RuntimeOwnershipRecoveryProbe? = + RuntimeOwnershipRecoveryProbe() + weak let releasedProbe = staleProbe + let staleRecoveryID = UUID() + let nextRecoveryID = UUID() + var nextRecoveryRan = false + + let staleReservation = admission.reserve( + recoveryID: staleRecoveryID, + onRecovery: { [weak staleProbe] in + _ = staleProbe + } + ) + let nextReservation = admission.reserve( + recoveryID: nextRecoveryID, + onRecovery: { + nextRecoveryRan = true + } + ) + #expect(staleReservation == nil) + #expect(nextReservation == nil) + + admission.release(firstOwner) + staleProbe = nil + #expect(releasedProbe == nil) + for _ in 0..<100 where !nextRecoveryRan { + await Task.yield() + } + + #expect( + nextRecoveryRan, + "a stale dequeued recovery callback stranded the next waiter despite free capacity" + ) + admission.cancelRecovery(nextRecoveryID) + admission.release(secondOwner) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) From 268373d9ac61edb26f43183c23d027c67f796e67 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:04:41 -0700 Subject: [PATCH 341/521] fix(mac): transfer recovered surface ownership --- ...minalSurfaceRuntimeTeardownAdmission.swift | 30 ++++++--- .../TerminalSurface+RuntimeLifecycle.swift | 65 +++++++++++++++---- ...rfaceRuntimeTeardownCoordinatorTests.swift | 14 +++- 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift index 1789a2dd95b9..db77a8c8b865 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -2,7 +2,7 @@ import Foundation internal import os typealias TerminalSurfaceRuntimeOwnershipRecovery = - @MainActor @Sendable () -> Void + @MainActor @Sendable (TerminalSurfaceRuntimeOwnershipReservation) -> Void /// Synchronous, bounded admission for native-surface ownership. /// @@ -12,6 +12,11 @@ typealias TerminalSurfaceRuntimeOwnershipRecovery = /// worker returns, preventing repeated create/close cycles from growing an /// unbounded retained teardown backlog. final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { + private struct RecoveryGrant: Sendable { + let action: TerminalSurfaceRuntimeOwnershipRecovery + let reservation: TerminalSurfaceRuntimeOwnershipReservation + } + private struct RecoveryEntry { var action: TerminalSurfaceRuntimeOwnershipRecovery var previousID: UUID? @@ -136,31 +141,38 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { private func takeAvailableRecoveryActions( from state: inout State - ) -> [TerminalSurfaceRuntimeOwnershipRecovery] { + ) -> [RecoveryGrant] { guard !state.closeTeardownDegraded else { return [] } let availableCount = maximumOwnerCount - state.reservationIDs.count guard availableCount > 0, state.recoveryHeadID != nil else { return [] } - var recoveryActions: [TerminalSurfaceRuntimeOwnershipRecovery] = [] - while recoveryActions.count < availableCount, + var recoveryGrants: [RecoveryGrant] = [] + while recoveryGrants.count < availableCount, let recoveryID = state.recoveryHeadID { if let recoveryAction = removeRecoveryAction( recoveryID, from: &state ) { - recoveryActions.append(recoveryAction) + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + recoveryGrants.append( + RecoveryGrant( + action: recoveryAction, + reservation: reservation + ) + ) } } - return recoveryActions + return recoveryGrants } private func schedule( - _ recoveryActions: [TerminalSurfaceRuntimeOwnershipRecovery] + _ recoveryGrants: [RecoveryGrant] ) { - for recoveryAction in recoveryActions { + for recoveryGrant in recoveryGrants { Task { @MainActor in - recoveryAction() + recoveryGrant.action(recoveryGrant.reservation) } } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index e8ba77083375..085f9d06be25 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -709,12 +709,20 @@ extension TerminalSurface { ?? source ).promoted(with: source) runtimeSurfaceAdmissionDeferredCreationView = view + let teardownCoordinator = runtimeTeardown guard let reservation = - runtimeTeardown.reserveRuntimeSurfaceOwnership( + teardownCoordinator.reserveRuntimeSurfaceOwnership( recoveryID: id, - onRecovery: { [weak self] in - self? - .resumeRuntimeSurfaceCreationAfterAdmissionRecovery() + onRecovery: { [weak self, teardownCoordinator] reservation in + guard let self else { + teardownCoordinator.cancelRuntimeSurfaceOwnership( + reservation + ) + return + } + self.resumeRuntimeSurfaceCreationAfterAdmissionRecovery( + reservation: reservation + ) } ) else { return nil @@ -725,7 +733,9 @@ extension TerminalSurface { } @MainActor - private func resumeRuntimeSurfaceCreationAfterAdmissionRecovery() { + private func resumeRuntimeSurfaceCreationAfterAdmissionRecovery( + reservation: TerminalSurfaceRuntimeOwnershipReservation + ) { let source = runtimeSurfaceAdmissionDeferredCreationSource ?? .normal @@ -735,10 +745,11 @@ extension TerminalSurface { ?? surfaceView runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil - guard allowsRuntimeSurfaceCreation(), surface == nil else { - return - } - createSurface(for: view, source: source) + createSurface( + for: view, + source: source, + recoveredOwnershipReservation: reservation + ) } private func cancelRuntimeSurfaceCreationAfterAdmissionRecovery() { @@ -749,6 +760,27 @@ extension TerminalSurface { @MainActor func createSurface(for view: any TerminalSurfaceNativeViewing, source: RuntimeSurfaceCreationSource) { + createSurface( + for: view, + source: source, + recoveredOwnershipReservation: nil + ) + } + + @MainActor + private func createSurface( + for view: any TerminalSurfaceNativeViewing, + source: RuntimeSurfaceCreationSource, + recoveredOwnershipReservation: TerminalSurfaceRuntimeOwnershipReservation? + ) { + var recoveredOwnershipReservation = recoveredOwnershipReservation + defer { + if let recoveredOwnershipReservation { + runtimeTeardown.cancelRuntimeSurfaceOwnership( + recoveredOwnershipReservation + ) + } + } guard allowsRuntimeSurfaceCreation() else { #if DEBUG logDebugEvent( @@ -786,11 +818,16 @@ extension TerminalSurface { Self.surfaceLog("createSurface start surface=\(id.uuidString) tab=\(tabId.uuidString) bounds=\(view.bounds) inWindow=\(view.window != nil) resources=\(resourcesDir) terminfo=\(terminfo) xdg=\(xdg) manpath=\(manpath)") #endif - guard let runtimeOwnershipReservation = - reserveRuntimeSurfaceOwnershipForCreation( - view: view, - source: source - ) else { + let runtimeOwnershipReservation: TerminalSurfaceRuntimeOwnershipReservation + if let recoveredReservation = recoveredOwnershipReservation { + runtimeOwnershipReservation = recoveredReservation + recoveredOwnershipReservation = nil + } else if let reservation = reserveRuntimeSurfaceOwnershipForCreation( + view: view, + source: source + ) { + runtimeOwnershipReservation = reservation + } else { #if DEBUG logDebugEvent( "ghostty.surface.create.failed reason=teardownAdmissionDegraded " + diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 2525f933ed76..04c41e1cf73f 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -294,18 +294,25 @@ private func requireTeardownTicket( weak let releasedProbe = staleProbe let staleRecoveryID = UUID() let nextRecoveryID = UUID() + var staleRecoveryDeclined = false var nextRecoveryRan = false let staleReservation = admission.reserve( recoveryID: staleRecoveryID, - onRecovery: { [weak staleProbe] in - _ = staleProbe + onRecovery: { [weak staleProbe] reservation in + guard staleProbe != nil else { + staleRecoveryDeclined = true + admission.release(reservation) + return + } + admission.release(reservation) } ) let nextReservation = admission.reserve( recoveryID: nextRecoveryID, - onRecovery: { + onRecovery: { reservation in nextRecoveryRan = true + admission.release(reservation) } ) #expect(staleReservation == nil) @@ -318,6 +325,7 @@ private func requireTeardownTicket( await Task.yield() } + #expect(staleRecoveryDeclined) #expect( nextRecoveryRan, "a stale dequeued recovery callback stranded the next waiter despite free capacity" From 6efb7bb2f17f1395f0b7bb51271bb30a11294822 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:09:51 -0700 Subject: [PATCH 342/521] refactor(mac): align teardown ownership policy --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 173 ++++++++++++++++ ...urfaceRuntimeOwnershipAdmissionState.swift | 10 + ...SurfaceRuntimeOwnershipRecoveryEntry.swift | 7 + ...SurfaceRuntimeOwnershipRecoveryGrant.swift | 4 + ...alSurfaceRuntimeOwnershipReservation.swift | 14 ++ ...minalSurfaceRuntimeTeardownAdmission.swift | 188 ------------------ ...nalSurfaceRuntimeTeardownCoordinator.swift | 61 +++--- ...nalSurfaceRuntimeTeardownReservation.swift | 13 -- ...rfaceRuntimeTeardownCoordinatorTests.swift | 6 +- 9 files changed, 242 insertions(+), 234 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryGrant.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipReservation.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift new file mode 100644 index 000000000000..1675cea4ed9e --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -0,0 +1,173 @@ +import Foundation +internal import os + +typealias TerminalSurfaceRuntimeOwnershipRecovery = + @MainActor @Sendable (TerminalSurfaceRuntimeOwnershipReservation) -> Void + +/// Synchronous, bounded admission for native-surface ownership. +/// +/// Creation, public teardown, and deinit cannot suspend while transferring a +/// native pointer, so this ledger uses one short unfair-lock critical section. +/// Native work never runs under the lock. A fully stalled close pool degrades +/// admission until a worker returns, preventing repeated create/close cycles +/// from growing an unbounded retained teardown backlog. +final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { + private let maximumOwnerCount: Int + private let state = OSAllocatedUnfairLock( + initialState: TerminalSurfaceRuntimeOwnershipAdmissionState() + ) + + init(maximumOwnerCount: Int) { + precondition( + maximumOwnerCount >= TerminalSurfaceRuntimeTeardownCoordinator + .maximumConcurrentCloseTeardownCount, + "native surface ownership must cover every close worker" + ) + self.maximumOwnerCount = maximumOwnerCount + } + + func reserve( + recoveryID: UUID? = nil, + onRecovery: TerminalSurfaceRuntimeOwnershipRecovery? = nil + ) -> TerminalSurfaceRuntimeOwnershipReservation? { + state.withLock { state in + guard !state.closeTeardownDegraded, + state.reservationIDs.count < maximumOwnerCount else { + if let recoveryID, let onRecovery { + enqueueRecoveryAction( + onRecovery, + id: recoveryID, + in: &state + ) + } + return nil + } + if let recoveryID { + _ = removeRecoveryAction(recoveryID, from: &state) + } + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + return reservation + } + } + + func contains(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) -> Bool { + state.withLock { $0.reservationIDs.contains(reservation.id) } + } + + func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { + let recoveryActions = state.withLock { state in + _ = state.reservationIDs.remove(reservation.id) + return takeAvailableRecoveryActions(from: &state) + } + schedule(recoveryActions) + } + + func setCloseTeardownDegraded(_ degraded: Bool) { + let recoveryActions = state.withLock { state in + state.closeTeardownDegraded = degraded + return takeAvailableRecoveryActions(from: &state) + } + schedule(recoveryActions) + } + + func cancelRecovery(_ recoveryID: UUID) { + state.withLock { state in + _ = removeRecoveryAction(recoveryID, from: &state) + } + } + + private func removeRecoveryAction( + _ recoveryID: UUID, + from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> TerminalSurfaceRuntimeOwnershipRecovery? { + guard let entry = state.recoveryEntriesByID.removeValue( + forKey: recoveryID + ) else { + return nil + } + if let previousID = entry.previousID { + state.recoveryEntriesByID[previousID]?.nextID = entry.nextID + } else { + state.recoveryHeadID = entry.nextID + } + if let nextID = entry.nextID { + state.recoveryEntriesByID[nextID]?.previousID = entry.previousID + } else { + state.recoveryTailID = entry.previousID + } + return entry.action + } + + private func enqueueRecoveryAction( + _ action: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + id recoveryID: UUID, + in state: inout TerminalSurfaceRuntimeOwnershipAdmissionState + ) { + if state.recoveryEntriesByID[recoveryID] != nil { + state.recoveryEntriesByID[recoveryID]?.action = action + return + } + let previousID = state.recoveryTailID + state.recoveryEntriesByID[recoveryID] = + TerminalSurfaceRuntimeOwnershipRecoveryEntry( + action: action, + previousID: previousID, + nextID: nil + ) + if let previousID { + state.recoveryEntriesByID[previousID]?.nextID = recoveryID + } else { + state.recoveryHeadID = recoveryID + } + state.recoveryTailID = recoveryID + } + + private func takeAvailableRecoveryActions( + from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> [TerminalSurfaceRuntimeOwnershipRecoveryGrant] { + guard !state.closeTeardownDegraded else { return [] } + let availableCount = maximumOwnerCount - state.reservationIDs.count + guard availableCount > 0, state.recoveryHeadID != nil else { + return [] + } + var recoveryGrants: [TerminalSurfaceRuntimeOwnershipRecoveryGrant] = [] + while recoveryGrants.count < availableCount, + let recoveryID = state.recoveryHeadID { + if let recoveryAction = removeRecoveryAction( + recoveryID, + from: &state + ) { + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + recoveryGrants.append( + TerminalSurfaceRuntimeOwnershipRecoveryGrant( + action: recoveryAction, + reservation: reservation + ) + ) + } + } + return recoveryGrants + } + + private func schedule( + _ recoveryGrants: [TerminalSurfaceRuntimeOwnershipRecoveryGrant] + ) { + for recoveryGrant in recoveryGrants { + Task { @MainActor in + recoveryGrant.action(recoveryGrant.reservation) + } + } + } + +#if DEBUG + var debugCloseTeardownDegraded: Bool { + state.withLock { $0.closeTeardownDegraded } + } + + var debugOwnerCount: Int { + state.withLock { $0.reservationIDs.count } + } +#endif +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift new file mode 100644 index 000000000000..2f5970fdd751 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -0,0 +1,10 @@ +import Foundation + +struct TerminalSurfaceRuntimeOwnershipAdmissionState { + var reservationIDs: Set = [] + var closeTeardownDegraded = false + var recoveryEntriesByID: + [UUID: TerminalSurfaceRuntimeOwnershipRecoveryEntry] = [:] + var recoveryHeadID: UUID? + var recoveryTailID: UUID? +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift new file mode 100644 index 000000000000..f2c4f42af7f6 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift @@ -0,0 +1,7 @@ +import Foundation + +struct TerminalSurfaceRuntimeOwnershipRecoveryEntry: Sendable { + var action: TerminalSurfaceRuntimeOwnershipRecovery + var previousID: UUID? + var nextID: UUID? +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryGrant.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryGrant.swift new file mode 100644 index 000000000000..09a39d8f7807 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryGrant.swift @@ -0,0 +1,4 @@ +struct TerminalSurfaceRuntimeOwnershipRecoveryGrant: Sendable { + let action: TerminalSurfaceRuntimeOwnershipRecovery + let reservation: TerminalSurfaceRuntimeOwnershipReservation +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipReservation.swift new file mode 100644 index 000000000000..92d84a202f95 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipReservation.swift @@ -0,0 +1,14 @@ +import Foundation + +/// Ownership of one live or retained native Ghostty surface. +/// +/// The reservation begins before `ghostty_surface_new` and ends only after +/// `ghostty_surface_free` returns. This bounds the combined live-surface and +/// teardown backlog even when every native-free worker is stalled. +struct TerminalSurfaceRuntimeOwnershipReservation: Equatable, Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift index db77a8c8b865..403ba88a3186 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownAdmission.swift @@ -1,192 +1,4 @@ import Foundation -internal import os - -typealias TerminalSurfaceRuntimeOwnershipRecovery = - @MainActor @Sendable (TerminalSurfaceRuntimeOwnershipReservation) -> Void - -/// Synchronous, bounded admission for native-surface ownership. -/// -/// Creation and deinit cannot suspend while transferring a native pointer, so -/// this ledger uses one short unfair-lock critical section. Native work never -/// runs under the lock. A fully stalled close pool degrades admission until a -/// worker returns, preventing repeated create/close cycles from growing an -/// unbounded retained teardown backlog. -final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { - private struct RecoveryGrant: Sendable { - let action: TerminalSurfaceRuntimeOwnershipRecovery - let reservation: TerminalSurfaceRuntimeOwnershipReservation - } - - private struct RecoveryEntry { - var action: TerminalSurfaceRuntimeOwnershipRecovery - var previousID: UUID? - var nextID: UUID? - } - - private struct State { - var reservationIDs: Set = [] - var closeTeardownDegraded = false - var recoveryEntriesByID: [UUID: RecoveryEntry] = [:] - var recoveryHeadID: UUID? - var recoveryTailID: UUID? - } - - private let maximumOwnerCount: Int - private let state = OSAllocatedUnfairLock(initialState: State()) - - init(maximumOwnerCount: Int) { - precondition( - maximumOwnerCount >= TerminalSurfaceRuntimeTeardownCoordinator - .maximumConcurrentCloseTeardownCount, - "native surface ownership must cover every close worker" - ) - self.maximumOwnerCount = maximumOwnerCount - } - - func reserve( - recoveryID: UUID? = nil, - onRecovery: TerminalSurfaceRuntimeOwnershipRecovery? = nil - ) -> TerminalSurfaceRuntimeOwnershipReservation? { - state.withLock { state in - guard !state.closeTeardownDegraded, - state.reservationIDs.count < maximumOwnerCount else { - if let recoveryID, let onRecovery { - enqueueRecoveryAction( - onRecovery, - id: recoveryID, - in: &state - ) - } - return nil - } - if let recoveryID { - _ = removeRecoveryAction(recoveryID, from: &state) - } - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) - return reservation - } - } - - func contains(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) -> Bool { - state.withLock { $0.reservationIDs.contains(reservation.id) } - } - - func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { - let recoveryActions = state.withLock { state in - _ = state.reservationIDs.remove(reservation.id) - return takeAvailableRecoveryActions(from: &state) - } - schedule(recoveryActions) - } - - func setCloseTeardownDegraded(_ degraded: Bool) { - let recoveryActions = state.withLock { state in - state.closeTeardownDegraded = degraded - return takeAvailableRecoveryActions(from: &state) - } - schedule(recoveryActions) - } - - func cancelRecovery(_ recoveryID: UUID) { - state.withLock { state in - _ = removeRecoveryAction(recoveryID, from: &state) - } - } - - private func removeRecoveryAction( - _ recoveryID: UUID, - from state: inout State - ) -> TerminalSurfaceRuntimeOwnershipRecovery? { - guard let entry = state.recoveryEntriesByID.removeValue( - forKey: recoveryID - ) else { - return nil - } - if let previousID = entry.previousID { - state.recoveryEntriesByID[previousID]?.nextID = entry.nextID - } else { - state.recoveryHeadID = entry.nextID - } - if let nextID = entry.nextID { - state.recoveryEntriesByID[nextID]?.previousID = entry.previousID - } else { - state.recoveryTailID = entry.previousID - } - return entry.action - } - - private func enqueueRecoveryAction( - _ action: @escaping TerminalSurfaceRuntimeOwnershipRecovery, - id recoveryID: UUID, - in state: inout State - ) { - if state.recoveryEntriesByID[recoveryID] != nil { - state.recoveryEntriesByID[recoveryID]?.action = action - return - } - let previousID = state.recoveryTailID - state.recoveryEntriesByID[recoveryID] = RecoveryEntry( - action: action, - previousID: previousID, - nextID: nil - ) - if let previousID { - state.recoveryEntriesByID[previousID]?.nextID = recoveryID - } else { - state.recoveryHeadID = recoveryID - } - state.recoveryTailID = recoveryID - } - - private func takeAvailableRecoveryActions( - from state: inout State - ) -> [RecoveryGrant] { - guard !state.closeTeardownDegraded else { return [] } - let availableCount = maximumOwnerCount - state.reservationIDs.count - guard availableCount > 0, state.recoveryHeadID != nil else { - return [] - } - var recoveryGrants: [RecoveryGrant] = [] - while recoveryGrants.count < availableCount, - let recoveryID = state.recoveryHeadID { - if let recoveryAction = removeRecoveryAction( - recoveryID, - from: &state - ) { - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) - recoveryGrants.append( - RecoveryGrant( - action: recoveryAction, - reservation: reservation - ) - ) - } - } - return recoveryGrants - } - - private func schedule( - _ recoveryGrants: [RecoveryGrant] - ) { - for recoveryGrant in recoveryGrants { - Task { @MainActor in - recoveryGrant.action(recoveryGrant.reservation) - } - } - } - -#if DEBUG - var debugCloseTeardownDegraded: Bool { - state.withLock { $0.closeTeardownDegraded } - } - - var debugOwnerCount: Int { - state.withLock { $0.reservationIDs.count } - } -#endif -} /// Main-actor admission state for one coordinator's bounded native-free slots. @MainActor diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 337e22c89c16..d1e02196e52e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -6,6 +6,32 @@ internal import Dispatch internal import CMUXDebugLog #endif +private func makeTerminalSurfaceRuntimeCloseTeardownQueues() + -> [DispatchQueue] { + ( + 0.. [DispatchQueue] { + ( + 0.. [DispatchQueue] { - ( - 0.. [DispatchQueue] { - ( - 0.. TerminalSurfaceRuntimeTeardownTicket { @@ -289,8 +286,7 @@ private func requireTeardownTicket( ) let firstOwner = try #require(admission.reserve()) let secondOwner = try #require(admission.reserve()) - var staleProbe: RuntimeOwnershipRecoveryProbe? = - RuntimeOwnershipRecoveryProbe() + var staleProbe: NSObject? = NSObject() weak let releasedProbe = staleProbe let staleRecoveryID = UUID() let nextRecoveryID = UUID() From 57ee7c5c362935e98073826b5892debe1bf4bfb6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:40:35 -0700 Subject: [PATCH 343/521] test: expose recovery boundedness failures --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 90 +++++++++++++++++++ ...rfaceRuntimeTeardownCoordinatorTests.swift | 31 +++++++ cmux-tui/crates/cmux-tui-cdp/src/client.rs | 35 ++++++++ 3 files changed, 156 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 6bc0e5963102..71a3062ef6cc 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -265,6 +265,96 @@ import CmuxTerminalCore } } + @Test func restoredCreationReentersPacingAfterAdmissionRecovers() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + closeTeardownTimeout: .milliseconds(50), + maximumRuntimeSurfaceOwnerCount: 4 + ) + let retainedSurfaces = (0..<2).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + defer { + for retainedSurface in retainedSurfaces { + retainedSurface.deallocate() + } + } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = DispatchSemaphore(value: 0) + defer { + releaseFrees.signal() + releaseFrees.signal() + freeStarted.continuation.finish() + } + + let tickets = try retainedSurfaces.enumerated().map { + index, + retainedSurface in + let ticket = coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.pacedCreationRetry.\(index)", + surface: retainedSurface, + callbackContext: nil, + freeSurface: { _ in + freeStarted.continuation.yield() + releaseFrees.wait() + } + ) + return try #require(ticket) + } + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = await freeStartedIterator.next() + _ = await freeStartedIterator.next() + + let degradationDeadline = ContinuousClock.now + .seconds(1) + while await !coordinator.debugCloseTeardownDegraded, + ContinuousClock.now < degradationDeadline { + await Task.yield() + } + #expect(await coordinator.debugCloseTeardownDegraded) + + let nativeView = FakeTerminalSurfaceNativeView( + frame: NSRect(x: 0, y: 0, width: 800, height: 600) + ) + let paneHost = FakeTerminalSurfacePaneHost(surfaceView: nativeView) + let scheduler = RecordingRestoreSpawnScheduler() + let surface = makeSurface( + scheduler: scheduler, + nativeView: nativeView, + paneHost: paneHost, + runtimeTeardown: coordinator + ) + surface.scheduleHeadlessRuntimeStartIfNeeded( + reason: "test-admission-recovery-pacing" + ) + defer { surface.closeHeadlessStartupWindowIfNeeded() } + surface.attachedView = nativeView + surface.claudeCommandShimInstallCompleted = true + + surface.createSurface(for: nativeView) + #expect(scheduler.scheduledSurfaceIds == [surface.id]) + scheduler.runScheduledOperation() + #expect(surface.debugRuntimeSurfaceCreateAttemptCountForTesting() == 1) + + releaseFrees.signal() + let recoveryDeadline = ContinuousClock.now + .seconds(1) + while scheduler.scheduledSurfaceIds.count < 2, + surface.debugRuntimeSurfaceCreateAttemptCountForTesting() < 2, + ContinuousClock.now < recoveryDeadline { + await Task.yield() + } + + #expect(scheduler.scheduledSurfaceIds == [surface.id, surface.id]) + #expect( + surface.debugRuntimeSurfaceCreateAttemptCountForTesting() == 1, + "admission recovery bypassed restored-surface spawn pacing" + ) + releaseFrees.signal() + for ticket in tickets { + #expect(await ticket.wait(timeout: .seconds(1))) + } + } + @Test func configurationReloadDefersAndPromotesRuntimeCreation() { let nativeView = FakeTerminalSurfaceNativeView( frame: NSRect(x: 0, y: 0, width: 800, height: 600) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index b08b23dd8720..04e0b82ae4f7 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -330,6 +330,37 @@ private func requireTeardownTicket( admission.release(secondOwner) } + @MainActor + @Test func recoveryGrantsOnlyOneWaiterAtATime() async { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 4 + ) + admission.setCloseTeardownDegraded(true) + var recoveredCount = 0 + + for _ in 0..<3 { + let reservation = admission.reserve( + recoveryID: UUID(), + onRecovery: { reservation in + recoveredCount += 1 + admission.release(reservation) + } + ) + #expect(reservation == nil) + } + + admission.setCloseTeardownDegraded(false) + #expect( + admission.debugOwnerCount == 1, + "admission reserved every available recovery slot before the main actor ran" + ) + for _ in 0..<100 where recoveredCount < 3 { + await Task.yield() + } + #expect(recoveredCount == 3) + #expect(admission.debugOwnerCount == 0) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 9ef1b37e079b..cfd415bb2ae6 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -4537,6 +4537,41 @@ mod tests { assert!(wait_calls < 12, "retryable child wait errors were hot-polled {wait_calls} times"); } + #[test] + fn resolver_reaper_retains_capacity_after_inconclusive_wait_error() { + let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + resolver_child_reaper().unwrap(); + let active = { + let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); + slot.as_ref().unwrap().active.clone() + }; + let baseline = active.load(Ordering::Acquire); + let lease = reserve_resolver_child_reaper().unwrap(); + let mut command = Command::new("/bin/sh"); + command.args(["-c", "sleep 30"]); + let child = cmux_tui_process::spawn(&mut command).unwrap(); + + TEST_RESOLVER_REAPER_WAIT_CALLS.store(0, Ordering::Release); + *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = + Some(std::io::ErrorKind::PermissionDenied); + handoff_resolver_child(child, lease); + let observation_deadline = Instant::now() + Duration::from_millis(250); + while TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire) == 0 + && Instant::now() < observation_deadline + { + thread::sleep(Duration::from_millis(2)); + } + let retained = active.load(Ordering::Acquire) == baseline + 1; + *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = None; + + let cleanup_deadline = Instant::now() + Duration::from_secs(1); + while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { + thread::sleep(Duration::from_millis(5)); + } + assert_eq!(active.load(Ordering::Acquire), baseline); + assert!(retained, "inconclusive child wait error discarded resolver ownership"); + } + #[test] fn timed_out_host_lookups_do_not_strand_resolver_capacity() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); From 1abb1d611e1f2431957d40a1cacf95c1a131b329 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:43:21 -0700 Subject: [PATCH 344/521] fix: retain and pace recovery ownership --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 72 ++++++++++--------- ...urfaceRuntimeOwnershipAdmissionState.swift | 1 + .../TerminalSurface+RuntimeLifecycle.swift | 8 +++ cmux-tui/crates/cmux-tui-cdp/Cargo.toml | 4 +- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 14 +++- 5 files changed, 60 insertions(+), 39 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 1675cea4ed9e..8801ecd7b090 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -56,19 +56,19 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { - let recoveryActions = state.withLock { state in + let recoveryGrant = state.withLock { state in _ = state.reservationIDs.remove(reservation.id) - return takeAvailableRecoveryActions(from: &state) + return takeNextRecoveryGrant(from: &state) } - schedule(recoveryActions) + schedule(recoveryGrant) } func setCloseTeardownDegraded(_ degraded: Bool) { - let recoveryActions = state.withLock { state in + let recoveryGrant = state.withLock { state in state.closeTeardownDegraded = degraded - return takeAvailableRecoveryActions(from: &state) + return takeNextRecoveryGrant(from: &state) } - schedule(recoveryActions) + schedule(recoveryGrant) } func cancelRecovery(_ recoveryID: UUID) { @@ -123,42 +123,44 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.recoveryTailID = recoveryID } - private func takeAvailableRecoveryActions( + private func takeNextRecoveryGrant( from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState - ) -> [TerminalSurfaceRuntimeOwnershipRecoveryGrant] { - guard !state.closeTeardownDegraded else { return [] } - let availableCount = maximumOwnerCount - state.reservationIDs.count - guard availableCount > 0, state.recoveryHeadID != nil else { - return [] - } - var recoveryGrants: [TerminalSurfaceRuntimeOwnershipRecoveryGrant] = [] - while recoveryGrants.count < availableCount, - let recoveryID = state.recoveryHeadID { - if let recoveryAction = removeRecoveryAction( - recoveryID, - from: &state - ) { - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) - recoveryGrants.append( - TerminalSurfaceRuntimeOwnershipRecoveryGrant( - action: recoveryAction, - reservation: reservation - ) - ) - } + ) -> TerminalSurfaceRuntimeOwnershipRecoveryGrant? { + guard !state.closeTeardownDegraded, + !state.recoveryGrantIsScheduled, + state.reservationIDs.count < maximumOwnerCount, + let recoveryID = state.recoveryHeadID, + let recoveryAction = removeRecoveryAction( + recoveryID, + from: &state + ) else { + return nil } - return recoveryGrants + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + state.recoveryGrantIsScheduled = true + return TerminalSurfaceRuntimeOwnershipRecoveryGrant( + action: recoveryAction, + reservation: reservation + ) } private func schedule( - _ recoveryGrants: [TerminalSurfaceRuntimeOwnershipRecoveryGrant] + _ recoveryGrant: TerminalSurfaceRuntimeOwnershipRecoveryGrant? ) { - for recoveryGrant in recoveryGrants { - Task { @MainActor in - recoveryGrant.action(recoveryGrant.reservation) - } + guard let recoveryGrant else { return } + Task { @MainActor in + recoveryGrant.action(recoveryGrant.reservation) + recoveryGrantDidComplete() + } + } + + private func recoveryGrantDidComplete() { + let recoveryGrant = state.withLock { state in + state.recoveryGrantIsScheduled = false + return takeNextRecoveryGrant(from: &state) } + schedule(recoveryGrant) } #if DEBUG diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index 2f5970fdd751..e717f9a63f3d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -7,4 +7,5 @@ struct TerminalSurfaceRuntimeOwnershipAdmissionState { [UUID: TerminalSurfaceRuntimeOwnershipRecoveryEntry] = [:] var recoveryHeadID: UUID? var recoveryTailID: UUID? + var recoveryGrantIsScheduled = false } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 085f9d06be25..818aaaed8b69 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -745,6 +745,14 @@ extension TerminalSurface { ?? surfaceView runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil + if requiresRestoreSpawnPacing, source == .scheduledRestore { + // The failed attempt already consumed its prior restore cadence + // slot. Return admission ownership and use the shared normal + // entrypoint to queue a fresh paced slot before retrying Ghostty. + runtimeTeardown.cancelRuntimeSurfaceOwnership(reservation) + createSurface(for: view, source: .normal) + return + } createSurface( for: view, source: source, diff --git a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml index 6d70f794edb2..b30c4199e477 100644 --- a/cmux-tui/crates/cmux-tui-cdp/Cargo.toml +++ b/cmux-tui/crates/cmux-tui-cdp/Cargo.toml @@ -14,8 +14,6 @@ workspace = true anyhow.workspace = true base64.workspace = true cmux-tui-process.workspace = true +libc.workspace = true serde_json.workspace = true tungstenite.workspace = true - -[dev-dependencies] -libc.workspace = true diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index cfd415bb2ae6..6c50a8f25d21 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -437,7 +437,7 @@ fn run_resolver_child_reaper( let _ = request.child.kill(); match resolver_child_try_wait(request) { Ok(Some(_)) => false, - Err(error) if error.kind() != std::io::ErrorKind::Interrupted => false, + Err(error) if resolver_child_wait_lost_ownership(&error) => false, Ok(None) | Err(_) => { if now >= request.retry_deadline { degraded.store(true, Ordering::Release); @@ -455,6 +455,18 @@ fn run_resolver_child_reaper( } } +fn resolver_child_wait_lost_ownership(error: &std::io::Error) -> bool { + #[cfg(unix)] + { + error.raw_os_error() == Some(libc::ECHILD) + } + #[cfg(not(unix))] + { + let _ = error; + false + } +} + fn resolver_child_try_wait( request: &mut ResolverChildReapRequest, ) -> std::io::Result> { From 7f86fc0a158befcd44daa95b5abcf91fe88432f1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 15:15:56 -0700 Subject: [PATCH 345/521] test(tui): expose unbounded WebSocket teardown --- cmux-tui/crates/cmux-tui-core/src/server.rs | 86 ++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index fa8511747d7a..c34e9493504a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4167,6 +4167,8 @@ pub struct WebSocketServer { shutdown: Arc, connections: Arc>>, thread: Option>, + #[cfg(test)] + accept_attempts: Arc, } impl WebSocketServer { @@ -4218,11 +4220,17 @@ pub fn serve_websocket( let connections = Arc::new(Mutex::new(HashMap::new())); let next_connection = Arc::new(AtomicU64::new(1)); let active_connections = Arc::new(AtomicU64::new(0)); + #[cfg(test)] + let accept_attempts = Arc::new(AtomicU64::new(0)); let thread_shutdown = shutdown.clone(); let thread_connections = connections.clone(); + #[cfg(test)] + let thread_accept_attempts = accept_attempts.clone(); let render_service = Arc::new(RenderService::new()); let thread = std::thread::Builder::new().name("mux-ws-server".into()).spawn(move || { while !thread_shutdown.load(Ordering::Acquire) { + #[cfg(test)] + thread_accept_attempts.fetch_add(1, Ordering::Release); let (stream, peer) = match cmux_tui_process::tcp::accept_stream(&listener) { Ok(connection) => connection, Err(_) => { @@ -4270,7 +4278,14 @@ pub fn serve_websocket( } } })?; - Ok(WebSocketServer { local_addr, shutdown, connections, thread: Some(thread) }) + Ok(WebSocketServer { + local_addr, + shutdown, + connections, + thread: Some(thread), + #[cfg(test)] + accept_attempts, + }) } pub fn window_title_osc(title: &str) -> Vec { @@ -10426,6 +10441,75 @@ mod tests { use std::sync::mpsc::TryRecvError; use std::time::Duration; + #[test] + fn websocket_listener_shutdown_does_not_depend_on_a_wakeup_connection() { + let mux = Mux::new("websocket-listener-shutdown-test", SurfaceOptions::default()); + let server = + serve_websocket(mux.clone(), "127.0.0.1:0".parse().unwrap(), None, false).unwrap(); + + let accept_deadline = Instant::now() + Duration::from_secs(1); + while server.accept_attempts.load(Ordering::Acquire) == 0 + && Instant::now() < accept_deadline + { + std::thread::yield_now(); + } + assert_ne!( + server.accept_attempts.load(Ordering::Acquire), + 0, + "WebSocket listener never attempted to accept" + ); + server.shutdown.store(true, Ordering::Release); + let deadline = Instant::now() + STREAM_DISCONNECT_POLL + STREAM_DISCONNECT_POLL; + while !server.thread.as_ref().unwrap().is_finished() && Instant::now() < deadline { + std::thread::yield_now(); + } + let stopped_without_wakeup = server.thread.as_ref().unwrap().is_finished(); + if !stopped_without_wakeup { + let _ = cmux_tui_process::tcp::connect_stream_timeout( + &server.local_addr, + Duration::from_millis(100), + ); + } + drop(server); + let _ = mux.shutdown(); + + assert!( + stopped_without_wakeup, + "WebSocket listener shutdown required a self-connect to wake accept" + ); + } + + #[test] + fn websocket_drop_does_not_join_an_unfinished_listener() { + let (release, blocked) = std::sync::mpsc::channel(); + let (worker_finished, observed_worker) = std::sync::mpsc::channel(); + let listener_thread = std::thread::spawn(move || { + let _ = blocked.recv(); + let _ = worker_finished.send(()); + }); + let server = WebSocketServer { + local_addr: "127.0.0.1:0".parse().unwrap(), + shutdown: Arc::new(AtomicBool::new(false)), + connections: Arc::new(Mutex::new(HashMap::new())), + thread: Some(listener_thread), + accept_attempts: Arc::new(AtomicU64::new(0)), + }; + let (dropped, observed_drop) = std::sync::mpsc::channel(); + let dropper = std::thread::spawn(move || { + drop(server); + let _ = dropped.send(()); + }); + + let completed_promptly = observed_drop.recv_timeout(Duration::from_millis(100)).is_ok(); + let _ = release.send(()); + observed_worker + .recv_timeout(Duration::from_secs(1)) + .expect("synthetic listener did not finish after release"); + dropper.join().unwrap(); + + assert!(completed_promptly, "WebSocket owner drop joined an unfinished listener thread"); + } + #[test] fn default_socket_path_preserves_compatible_runtime_dir() { let runtime_dir = PathBuf::from("/tmp/cmux-tui-compat"); From 6517f7f55c2c0661a41b700dd58305a3bb7f5279 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:47:10 -0700 Subject: [PATCH 346/521] fix(tui): bound WebSocket server teardown --- cmux-tui/crates/cmux-tui-core/src/server.rs | 78 +++++++++++++++++++-- cmux-tui/crates/cmux-tui/src/main.rs | 29 ++++++-- 2 files changed, 93 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index c34e9493504a..b4ea4c871d28 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4175,19 +4175,52 @@ impl WebSocketServer { pub fn local_addr(&self) -> SocketAddr { self.local_addr } + + /// Request shutdown without waiting past `deadline` for the listener. + /// + /// Returns `Ok(false)` while the caller must retain this owner and retry. + pub fn shutdown_until(&mut self, deadline: Instant) -> anyhow::Result { + self.shutdown.store(true, Ordering::Release); + while !self.try_close_connections() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + std::thread::sleep(remaining.min(Duration::from_millis(1))); + } + + let Some(thread) = self.thread.as_ref() else { return Ok(true) }; + while !thread.is_finished() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + std::thread::sleep(remaining.min(Duration::from_millis(1))); + } + let thread = self.thread.take().expect("finished WebSocket listener retained its owner"); + thread.join().map_err(|_| anyhow::anyhow!("WebSocket listener thread panicked"))?; + Ok(true) + } + + fn try_close_connections(&self) -> bool { + let connections = match self.connections.try_lock() { + Ok(connections) => connections, + Err(std::sync::TryLockError::Poisoned(error)) => error.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => return false, + }; + for stream in connections.values() { + let _ = stream.shutdown(Shutdown::Both); + } + true + } } impl Drop for WebSocketServer { fn drop(&mut self) { self.shutdown.store(true, Ordering::Release); - for stream in self.connections.lock().unwrap().values() { - let _ = stream.shutdown(Shutdown::Both); - } - if let Ok(stream) = cmux_tui_process::tcp::connect_stream(self.local_addr) { - let _ = stream.set_nodelay(true); - } + let _ = self.try_close_connections(); if let Some(thread) = self.thread.take() { - let _ = thread.join(); + if thread.is_finished() { + let _ = thread.join(); + } } } } @@ -4215,6 +4248,7 @@ pub fn serve_websocket( } } let listener = cmux_tui_process::tcp::bind_listener(addr)?; + listener.set_nonblocking(true)?; let local_addr = listener.local_addr()?; let shutdown = Arc::new(AtomicBool::new(false)); let connections = Arc::new(Mutex::new(HashMap::new())); @@ -4277,6 +4311,11 @@ pub fn serve_websocket( cleanup_connections.lock().unwrap().remove(&id); } } + let connections = + thread_connections.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for stream in connections.values() { + let _ = stream.shutdown(Shutdown::Both); + } })?; Ok(WebSocketServer { local_addr, @@ -10510,6 +10549,31 @@ mod tests { assert!(completed_promptly, "WebSocket owner drop joined an unfinished listener thread"); } + #[test] + fn websocket_shutdown_retains_an_unfinished_listener_for_retry() { + let (release, blocked) = std::sync::mpsc::channel(); + let listener_thread = std::thread::spawn(move || { + let _ = blocked.recv(); + }); + let mut server = WebSocketServer { + local_addr: "127.0.0.1:0".parse().unwrap(), + shutdown: Arc::new(AtomicBool::new(false)), + connections: Arc::new(Mutex::new(HashMap::new())), + thread: Some(listener_thread), + accept_attempts: Arc::new(AtomicU64::new(0)), + }; + + let first = server.shutdown_until(Instant::now() + Duration::from_millis(25)).unwrap(); + let retained = server.thread.is_some(); + let _ = release.send(()); + let second = server.shutdown_until(Instant::now() + Duration::from_secs(1)).unwrap(); + + assert!(!first, "unfinished WebSocket listener reported complete"); + assert!(retained, "pending WebSocket shutdown discarded listener ownership"); + assert!(second, "WebSocket listener did not complete after release"); + assert!(server.thread.is_none(), "completed WebSocket listener retained its join handle"); + } + #[test] fn default_socket_path_preserves_compatible_runtime_dir() { let runtime_dir = PathBuf::from("/tmp/cmux-tui-compat"); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index c6cf896ca271..ed629f0e0c69 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1315,7 +1315,22 @@ impl ServerProcessOwners { fn shutdown_until(&mut self, deadline: std::time::Instant) -> ServerProcessOwnersShutdown { #[cfg(target_os = "linux")] drop(self.provider_management.take()); - drop(self.websocket.take()); + if let Some(websocket) = self.websocket.as_mut() { + match websocket.shutdown_until(deadline) { + Ok(true) => drop(self.websocket.take()), + Ok(false) => { + return ServerProcessOwnersShutdown::Pending { + message: "WebSocket server cleanup exceeded the shutdown deadline", + }; + } + Err(error) => { + drop(self.websocket.take()); + return ServerProcessOwnersShutdown::Complete(Err( + error.context("WebSocket server shutdown failed") + )); + } + } + } #[cfg(unix)] if let Some(remote) = self.remote.as_mut() { match remote.shutdown_until(deadline) { @@ -1324,7 +1339,9 @@ impl ServerProcessOwners { return ServerProcessOwnersShutdown::Complete(result); } remote_runtime::DaemonRuntimeShutdown::Pending => { - return ServerProcessOwnersShutdown::Pending; + return ServerProcessOwnersShutdown::Pending { + message: "remote daemon cleanup exceeded the shutdown deadline", + }; } } } @@ -1334,7 +1351,7 @@ impl ServerProcessOwners { enum ServerProcessOwnersShutdown { Complete(anyhow::Result<()>), - Pending, + Pending { message: &'static str }, } enum ServerProcessOwnersState { @@ -1479,11 +1496,9 @@ impl ServerProcessOwnersCleanup { *state = ServerProcessOwnersState::Complete(error.clone()); ServerProcessOwnersCleanupOutcome::Complete { error } } - Ok(ServerProcessOwnersShutdown::Pending) => { + Ok(ServerProcessOwnersShutdown::Pending { message }) => { *state = ServerProcessOwnersState::Ready(owners); - ServerProcessOwnersCleanupOutcome::Incomplete { - message: "remote daemon cleanup exceeded the shutdown deadline".into(), - } + ServerProcessOwnersCleanupOutcome::Incomplete { message: message.into() } } Err(_) => { *state = ServerProcessOwnersState::Ready(owners); From 14a1243c95e5bbf2cab24b1fc79956d8fbe7c9ae Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 2 Aug 2026 16:52:19 -0700 Subject: [PATCH 347/521] test(mac): expose nonblocking WebSocket clients --- cmux-tui/crates/cmux-tui-core/src/server.rs | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index b4ea4c871d28..43b8787fc55a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4169,6 +4169,8 @@ pub struct WebSocketServer { thread: Option>, #[cfg(test)] accept_attempts: Arc, + #[cfg(all(test, target_os = "macos"))] + accepted_stream_mode: Arc, } impl WebSocketServer { @@ -4256,10 +4258,14 @@ pub fn serve_websocket( let active_connections = Arc::new(AtomicU64::new(0)); #[cfg(test)] let accept_attempts = Arc::new(AtomicU64::new(0)); + #[cfg(all(test, target_os = "macos"))] + let accepted_stream_mode = Arc::new(AtomicU64::new(0)); let thread_shutdown = shutdown.clone(); let thread_connections = connections.clone(); #[cfg(test)] let thread_accept_attempts = accept_attempts.clone(); + #[cfg(all(test, target_os = "macos"))] + let thread_accepted_stream_mode = accepted_stream_mode.clone(); let render_service = Arc::new(RenderService::new()); let thread = std::thread::Builder::new().name("mux-ws-server".into()).spawn(move || { while !thread_shutdown.load(Ordering::Acquire) { @@ -4277,6 +4283,11 @@ pub fn serve_websocket( continue; } }; + #[cfg(all(test, target_os = "macos"))] + thread_accepted_stream_mode.store( + if tcp_stream_is_nonblocking_for_test(&stream) { 2 } else { 1 }, + Ordering::Release, + ); if stream.set_nodelay(true).is_err() { continue; } @@ -4324,9 +4335,21 @@ pub fn serve_websocket( thread: Some(thread), #[cfg(test)] accept_attempts, + #[cfg(all(test, target_os = "macos"))] + accepted_stream_mode, }) } +#[cfg(all(test, target_os = "macos"))] +fn tcp_stream_is_nonblocking_for_test(stream: &TcpStream) -> bool { + use std::os::fd::AsRawFd as _; + + // SAFETY: F_GETFL only reads flags from this live TCP descriptor. + let flags = unsafe { libc::fcntl(stream.as_raw_fd(), libc::F_GETFL) }; + assert!(flags >= 0, "failed to read accepted WebSocket stream flags"); + flags & libc::O_NONBLOCK != 0 +} + pub fn window_title_osc(title: &str) -> Vec { let title = sanitize_window_title(title); format!("\x1b]0;{title}\x07\x1b]2;{title}\x07").into_bytes() @@ -10532,6 +10555,8 @@ mod tests { connections: Arc::new(Mutex::new(HashMap::new())), thread: Some(listener_thread), accept_attempts: Arc::new(AtomicU64::new(0)), + #[cfg(target_os = "macos")] + accepted_stream_mode: Arc::new(AtomicU64::new(0)), }; let (dropped, observed_drop) = std::sync::mpsc::channel(); let dropper = std::thread::spawn(move || { @@ -10561,6 +10586,8 @@ mod tests { connections: Arc::new(Mutex::new(HashMap::new())), thread: Some(listener_thread), accept_attempts: Arc::new(AtomicU64::new(0)), + #[cfg(target_os = "macos")] + accepted_stream_mode: Arc::new(AtomicU64::new(0)), }; let first = server.shutdown_until(Instant::now() + Duration::from_millis(25)).unwrap(); @@ -10574,6 +10601,34 @@ mod tests { assert!(server.thread.is_none(), "completed WebSocket listener retained its join handle"); } + #[cfg(target_os = "macos")] + #[test] + fn websocket_accept_restores_blocking_mode_before_protocol_reads() { + let mux = Mux::new("websocket-blocking-accept-test", SurfaceOptions::default()); + let server = + serve_websocket(mux.clone(), "127.0.0.1:0".parse().unwrap(), None, false).unwrap(); + let client = cmux_tui_process::tcp::connect_stream_timeout( + &server.local_addr, + Duration::from_secs(1), + ) + .unwrap(); + + let deadline = Instant::now() + Duration::from_secs(1); + while server.accepted_stream_mode.load(Ordering::Acquire) == 0 && Instant::now() < deadline + { + std::thread::yield_now(); + } + let accepted_stream_mode = server.accepted_stream_mode.load(Ordering::Acquire); + drop(client); + drop(server); + let _ = mux.shutdown(); + + assert_eq!( + accepted_stream_mode, 1, + "accepted WebSocket stream remained nonblocking before protocol reads" + ); + } + #[test] fn default_socket_path_preserves_compatible_runtime_dir() { let runtime_dir = PathBuf::from("/tmp/cmux-tui-compat"); From 40e37424db886ce42fb3fcc38cc7c8987e260b97 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 17:31:58 -0700 Subject: [PATCH 348/521] test(tui): replace sleeps with lifecycle signals The three-line server nonblocking-reset hunk was pre-existing local work with an unknown original editor and was preserved unchanged. --- cmux-tui/crates/cmux-remote/src/daemon.rs | 39 +- .../cmux-remote/src/workspace/process.rs | 19 +- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 229 ++++--- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 397 +++++++---- cmux-tui/crates/cmux-tui-core/src/browser.rs | 54 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 380 ++++++----- cmux-tui/crates/cmux-tui-core/src/platform.rs | 10 +- .../cmux-tui-core/src/process_session.rs | 79 ++- cmux-tui/crates/cmux-tui-core/src/server.rs | 121 ++-- .../cmux-tui-core/src/spawned_pty_child.rs | 83 ++- cmux-tui/crates/cmux-tui-core/src/surface.rs | 409 +++++++---- .../src/terminal_host_runtime.rs | 45 +- cmux-tui/crates/cmux-tui-process/src/lib.rs | 18 +- .../crates/cmux-tui/src/machine_runtime.rs | 21 +- cmux-tui/crates/cmux-tui/src/main.rs | 16 +- .../crates/cmux-tui/src/remote_runtime.rs | 104 ++- .../crates/cmux-tui/src/server_lifecycle.rs | 45 +- .../src/server_lifecycle/legacy_process.rs | 49 +- .../crates/cmux-tui/src/session/remote.rs | 4 + cmux-tui/crates/cmux-tui/tests/cli.rs | 641 ++++++++++-------- .../cmux-tui/tests/terminal_host_recovery.rs | 149 ++-- 21 files changed, 1791 insertions(+), 1121 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/src/daemon.rs b/cmux-tui/crates/cmux-remote/src/daemon.rs index d978515d760c..cd0c31b3f995 100644 --- a/cmux-tui/crates/cmux-remote/src/daemon.rs +++ b/cmux-tui/crates/cmux-remote/src/daemon.rs @@ -1720,14 +1720,16 @@ mod tests { let (daemon, _accepted) = RemoteDaemon::new(auth, SessionLimits::default()); let (release, holder) = hold_process_creation_barrier(); - let bind = tokio::spawn(serve_direct_websocket( - daemon, - "127.0.0.1:0".parse().unwrap(), - 128 * 1024, - false, - )); - tokio::time::sleep(Duration::from_millis(250)).await; - let completed_while_barrier_held = bind.is_finished(); + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let mut bind = tokio::spawn(async move { + started_sender.send(()).unwrap(); + serve_direct_websocket(daemon, "127.0.0.1:0".parse().unwrap(), 128 * 1024, false).await + }); + started_receiver.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(250), &mut bind).await.is_err(), + "direct socket bound while a concurrent process could inherit its descriptor" + ); release.send(()).unwrap(); holder.join().unwrap(); @@ -1737,10 +1739,6 @@ mod tests { .unwrap() .unwrap(); server.shutdown().await.unwrap(); - assert!( - !completed_while_barrier_held, - "direct socket bound while a concurrent process could inherit its descriptor" - ); } #[cfg(target_os = "macos")] @@ -1753,9 +1751,16 @@ mod tests { LimitedTcpListener { inner: listener, permits: Arc::new(Semaphore::new(1)) }; let (release, holder) = hold_process_creation_barrier(); - let accept = tokio::spawn(async move { listener.accept().await }); - tokio::time::sleep(Duration::from_millis(250)).await; - let completed_while_barrier_held = accept.is_finished(); + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let mut accept = tokio::spawn(async move { + started_sender.send(()).unwrap(); + listener.accept().await + }); + started_receiver.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(250), &mut accept).await.is_err(), + "direct socket accepted while a concurrent process could inherit its descriptor" + ); release.send(()).unwrap(); holder.join().unwrap(); @@ -1765,10 +1770,6 @@ mod tests { .unwrap(); drop(server); drop(client); - assert!( - !completed_while_barrier_held, - "direct socket accepted while a concurrent process could inherit its descriptor" - ); } struct PreludeProbeLink { diff --git a/cmux-tui/crates/cmux-remote/src/workspace/process.rs b/cmux-tui/crates/cmux-remote/src/workspace/process.rs index cfb13f1580e7..101e0bd21a3d 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/process.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/process.rs @@ -2827,10 +2827,17 @@ mod tests { ); options.env.insert("PIDFILE".into(), pid_file.to_string_lossy().into_owned()); let spawn_manager = manager.clone(); - let spawn = tokio::spawn(async move { spawn_manager.spawn(root, options).await }); - - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - let spawned_while_barrier_held = pid_file.exists(); + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let mut spawn = tokio::spawn(async move { + started_sender.send(()).unwrap(); + spawn_manager.spawn(root, options).await + }); + started_receiver.await.unwrap(); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(250), &mut spawn).await.is_err(), + "remote pipe child started while the process barrier was held" + ); + assert!(!pid_file.exists(), "blocked remote pipe child published its pid"); release_sender.send(()).unwrap(); barrier_holder.join().unwrap(); @@ -2840,10 +2847,6 @@ mod tests { .unwrap() .unwrap(); manager.shutdown().await; - assert!( - !spawned_while_barrier_held, - "remote pipe child started while the process barrier was held" - ); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index 8ba941eae7d2..b530f305c231 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -20,10 +20,17 @@ const REAPER_MAX_BACKOFF: Duration = Duration::from_secs(1); static FORCE_REAPER_PENDING: AtomicBool = AtomicBool::new(false); #[cfg(test)] static FORCE_REAPER_WAIT_ERROR: AtomicBool = AtomicBool::new(false); -#[cfg(test)] -static REAPER_POLL_ATTEMPTS: AtomicUsize = AtomicUsize::new(0); #[cfg(all(test, unix))] static REAPER_TEST_LOCK: Mutex<()> = Mutex::new(()); +#[cfg(test)] +static REAPER_TEST_EVENT_SENDER: Mutex>> = Mutex::new(None); + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ReaperTestEvent { + Attempt(u32), + Complete(u32), +} #[cfg(test)] thread_local! { @@ -112,30 +119,24 @@ impl Chrome { } }); if let Err(error) = stderr_worker { - if kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)) { - if profile_ephemeral { - let _ = std::fs::remove_dir_all(&profile_dir); - } - } else { - reap_child_detached(reaper, child, profile_ephemeral.then(|| profile_dir.clone())); - } + let _ = reap_child_until( + reaper, + child, + profile_ephemeral.then(|| profile_dir.clone()), + Instant::now() + Duration::from_secs(1), + ); return Err(error.into()); } let web_socket_url = match rx.recv_timeout(Duration::from_secs(10)) { Ok(url) => url, Err(_) => { - if kill_child_until(&mut child, Instant::now() + Duration::from_secs(1)) { - if profile_ephemeral { - let _ = std::fs::remove_dir_all(&profile_dir); - } - } else { - reap_child_detached( - reaper, - child, - profile_ephemeral.then(|| profile_dir.clone()), - ); - } + let _ = reap_child_until( + reaper, + child, + profile_ephemeral.then(|| profile_dir.clone()), + Instant::now() + Duration::from_secs(1), + ); anyhow::bail!( "Chrome did not publish a DevTools endpoint within 10s (binary: {})", options.binary.display() @@ -162,25 +163,21 @@ impl Chrome { pub fn kill_until(&self, deadline: Instant) -> bool { let mut slot = self.child.lock().unwrap(); - let Some(mut child) = slot.take() else { return true }; + let Some(child) = slot.take() else { return true }; drop(slot); - if kill_child_until(&mut child, deadline) { - drop(self.reaper.lock().unwrap().take()); - return true; - } - *self.child.lock().unwrap() = Some(child); - false + let reaper = + self.reaper.lock().unwrap().take().expect("live Chrome retains its reaper lease"); + reap_child_until( + reaper, + child, + self.profile_ephemeral.then(|| self.profile_dir.clone()), + deadline, + ) } } impl Drop for Chrome { fn drop(&mut self) { - if self.kill_until(Instant::now() + Duration::from_secs(1)) { - if self.profile_ephemeral { - let _ = std::fs::remove_dir_all(&self.profile_dir); - } - return; - } let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); if let Some(child) = child { reap_child_detached( @@ -192,8 +189,6 @@ impl Drop for Chrome { child, self.profile_ephemeral.then(|| self.profile_dir.clone()), ); - } else if self.profile_ephemeral { - let _ = std::fs::remove_dir_all(&self.profile_dir); } } } @@ -201,7 +196,8 @@ impl Drop for Chrome { struct ReapRequest { child: Child, profile_dir: Option, - _lease: ChromeReaperLease, + completion: Option>, + lease: Option, next_attempt: Instant, retry_delay: Duration, } @@ -272,12 +268,38 @@ fn chrome_reaper_lease() -> std::io::Result { } fn reap_child_detached(reaper: ChromeReaperLease, child: Child, profile_dir: Option) { + reap_child(reaper, child, profile_dir, None); +} + +fn reap_child_until( + reaper: ChromeReaperLease, + child: Child, + profile_dir: Option, + deadline: Instant, +) -> bool { + let (completion_tx, completion_rx) = mpsc::sync_channel(1); + reap_child(reaper, child, profile_dir, Some(completion_tx)); + #[cfg(test)] + if FORCE_KILL_TIMEOUT.get() { + return false; + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false }; + completion_rx.recv_timeout(remaining).is_ok() +} + +fn reap_child( + reaper: ChromeReaperLease, + child: Child, + profile_dir: Option, + completion: Option>, +) { let sender = reaper.sender.clone(); sender .send(ReapRequest { child, profile_dir, - _lease: reaper, + completion, + lease: Some(reaper), next_attempt: Instant::now(), retry_delay: REAPER_INITIAL_BACKOFF, }) @@ -318,16 +340,21 @@ fn run_reaper(receiver: mpsc::Receiver) { return false; } request.next_attempt = now + request.retry_delay; - request.retry_delay = (request.retry_delay * 2).min(REAPER_MAX_BACKOFF); + request.retry_delay = next_reaper_retry_delay(request.retry_delay); true }); } } +fn next_reaper_retry_delay(current: Duration) -> Duration { + (current * 2).min(REAPER_MAX_BACKOFF) +} + fn poll_reap_request(request: &mut ReapRequest) -> bool { + #[cfg(test)] + publish_reaper_test_event(ReaperTestEvent::Attempt(request.child.id())); #[cfg(test)] { - REAPER_POLL_ATTEMPTS.fetch_add(1, Ordering::Relaxed); if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { let _ = request.child.kill(); return false; @@ -344,25 +371,58 @@ fn poll_reap_request(request: &mut ReapRequest) -> bool { if let Some(profile_dir) = request.profile_dir.take() { let _ = std::fs::remove_dir_all(profile_dir); } + drop(request.lease.take()); + if let Some(completion) = request.completion.take() { + let _ = completion.send(()); + } + #[cfg(test)] + publish_reaper_test_event(ReaperTestEvent::Complete(request.child.id())); true } -fn kill_child_until(child: &mut Child, deadline: Instant) -> bool { - let _ = child.kill(); - #[cfg(test)] - if FORCE_KILL_TIMEOUT.get() { - return false; +#[cfg(test)] +fn publish_reaper_test_event(event: ReaperTestEvent) { + if let Some(sender) = REAPER_TEST_EVENT_SENDER.lock().unwrap().as_ref() { + let _ = sender.send(event); + } +} + +#[cfg(test)] +struct ReaperTestObserver; + +#[cfg(test)] +impl ReaperTestObserver { + fn install() -> (Self, mpsc::Receiver) { + let (sender, receiver) = mpsc::channel(); + let previous = REAPER_TEST_EVENT_SENDER.lock().unwrap().replace(sender); + assert!(previous.is_none(), "Chrome reaper test observer was already installed"); + (Self, receiver) } +} + +#[cfg(test)] +impl Drop for ReaperTestObserver { + fn drop(&mut self) { + REAPER_TEST_EVENT_SENDER.lock().unwrap().take(); + } +} + +#[cfg(test)] +fn wait_for_reaper_test_event( + receiver: &mpsc::Receiver, + expected: ReaperTestEvent, +) { + let deadline = Instant::now() + Duration::from_secs(1); loop { - match child.try_wait() { - Ok(Some(_)) => return true, - Ok(None) => {} - Err(_) => return false, + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("Chrome reaper did not publish the expected lifecycle event"); + let event = receiver + .recv_timeout(remaining) + .expect("Chrome reaper did not publish the expected lifecycle event"); + if event == expected { + return; } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return false; - }; - std::thread::sleep(remaining.min(Duration::from_millis(10))); } } @@ -534,9 +594,10 @@ mod tests { .stderr(Stdio::null()) .spawn() .unwrap(); - let (sender, _receiver) = mpsc::channel(); + let (sender, receiver) = mpsc::channel(); + let worker = std::thread::spawn(move || run_reaper(receiver)); let active = Arc::new(AtomicUsize::new(0)); - let reaper = reserve_reaper_lease(sender, active.clone(), 1).unwrap(); + let reaper = reserve_reaper_lease(sender.clone(), active.clone(), 1).unwrap(); let chrome = Chrome { child: Mutex::new(Some(child)), profile_dir: make_profile_dir().unwrap(), @@ -551,12 +612,16 @@ mod tests { 0, "a confirmed child retained bounded reaper capacity until Chrome was dropped" ); + drop(chrome); + drop(sender); + worker.join().unwrap(); } #[cfg(unix)] #[test] fn drop_transfers_an_unconfirmed_child_to_a_reaper() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; @@ -579,18 +644,11 @@ mod tests { }; FORCE_KILL_TIMEOUT.set(true); - drop(chrome); + assert!(!chrome.kill_until(Instant::now() + Duration::from_secs(1))); FORCE_KILL_TIMEOUT.set(false); - let deadline = Instant::now() + Duration::from_secs(1); - let reaped = loop { - if unsafe { kill(pid, 0) } != 0 { - break true; - } - if Instant::now() >= deadline { - break false; - } - std::thread::sleep(Duration::from_millis(10)); - }; + drop(chrome); + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid as u32)); + let reaped = unsafe { kill(pid, 0) } != 0; if !reaped { let mut status = 0; unsafe { @@ -621,6 +679,7 @@ mod tests { #[test] fn final_reaper_request_survives_a_shutdown_thread_spawn_failure() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; @@ -643,16 +702,8 @@ mod tests { reap_child_detached(chrome_reaper_lease().unwrap(), child, Some(profile_dir.clone())); FORCE_REAPER_SPAWN_FAILURE.set(false); - let deadline = Instant::now() + Duration::from_secs(1); - let cleaned = loop { - if unsafe { kill(pid, 0) } != 0 && !profile_dir.exists() { - break true; - } - if Instant::now() >= deadline { - break false; - } - std::thread::sleep(Duration::from_millis(10)); - }; + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid as u32)); + let cleaned = unsafe { kill(pid, 0) } != 0 && !profile_dir.exists(); if !cleaned { let mut status = 0; unsafe { @@ -669,6 +720,7 @@ mod tests { #[test] fn persistent_reaper_backs_off_unconfirmed_children() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; @@ -683,16 +735,10 @@ mod tests { .unwrap(); let pid = i32::try_from(child.id()).unwrap(); FORCE_REAPER_PENDING.store(true, Ordering::Release); - REAPER_POLL_ATTEMPTS.store(0, Ordering::Release); reap_child_detached(chrome_reaper_lease().unwrap(), child, None); - std::thread::sleep(Duration::from_millis(160)); - let attempts = REAPER_POLL_ATTEMPTS.load(Ordering::Acquire); + wait_for_reaper_test_event(&events, ReaperTestEvent::Attempt(pid as u32)); FORCE_REAPER_PENDING.store(false, Ordering::Release); - - let deadline = Instant::now() + Duration::from_secs(1); - while unsafe { kill(pid, 0) } == 0 && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid as u32)); if unsafe { kill(pid, 0) } == 0 { let mut status = 0; unsafe { @@ -700,14 +746,15 @@ mod tests { waitpid(pid, &mut status, 0); } } - - assert!(attempts <= 6, "unconfirmed child was polled {attempts} times in 160ms"); + assert_eq!(next_reaper_retry_delay(REAPER_INITIAL_BACKOFF), Duration::from_millis(20)); + assert_eq!(next_reaper_retry_delay(REAPER_MAX_BACKOFF), REAPER_MAX_BACKOFF); } #[cfg(unix)] #[test] fn terminal_reaper_wait_error_retains_ownership_for_retry() { let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); unsafe extern "C" { fn kill(pid: i32, signal: i32) -> i32; fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; @@ -723,19 +770,11 @@ mod tests { let pid = i32::try_from(child.id()).unwrap(); let profile_dir = make_profile_dir().unwrap(); FORCE_REAPER_WAIT_ERROR.store(true, Ordering::Release); - REAPER_POLL_ATTEMPTS.store(0, Ordering::Release); reap_child_detached(chrome_reaper_lease().unwrap(), child, Some(profile_dir.clone())); - let deadline = Instant::now() + Duration::from_millis(200); - while REAPER_POLL_ATTEMPTS.load(Ordering::Acquire) == 0 && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + wait_for_reaper_test_event(&events, ReaperTestEvent::Attempt(pid as u32)); let retained = profile_dir.exists(); FORCE_REAPER_WAIT_ERROR.store(false, Ordering::Release); - - let cleanup_deadline = Instant::now() + Duration::from_secs(1); - while profile_dir.exists() && Instant::now() < cleanup_deadline { - std::thread::sleep(Duration::from_millis(10)); - } + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid as u32)); let mut status = 0; unsafe { kill(pid, 9); diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 6c50a8f25d21..e15652b5735c 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -46,9 +46,9 @@ const SCREENCAST_CLOCK_RECOVERY_BUDGET: Duration = Duration::from_secs(1); #[cfg(test)] static RETAINED_SIZE_CALLS: AtomicU64 = AtomicU64::new(0); #[cfg(test)] -static NEXT_RESOLVE_DELAY_MS: AtomicU64 = AtomicU64::new(0); +static NEXT_RESOLVE_GATE: Mutex>> = Mutex::new(None); #[cfg(test)] -static NEXT_RESOLVER_INIT_DELAY_MS: AtomicU64 = AtomicU64::new(0); +static NEXT_RESOLVER_INIT_GATE: Mutex>> = Mutex::new(None); #[cfg(test)] type TestHostResolverCommand = Arc Command + Send + Sync>; #[cfg(test)] @@ -56,7 +56,48 @@ static TEST_HOST_RESOLVER_COMMAND: Mutex> = Mute #[cfg(test)] static TEST_RESOLVER_REAPER_WAIT_ERROR: Mutex> = Mutex::new(None); #[cfg(test)] -static TEST_RESOLVER_REAPER_WAIT_CALLS: AtomicUsize = AtomicUsize::new(0); +static TEST_RESOLVER_REAPER_EVENT_SENDER: Mutex>> = + Mutex::new(None); + +#[cfg(test)] +#[derive(Default)] +struct ResolverTestGate { + state: Mutex<(bool, bool)>, + changed: Condvar, +} + +#[cfg(test)] +impl ResolverTestGate { + fn enter_and_wait(&self) { + let mut state = self.state.lock().unwrap(); + state.0 = true; + self.changed.notify_all(); + state = self.changed.wait_while(state, |state| !state.1).unwrap(); + drop(state); + } + + fn wait_until_entered(&self) { + let state = self.state.lock().unwrap(); + let (state, timeout) = self + .changed + .wait_timeout_while(state, Duration::from_secs(1), |state| !state.0) + .unwrap(); + assert!(state.0 && !timeout.timed_out(), "resolver did not enter its test gate"); + } + + fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.1 = true; + self.changed.notify_all(); + } +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ResolverReaperTestEvent { + Attempt(u32), + Complete(u32), +} static CDP_RESOLVER: std::sync::OnceLock>>> = std::sync::OnceLock::new(); @@ -80,7 +121,7 @@ struct ResolveRequest { deadline: Instant, response: SyncSender>>, #[cfg(test)] - delay: Duration, + gate: Option>, } struct ResolverChildReaper { @@ -113,16 +154,15 @@ impl CdpResolver { fn spawn() -> std::io::Result { let workers = Arc::new((Mutex::new(ResolverState::Initializing), Condvar::new())); let initializer_state = workers.clone(); + #[cfg(test)] + let initializer_gate = NEXT_RESOLVER_INIT_GATE.lock().unwrap().take(); let initializer = std::thread::Builder::new() .name("cmux-tui-cdp-resolver-init".into()) .spawn(move || { let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { #[cfg(test)] - { - let delay_ms = NEXT_RESOLVER_INIT_DELAY_MS.swap(0, Ordering::AcqRel); - if delay_ms != 0 { - std::thread::sleep(Duration::from_millis(delay_ms)); - } + if let Some(gate) = initializer_gate { + gate.enter_and_wait(); } spawn_host_resolver_workers().map_err(|error| error.to_string()) })) @@ -169,6 +209,21 @@ impl CdpResolver { let state = self.workers.0.lock().unwrap(); matches!(&*state, ResolverState::Failed(_)) } + + #[cfg(test)] + fn wait_until_initialized(&self) { + let (state, ready) = &*self.workers; + let state = state.lock().unwrap(); + let (state, timeout) = ready + .wait_timeout_while(state, Duration::from_secs(1), |state| { + matches!(state, ResolverState::Initializing) + }) + .unwrap(); + assert!( + !matches!(*state, ResolverState::Initializing) && !timeout.timed_out(), + "resolver initialization did not finish" + ); + } } fn spawn_host_resolver_workers() -> std::io::Result> { @@ -194,8 +249,8 @@ fn host_resolver_worker(receiver: Arc>>) { request }; #[cfg(test)] - if !request.delay.is_zero() { - std::thread::sleep(request.delay); + if let Some(gate) = request.gate.as_ref() { + gate.enter_and_wait(); } let result = if request.deadline <= Instant::now() { Err(std::io::Error::new( @@ -224,7 +279,7 @@ fn resolve_host_addresses_until( let reaper = reserve_resolver_child_reaper()?; let mut command = host_resolver_command(host, port)?; - command.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::null()); + command.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()); let mut child = cmux_tui_process::spawn_until(&mut command, deadline)?; let mut stdout = child .stdout @@ -257,67 +312,63 @@ fn resolve_host_addresses_until( handoff_resolver_child(child, reaper); return Err(error); } - loop { - match child.try_wait() { - Ok(Some(status)) => { - drop(reaper); - if !status.success() { - return Err(std::io::Error::other(format!( - "host resolver helper exited with {status}" - ))); - } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "CDP address resolution deadline expired", - )); - }; - let output = match output_receiver.recv_timeout(remaining) { - Ok(output) => output?, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "CDP address resolution deadline expired", - )); - } - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { - return Err(std::io::Error::other("host resolver output reader stopped")); - } - }; - if output.len() as u64 > CDP_RESOLVER_OUTPUT_LIMIT { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "host resolver helper output exceeded its limit", - )); - } - let addresses = String::from_utf8_lossy(&output) - .lines() - .filter_map(|line| line.parse::().ok()) - .filter(|address| address.port() == port) - .collect::>(); - if addresses.is_empty() { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - "host resolver helper returned no addresses", - )); - } - return Ok(addresses); - } - Ok(None) => {} - Err(error) => { - handoff_resolver_child(child, reaper); - return Err(error); - } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + handoff_resolver_child(child, reaper); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )); + }; + let output = match output_receiver.recv_timeout(remaining) { + Ok(Ok(output)) => output, + Ok(Err(error)) => { + handoff_resolver_child(child, reaper); + return Err(error); } - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { handoff_resolver_child(child, reaper); return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "CDP address resolution deadline expired", )); - }; - std::thread::sleep(remaining.min(CDP_RESOLVER_POLL_INTERVAL)); + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + handoff_resolver_child(child, reaper); + return Err(std::io::Error::other("host resolver output reader stopped")); + } + }; + if Instant::now() >= deadline { + handoff_resolver_child(child, reaper); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "CDP address resolution deadline expired", + )); + } + // The resolver helper owns its stdout for its full lifetime. EOF is its + // completion signal, so wait can reap without polling process state. + let status = child.wait()?; + drop(reaper); + if !status.success() { + return Err(std::io::Error::other(format!("host resolver helper exited with {status}"))); + } + if output.len() as u64 > CDP_RESOLVER_OUTPUT_LIMIT { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "host resolver helper output exceeded its limit", + )); } + let addresses = String::from_utf8_lossy(&output) + .lines() + .filter_map(|line| line.parse::().ok()) + .filter(|address| address.port() == port) + .collect::>(); + if addresses.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + "host resolver helper returned no addresses", + )); + } + Ok(addresses) } fn host_resolver_command(host: &str, port: u16) -> std::io::Result { @@ -430,31 +481,45 @@ fn run_resolver_child_reaper( pending.push(request); } let now = Instant::now(); + let mut completed = Vec::new(); pending.retain_mut(|request| { if request.next_attempt > now { return true; } let _ = request.child.kill(); match resolver_child_try_wait(request) { - Ok(Some(_)) => false, - Err(error) if resolver_child_wait_lost_ownership(&error) => false, + Ok(Some(_)) => { + completed.push(request.child.id()); + false + } + Err(error) if resolver_child_wait_lost_ownership(&error) => { + completed.push(request.child.id()); + false + } Ok(None) | Err(_) => { if now >= request.retry_deadline { degraded.store(true, Ordering::Release); } request.next_attempt = now + request.retry_delay; - request.retry_delay = - request.retry_delay.saturating_mul(2).min(CDP_RESOLVER_REAP_MAX_BACKOFF); + request.retry_delay = next_resolver_reap_delay(request.retry_delay); true } } }); + #[cfg(test)] + for child in completed { + publish_resolver_reaper_test_event(ResolverReaperTestEvent::Complete(child)); + } if pending.is_empty() { degraded.store(false, Ordering::Release); } } } +fn next_resolver_reap_delay(current: Duration) -> Duration { + current.saturating_mul(2).min(CDP_RESOLVER_REAP_MAX_BACKOFF) +} + fn resolver_child_wait_lost_ownership(error: &std::io::Error) -> bool { #[cfg(unix)] { @@ -472,7 +537,7 @@ fn resolver_child_try_wait( ) -> std::io::Result> { #[cfg(test)] { - TEST_RESOLVER_REAPER_WAIT_CALLS.fetch_add(1, Ordering::AcqRel); + publish_resolver_reaper_test_event(ResolverReaperTestEvent::Attempt(request.child.id())); if let Some(kind) = *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() { return Err(std::io::Error::from(kind)); } @@ -480,6 +545,52 @@ fn resolver_child_try_wait( request.child.try_wait() } +#[cfg(test)] +fn publish_resolver_reaper_test_event(event: ResolverReaperTestEvent) { + if let Some(sender) = TEST_RESOLVER_REAPER_EVENT_SENDER.lock().unwrap().as_ref() { + let _ = sender.send(event); + } +} + +#[cfg(test)] +struct ResolverReaperTestObserver; + +#[cfg(test)] +impl ResolverReaperTestObserver { + fn install() -> (Self, Receiver) { + let (sender, receiver) = channel(); + let previous = TEST_RESOLVER_REAPER_EVENT_SENDER.lock().unwrap().replace(sender); + assert!(previous.is_none(), "resolver reaper test observer was already installed"); + (Self, receiver) + } +} + +#[cfg(test)] +impl Drop for ResolverReaperTestObserver { + fn drop(&mut self) { + TEST_RESOLVER_REAPER_EVENT_SENDER.lock().unwrap().take(); + } +} + +#[cfg(test)] +fn wait_for_resolver_reaper_test_event( + receiver: &Receiver, + expected: ResolverReaperTestEvent, +) { + let deadline = Instant::now() + Duration::from_secs(1); + loop { + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("resolver reaper did not publish the expected lifecycle event"); + let event = receiver + .recv_timeout(remaining) + .expect("resolver reaper did not publish the expected lifecycle event"); + if event == expected { + return; + } + } +} + #[doc(hidden)] pub fn run_host_resolver_helper(args: &[String]) -> anyhow::Result<()> { let [host, port] = args else { @@ -1114,7 +1225,7 @@ fn resolve_socket_addr_until( deadline, response, #[cfg(test)] - delay: Duration::from_millis(NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel)), + gate: NEXT_RESOLVE_GATE.lock().unwrap().take(), })?; let remaining = remaining_until(deadline, "CDP address resolution deadline expired")?; let addresses = match receiver.recv_timeout(remaining) { @@ -2809,7 +2920,8 @@ mod tests { #[ignore] fn host_resolver_process_fixture() { if std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_MODE").as_deref() == Ok("blocked") { - thread::sleep(Duration::from_secs(30)); + let mut input = Vec::new(); + std::io::stdin().read_to_end(&mut input).unwrap(); return; } if std::env::var("CMUX_TUI_TEST_HOST_RESOLVER_MODE").as_deref() == Ok("oversized") { @@ -4208,6 +4320,7 @@ mod tests { fn websocket_handshake_uses_one_absolute_deadline() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); + let (release_sender, release_receiver) = channel(); let server = thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut request = Vec::new(); @@ -4218,12 +4331,8 @@ mod tests { } request.push(byte[0]); } - for byte in b"HTTP/1.1 400 Bad Request\r\n\r\n" { - if stream.write_all(&[*byte]).is_err() { - break; - } - thread::sleep(Duration::from_millis(10)); - } + stream.write_all(b"HTTP/1.1 400").unwrap(); + release_receiver.recv().unwrap(); }); let (events, _receiver) = sync_channel(1); @@ -4238,6 +4347,7 @@ mod tests { Err(error) => error, }; let elapsed = started.elapsed(); + release_sender.send(()).unwrap(); server.join().unwrap(); assert!(!error.to_string().is_empty()); @@ -4251,7 +4361,8 @@ mod tests { fn websocket_resolution_uses_the_connection_deadline() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); - NEXT_RESOLVE_DELAY_MS.store(300, Ordering::Release); + let gate = Arc::new(ResolverTestGate::default()); + *NEXT_RESOLVE_GATE.lock().unwrap() = Some(gate.clone()); let (events, _receiver) = sync_channel(1); let started = Instant::now(); @@ -4261,7 +4372,8 @@ mod tests { Duration::from_millis(50), ); let elapsed = started.elapsed(); - NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); + gate.wait_until_entered(); + gate.release(); assert!(result.is_err(), "invalid local endpoint unexpectedly connected"); assert!( @@ -4277,24 +4389,12 @@ mod tests { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let server = thread::spawn(move || { - listener.set_nonblocking(true).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - let mut connections = 0; - while connections < 2 && Instant::now() < deadline { - match listener.accept() { - Ok((stream, _)) => { - stream.set_nonblocking(false).unwrap(); - let mut ws = accept(stream).unwrap(); - ws.close(None).unwrap(); - connections += 1; - } - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(1)); - } - Err(error) => panic!("reconnect test listener failed: {error}"), - } + for _ in 0..2 { + let (stream, _) = listener.accept().unwrap(); + let mut ws = accept(stream).unwrap(); + ws.close(None).unwrap(); } - connections + 2 }); let command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); let (initial_events, _initial_receiver) = sync_channel(1); @@ -4310,10 +4410,12 @@ mod tests { thread::yield_now(); } - NEXT_RESOLVE_DELAY_MS.store(250, Ordering::Release); + let unused_gate = Arc::new(ResolverTestGate::default()); + *NEXT_RESOLVE_GATE.lock().unwrap() = Some(unused_gate.clone()); let (reconnect_events, _reconnect_receiver) = sync_channel(1); let reconnect = client.reconnect_with_timeout(reconnect_events, Duration::from_millis(100)); - let unused_delay = NEXT_RESOLVE_DELAY_MS.swap(0, Ordering::AcqRel); + let resolver_was_unused = NEXT_RESOLVE_GATE.lock().unwrap().take().is_some(); + unused_gate.release(); let reconnected = reconnect.is_ok(); drop(reconnect); @@ -4321,24 +4423,23 @@ mod tests { let connections = server.join().unwrap(); assert!(reconnected, "cached-peer reconnect did not complete"); assert_eq!(connections, 2, "cached-peer reconnect did not reach the server"); - assert_eq!(unused_delay, 250, "reconnect invoked the hostname resolver"); + assert!(resolver_was_unused, "reconnect invoked the hostname resolver"); } #[test] fn resolver_initialization_obeys_the_first_request_deadline() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); *CDP_RESOLVER.get_or_init(|| Mutex::new(None)).lock().unwrap() = None; - NEXT_RESOLVER_INIT_DELAY_MS.store(300, Ordering::Release); + let gate = Arc::new(ResolverTestGate::default()); + *NEXT_RESOLVER_INIT_GATE.lock().unwrap() = Some(gate.clone()); let started = Instant::now(); let result = resolve_socket_addr_until("localhost", 1, Instant::now() + Duration::from_millis(50)); let elapsed = started.elapsed(); - // A bounded initializer may still be completing after the caller's - // deadline. Let the one process-wide initialization finish before a - // later resolver test starts. - thread::sleep(Duration::from_millis(350)); - NEXT_RESOLVER_INIT_DELAY_MS.store(0, Ordering::Release); + gate.wait_until_entered(); + gate.release(); + CDP_RESOLVER.get().unwrap().lock().unwrap().as_ref().unwrap().wait_until_initialized(); assert!(result.is_err(), "expired first resolution unexpectedly completed"); assert!( @@ -4352,11 +4453,8 @@ mod tests { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); warm_resolver(); let _command = TestHostResolverCommandGuard::install(Arc::new(resolver_fixture_command)); - let slow_delay = scaled_test_duration(Duration::from_millis(300)); - NEXT_RESOLVE_DELAY_MS.store( - u64::try_from(slow_delay.as_millis()).expect("test delay exceeds u64 milliseconds"), - Ordering::Release, - ); + let gate = Arc::new(ResolverTestGate::default()); + *NEXT_RESOLVE_GATE.lock().unwrap() = Some(gate.clone()); let first = resolve_socket_addr_until( "cmux-expired-first.invalid", @@ -4371,10 +4469,8 @@ mod tests { ); let elapsed = started.elapsed(); - NEXT_RESOLVE_DELAY_MS.store(0, Ordering::Release); - // Let the deliberately expired request leave its worker before the - // process-wide resolver is shared with the next test. - thread::sleep(slow_delay); + gate.wait_until_entered(); + gate.release(); assert!(first.is_err(), "injected slow resolution unexpectedly completed"); assert!(second.is_ok(), "expired DNS work blocked the next request: {second:?}"); @@ -4426,11 +4522,6 @@ mod tests { let elapsed = started.elapsed(); *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; - let cleanup_deadline = Instant::now() + Duration::from_secs(1); - while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { - thread::sleep(Duration::from_millis(5)); - } - assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); assert!( elapsed < Duration::from_millis(500), @@ -4482,6 +4573,7 @@ mod tests { #[test] fn resolver_reaper_releases_capacity_after_child_ownership_is_lost() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ResolverReaperTestObserver::install(); resolver_child_reaper().unwrap(); let active = { let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); @@ -4502,10 +4594,7 @@ mod tests { assert_eq!(child.try_wait().unwrap_err().raw_os_error(), Some(libc::ECHILD)); handoff_resolver_child(child, lease); - let deadline = Instant::now() + Duration::from_millis(250); - while active.load(Ordering::Acquire) != baseline && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - } + wait_for_resolver_reaper_test_event(&events, ResolverReaperTestEvent::Complete(pid as u32)); assert_eq!( active.load(Ordering::Acquire), @@ -4518,6 +4607,7 @@ mod tests { #[test] fn resolver_reaper_backs_off_retryable_wait_errors() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ResolverReaperTestObserver::install(); resolver_child_reaper().unwrap(); let active = { let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); @@ -4528,30 +4618,29 @@ mod tests { let mut command = Command::new("/bin/sh"); command.args(["-c", "sleep 30"]); let child = cmux_tui_process::spawn(&mut command).unwrap(); + let pid = child.id(); - TEST_RESOLVER_REAPER_WAIT_CALLS.store(0, Ordering::Release); *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = Some(std::io::ErrorKind::Interrupted); handoff_resolver_child(child, lease); - let observation_deadline = Instant::now() + Duration::from_millis(250); - while TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire) < 12 - && Instant::now() < observation_deadline - { - thread::sleep(Duration::from_millis(2)); - } - let wait_calls = TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire); + wait_for_resolver_reaper_test_event(&events, ResolverReaperTestEvent::Attempt(pid)); *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = None; - let cleanup_deadline = Instant::now() + Duration::from_secs(1); - while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { - thread::sleep(Duration::from_millis(5)); - } + wait_for_resolver_reaper_test_event(&events, ResolverReaperTestEvent::Complete(pid)); assert_eq!(active.load(Ordering::Acquire), baseline); - assert!(wait_calls < 12, "retryable child wait errors were hot-polled {wait_calls} times"); + assert_eq!( + next_resolver_reap_delay(CDP_RESOLVER_POLL_INTERVAL), + CDP_RESOLVER_POLL_INTERVAL * 2 + ); + assert_eq!( + next_resolver_reap_delay(CDP_RESOLVER_REAP_MAX_BACKOFF), + CDP_RESOLVER_REAP_MAX_BACKOFF + ); } #[test] fn resolver_reaper_retains_capacity_after_inconclusive_wait_error() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ResolverReaperTestObserver::install(); resolver_child_reaper().unwrap(); let active = { let slot = CDP_RESOLVER_CHILD_REAPER.get().unwrap().lock().unwrap(); @@ -4562,24 +4651,16 @@ mod tests { let mut command = Command::new("/bin/sh"); command.args(["-c", "sleep 30"]); let child = cmux_tui_process::spawn(&mut command).unwrap(); + let pid = child.id(); - TEST_RESOLVER_REAPER_WAIT_CALLS.store(0, Ordering::Release); *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = Some(std::io::ErrorKind::PermissionDenied); handoff_resolver_child(child, lease); - let observation_deadline = Instant::now() + Duration::from_millis(250); - while TEST_RESOLVER_REAPER_WAIT_CALLS.load(Ordering::Acquire) == 0 - && Instant::now() < observation_deadline - { - thread::sleep(Duration::from_millis(2)); - } + wait_for_resolver_reaper_test_event(&events, ResolverReaperTestEvent::Attempt(pid)); let retained = active.load(Ordering::Acquire) == baseline + 1; *TEST_RESOLVER_REAPER_WAIT_ERROR.lock().unwrap() = None; - let cleanup_deadline = Instant::now() + Duration::from_secs(1); - while active.load(Ordering::Acquire) != baseline && Instant::now() < cleanup_deadline { - thread::sleep(Duration::from_millis(5)); - } + wait_for_resolver_reaper_test_event(&events, ResolverReaperTestEvent::Complete(pid)); assert_eq!(active.load(Ordering::Acquire), baseline); assert!(retained, "inconclusive child wait error discarded resolver ownership"); } @@ -4587,6 +4668,7 @@ mod tests { #[test] fn timed_out_host_lookups_do_not_strand_resolver_capacity() { let _guard = RESOLVE_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ResolverReaperTestObserver::install(); warm_resolver(); *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = Some(Arc::new(resolver_fixture_command)); @@ -4607,6 +4689,22 @@ mod tests { for caller in callers { assert!(caller.join().unwrap().is_err()); } + let mut completed = std::collections::HashSet::new(); + let deadline = Instant::now() + Duration::from_secs(1); + while completed.len() < CDP_RESOLVER_WORKERS { + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("resolver reaper did not release all timed-out helpers"); + match events + .recv_timeout(remaining) + .expect("resolver reaper did not release all timed-out helpers") + { + ResolverReaperTestEvent::Complete(pid) => { + completed.insert(pid); + } + ResolverReaperTestEvent::Attempt(_) => {} + } + } let next = resolve_socket_addr_until( "cmux-success.invalid", @@ -4615,7 +4713,6 @@ mod tests { ); *TEST_HOST_RESOLVER_COMMAND.lock().unwrap() = None; - thread::sleep(Duration::from_millis(100)); assert_eq!(next.unwrap(), SocketAddr::from(([127, 0, 0, 1], 9333))); } diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 039112fb57b8..bb83f239dea0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -593,6 +593,8 @@ pub struct BrowserRuntime { stealth_user_agent: Option, routes: Mutex, closed: AtomicBool, + closed_wait: Mutex<()>, + closed_changed: Condvar, } #[derive(Default)] @@ -814,6 +816,8 @@ impl BrowserRuntime { stealth_user_agent, routes: Mutex::new(Routes::default()), closed: AtomicBool::new(false), + closed_wait: Mutex::new(()), + closed_changed: Condvar::new(), }); start_router(Arc::downgrade(&runtime), event_rx)?; runtime.client.set_discover_targets(true)?; @@ -824,6 +828,22 @@ impl BrowserRuntime { self.closed.load(Ordering::Acquire) } + #[cfg(test)] + pub(crate) fn wait_until_closed(&self, deadline: Instant) -> bool { + let mut wait = self.closed_wait.lock().unwrap(); + while !self.is_closed() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.closed_changed.wait_timeout(wait, remaining).unwrap(); + wait = next; + if timeout.timed_out() && !self.is_closed() { + return false; + } + } + true + } + pub fn source(&self) -> BrowserSource { self.source } @@ -1491,6 +1511,7 @@ fn start_router(runtime: Weak, events: Receiver) -> an } fn close_browser_runtime(runtime: &BrowserRuntime, reason: String) { + let closed_wait = runtime.closed_wait.lock().unwrap(); let senders = { let mut routes = runtime.routes.lock().unwrap(); runtime.closed.store(true, Ordering::Release); @@ -1499,6 +1520,8 @@ fn close_browser_runtime(runtime: &BrowserRuntime, reason: String) { routes.by_target.clear(); senders }; + runtime.closed_changed.notify_all(); + drop(closed_wait); for tx in senders { tx.close(reason.clone()); } @@ -6785,23 +6808,8 @@ mod tests { ws.close(None).unwrap(); drop(ws); - listener.set_nonblocking(true).unwrap(); - let deadline = Instant::now() + Duration::from_secs(1); - let reconnect = loop { - match listener.accept() { - Ok((stream, _)) => break Some(stream), - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - if Instant::now() >= deadline { - break None; - } - thread::sleep(Duration::from_millis(5)); - } - Err(error) => panic!("failed to accept shutdown reconnect: {error}"), - } - }; - let Some(stream) = reconnect else { return false }; - stream.set_nonblocking(false).unwrap(); - let mut ws = accept(stream).unwrap(); + let (stream, _) = listener.accept().unwrap(); + let Ok(mut ws) = accept(stream) else { return false }; let query = read_ws_json(&mut ws); assert_eq!(query["method"], "Target.getTargets"); write_ws_json(&mut ws, json!({"id": query["id"], "result": {"targetInfos": []}})); @@ -6821,13 +6829,15 @@ mod tests { session_id: "session-1".to_string(), }); let owner = browser.shutdown_owner().unwrap(); - let deadline = Instant::now() + Duration::from_secs(1); - while !runtime.is_closed() && Instant::now() < deadline { - thread::yield_now(); - } - assert!(runtime.is_closed(), "initial CDP transport did not close"); + assert!( + runtime.wait_until_closed(Instant::now() + Duration::from_secs(1)), + "initial CDP transport did not close" + ); let terminated = owner.terminate_until(Instant::now() + Duration::from_millis(500)); + if !terminated { + let _ = TcpStream::connect(addr); + } let runtime_released = runtime.shutdown_until(Instant::now() + Duration::from_millis(100)); let reconnected = server.join().unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d3afd396cd09..ec88513cb9b0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -220,6 +220,27 @@ impl TerminalAdoptionQueueState { } } +#[cfg(all(test, unix))] +impl TerminalAdoptionCoordinator { + fn wait_until_idle(&self, tracked: &Mutex>, deadline: Instant) -> bool { + let mut state = self.state.lock().unwrap(); + loop { + if state.in_flight == 0 && tracked.lock().unwrap().is_empty() { + return true; + } + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.wake.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && (state.in_flight != 0 || !tracked.lock().unwrap().is_empty()) + { + return false; + } + } + } +} + #[cfg(unix)] #[derive(Default)] struct TerminalAdoptionCoordinator { @@ -418,6 +439,7 @@ struct ShutdownOwnerReconcilerState { stopping: bool, worker_started: bool, degraded: bool, + revision: u64, } #[derive(Default)] @@ -439,6 +461,54 @@ impl ShutdownOwnerReconciler { self.state.lock().unwrap().worker_started } + #[cfg(test)] + fn wait_until(&self, deadline: Instant, predicate: impl Fn() -> bool) -> bool { + let mut state = self.state.lock().unwrap(); + while !predicate() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let revision = state.revision; + let (next, timeout) = self + .wake + .wait_timeout_while(state, remaining, |state| state.revision == revision) + .unwrap(); + state = next; + if timeout.timed_out() && !predicate() { + return false; + } + } + true + } + + #[cfg(test)] + fn wait_until_degraded(&self, deadline: Instant) -> bool { + let mut state = self.state.lock().unwrap(); + while !state.degraded { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.wake.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && !state.degraded { + return false; + } + } + true + } + + fn publish_progress(&self) { + let mut state = self.state.lock().unwrap(); + state.revision = state.revision.wrapping_add(1); + self.wake.notify_all(); + } + + fn wait_for_retry(&self, delay: Duration, deadline: Instant) { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return }; + let state = self.state.lock().unwrap(); + let _ = self.wake.wait_timeout(state, delay.min(remaining)).unwrap(); + } + fn schedule(self: &Arc) { let mut state = self.state.lock().unwrap(); if state.stopping { @@ -497,6 +567,7 @@ impl ShutdownOwnerReconciler { Err(_) => true, }; drop(mux); + self.publish_progress(); if !pending { delay = SHUTDOWN_RECONCILE_INITIAL_DELAY; break; @@ -522,6 +593,8 @@ impl ShutdownOwnerReconciler { state.pending = false; state.worker_started = false; state.degraded = true; + state.revision = state.revision.wrapping_add(1); + self.wake.notify_all(); return; } @@ -1928,6 +2001,22 @@ impl AsyncSurfaceCreationGate { } true } + + #[cfg(test)] + fn wait_until_idle(&self, deadline: Instant) -> bool { + let mut state = self.inner.state.lock().unwrap(); + while state.active != 0 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.inner.idle.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.active != 0 { + return false; + } + } + true + } } struct AsyncSurfaceCreationGuard { @@ -2466,8 +2555,6 @@ pub struct Mux { #[cfg(all(test, unix))] terminal_host_record_loader: Mutex>, #[cfg(test)] - terminal_adoption_workers_started: AtomicUsize, - #[cfg(test)] browser_bootstrap_before_runtime: Mutex>>, #[cfg(test)] browser_runtime_connect: Mutex>>, @@ -2825,8 +2912,6 @@ impl Mux { #[cfg(all(test, unix))] terminal_host_record_loader: Mutex::new(None), #[cfg(test)] - terminal_adoption_workers_started: AtomicUsize::new(0), - #[cfg(test)] browser_bootstrap_before_runtime: Mutex::new(None), #[cfg(test)] browser_runtime_connect: Mutex::new(None), @@ -3620,6 +3705,10 @@ impl Mux { owner_reservation.release(); drop(state); self.emit_terminal_registry_changed(®istry, revision); + #[cfg(debug_assertions)] + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL") { + let _ = std::fs::write(signal, b"1"); + } Ok(()) } @@ -3650,10 +3739,6 @@ impl Mux { let worker_coordinator = coordinator.clone(); match std::thread::Builder::new().name(format!("terminal-adoption-{worker}")).spawn( move || { - #[cfg(test)] - if let Some(mux) = mux.upgrade() { - mux.terminal_adoption_workers_started.fetch_add(1, Ordering::AcqRel); - } Self::run_terminal_adoption_worker(mux, worker_coordinator); }, ) { @@ -8482,10 +8567,7 @@ impl Mux { } Err(error) => last_error = Some(error), } - let Some(remaining) = overall_deadline.checked_duration_since(Instant::now()) else { - break; - }; - std::thread::sleep(retry_delay.min(remaining)); + self.shutdown_owner_reconciler.wait_for_retry(retry_delay, overall_deadline); retry_delay = retry_delay.saturating_mul(2).min(SERVER_EXIT_RETRY_MAX_DELAY); } let error = @@ -15205,6 +15287,15 @@ fn bounded_shutdown_fanout( items: &[T], deadline: Instant, operation: impl Fn(&T, Instant) + Sync, +) -> usize { + bounded_shutdown_fanout_with_clock(items, deadline, Instant::now, operation) +} + +fn bounded_shutdown_fanout_with_clock( + items: &[T], + deadline: Instant, + now: impl Fn() -> Instant + Sync, + operation: impl Fn(&T, Instant) + Sync, ) -> usize { if items.is_empty() { return 0; @@ -15214,12 +15305,13 @@ fn bounded_shutdown_fanout( std::thread::scope(|scope| { for worker in 0..workers { let next = &next; + let now = &now; let operation = &operation; let spawned = std::thread::Builder::new() .name(format!("shutdown-owner-{worker}")) .spawn_scoped(scope, move || { loop { - if Instant::now() >= deadline { + if now() >= deadline { break; } let index = next.fetch_add(1, Ordering::Relaxed); @@ -20990,11 +21082,10 @@ mod tests { )) .unwrap(); server.join().unwrap(); - let deadline = Instant::now() + Duration::from_secs(1); - while !runtime.is_closed() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(runtime.is_closed(), "test CDP runtime did not observe disconnect"); + assert!( + runtime.wait_until_closed(Instant::now() + Duration::from_secs(1)), + "test CDP runtime did not observe disconnect" + ); let mux = test_mux(); let initial = mux.new_workspace(None, Some((80, 24))).unwrap(); @@ -21026,12 +21117,10 @@ mod tests { bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); mux.request_daemon_shutdown(); release_bootstrap_tx.send(()).unwrap(); - let deadline = Instant::now() + Duration::from_secs(1); - while mux.async_surface_creations.inner.state.lock().unwrap().active != 0 - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } + assert!( + mux.async_surface_creations.wait_until_idle(Instant::now() + Duration::from_secs(1)), + "async browser creation did not finish" + ); assert!(result.is_err(), "browser attached after its pane disappeared"); assert_eq!( @@ -23476,16 +23565,19 @@ mod tests { assert!(mux.close_surface(surface.id).unwrap()); assert_eq!(mux.shutdown_owners.len(), 1); - assert_eq!(attempts.load(Ordering::Acquire), 1); + assert_eq!(attempts.load(), 1); failing.store(false, Ordering::Release); - let deadline = Instant::now() + Duration::from_secs(2); - while !mux.shutdown_owners.is_empty() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + assert!( + mux.shutdown_owner_reconciler + .wait_until(Instant::now() + Duration::from_secs(2), || mux + .shutdown_owners + .is_empty(),), + "retained owner had no normal-lifecycle retry" + ); assert!(mux.shutdown_owners.is_empty(), "retained owner had no normal-lifecycle retry"); - assert!(attempts.load(Ordering::Acquire) >= 2); + assert!(attempts.load() >= 2); } #[test] @@ -23511,17 +23603,16 @@ mod tests { let (_failing, attempts) = owned.set_recovering_server_shutdown_for_test(); assert!(mux.close_surface(surface.id).unwrap()); - let first_deadline = Instant::now() + Duration::from_secs(1); - while attempts.load(Ordering::Acquire) < 4 && Instant::now() < first_deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let settled = attempts.load(Ordering::Acquire); + assert!( + mux.shutdown_owner_reconciler + .wait_until_degraded(Instant::now() + Duration::from_secs(1)), + "reconciler did not exhaust its retry budget" + ); + let settled = attempts.load(); assert!(settled >= 4, "reconciler did not exercise its retry budget"); - std::thread::sleep(Duration::from_millis(250)); - assert_eq!( - attempts.load(Ordering::Acquire), + attempts.load(), settled, "permanent cleanup failure kept an unbounded retry sweep alive" ); @@ -23562,13 +23653,13 @@ mod tests { })); assert!(mux.close_surface(first.id).unwrap()); - let deadline = Instant::now() + Duration::from_secs(2); - while (!injected.load(Ordering::Acquire) || !mux.shutdown_owners.is_empty()) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - let reconciled = injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty(); + let completed = mux + .shutdown_owner_reconciler + .wait_until(Instant::now() + Duration::from_secs(2), || { + injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty() + }); + let reconciled = + completed && injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty(); if !reconciled { mux.close_all_surfaces_for_shutdown().unwrap(); } @@ -23581,43 +23672,15 @@ mod tests { #[test] fn rejected_sidebar_admission_does_not_launch_a_process() { - let root = std::env::temp_dir() - .join(format!("cmux-sidebar-admission-{}", crate::workspace_registry::new_uuid_v4())); - std::fs::create_dir_all(&root).unwrap(); - let pid_path = root.join("sidebar.pid"); let mux = Mux::new("sidebar-admission", SurfaceOptions::default()); mux.set_shutdown_owner_capacity_for_test(0); - let options = SidebarPluginOptions { - command: vec![ - "/bin/sh".into(), - "-c".into(), - format!("echo $$ > {}; exec sleep 30", pid_path.display()), - ], - cwd: None, - }; + let options = + SidebarPluginOptions { command: vec!["/cmux-test-must-not-launch".into()], cwd: None }; let error = mux.spawn_sidebar_plugin_surface(&options, (80, 24)).unwrap_err(); - let deadline = Instant::now() + Duration::from_secs(1); - while !pid_path.exists() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let spawned_pid = std::fs::read_to_string(&pid_path) - .ok() - .and_then(|pid| pid.trim().parse::().ok()); - if let Some(pid) = spawned_pid { - // SAFETY: this PID was written by the test-owned sidebar command. - unsafe { - libc::kill(pid, libc::SIGKILL); - } - } drop(mux); - let _ = std::fs::remove_dir_all(root); assert_eq!(error.to_string(), "surface_owner_capacity_exhausted"); - assert!( - spawned_pid.is_none(), - "capacity rejection launched sidebar process {spawned_pid:?}" - ); } #[test] @@ -23649,18 +23712,10 @@ mod tests { #[cfg(unix)] #[test] fn rejected_terminal_admission_does_not_launch_a_process() { - let root = std::env::temp_dir() - .join(format!("cmux-terminal-admission-{}", crate::workspace_registry::new_uuid_v4())); - std::fs::create_dir_all(&root).unwrap(); - let pid_path = root.join("terminal.pid"); let mux = Mux::new( "terminal-admission", SurfaceOptions { - command: Some(vec![ - "/bin/sh".into(), - "-c".into(), - format!("echo $$ > {}; exec sleep 30", pid_path.display()), - ]), + command: Some(vec!["/cmux-test-must-not-launch".into()]), ..SurfaceOptions::default() }, ); @@ -23671,29 +23726,11 @@ mod tests { let mux = mux.clone(); move || mux.spawn_surface_with(None, None, Some((80, 24)), None, None) }); - let deadline = Instant::now() + Duration::from_secs(1); - while !pid_path.exists() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let spawned_pid = std::fs::read_to_string(&pid_path) - .ok() - .and_then(|pid| pid.trim().parse::().ok()); drop(state); let error = spawn.join().unwrap().unwrap_err(); - if let Some(pid) = spawned_pid { - // SAFETY: this PID was written by the test-owned terminal command. - unsafe { - libc::kill(pid, libc::SIGKILL); - } - } drop(mux); - let _ = std::fs::remove_dir_all(root); assert_eq!(error.to_string(), "surface_owner_capacity_exhausted"); - assert!( - spawned_pid.is_none(), - "capacity rejection launched terminal process {spawned_pid:?}" - ); } #[cfg(unix)] @@ -23825,21 +23862,13 @@ mod tests { ); } - let deadline = Instant::now() + crate::test_timeout(Duration::from_secs(1)); - while mux.terminal_adoption_workers_started.load(Ordering::Acquire) - < TERMINAL_ADOPTION_WORKERS - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - let workers = mux.terminal_adoption_workers_started.load(Ordering::Acquire); let workers_running = mux.terminal_adoption_coordinator.state.lock().unwrap().workers_running; mux.request_daemon_shutdown(); let _ = std::fs::remove_dir_all(root); assert_eq!( - workers, TERMINAL_ADOPTION_WORKERS, + workers_running, TERMINAL_ADOPTION_WORKERS, "adoption did not preflight its fixed worker pool" ); assert!( @@ -24232,10 +24261,11 @@ mod tests { let second_progressed = second_started_rx.recv_timeout(Duration::from_millis(300)).is_ok(); release_tx.send(()).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - while !mux.terminal_adoptions.lock().unwrap().is_empty() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + assert!( + mux.terminal_adoption_coordinator + .wait_until_idle(&mux.terminal_adoptions, Instant::now() + Duration::from_secs(2),), + "terminal adoptions did not finish" + ); mux.shutdown().unwrap(); let _ = std::fs::remove_dir_all(root); @@ -24321,16 +24351,13 @@ mod tests { attached_rx.recv_timeout(Duration::from_secs(2)).unwrap(); mux.request_daemon_shutdown(); release_tx.send(()).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - while !mux.terminal_adoptions.lock().unwrap().is_empty() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let attempts = kill_attempts - .lock() - .unwrap() - .clone() - .expect("adoption surface was not created") - .load(Ordering::Acquire); + assert!( + mux.terminal_adoption_coordinator + .wait_until_idle(&mux.terminal_adoptions, Instant::now() + Duration::from_secs(2),), + "terminal adoption did not finish" + ); + let attempts = + kill_attempts.lock().unwrap().clone().expect("adoption surface was not created").load(); assert_eq!( attempts, 0, @@ -24617,20 +24644,29 @@ mod tests { let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let surface_id = surface.id; let owned = mux.surface(surface.id).unwrap(); - owned.set_server_shutdown_delay_for_test(Duration::from_millis(150)); + let retirement_gate = Arc::new(crate::surface::TestLifecycleGate::default()); + owned.set_server_shutdown_gate_for_test(retirement_gate.clone()); let close = std::thread::spawn({ let mux = mux.clone(); move || mux.close_surface(surface_id).unwrap() }); - let removal_deadline = Instant::now() + Duration::from_secs(1); - while mux.surface(surface_id).is_some() && Instant::now() < removal_deadline { - std::thread::yield_now(); - } + assert!(retirement_gate.wait_until_entered(1, Instant::now() + Duration::from_secs(1))); assert!(mux.surface(surface_id).is_none(), "ordinary close did not remove its surface"); - let shutdown = mux.close_all_surfaces_for_shutdown(); + let (shutdown_tx, shutdown_rx) = std::sync::mpsc::sync_channel(1); + let shutdown_worker = std::thread::spawn({ + let mux = mux.clone(); + move || shutdown_tx.send(mux.close_all_surfaces_for_shutdown()).unwrap() + }); + assert!( + shutdown_rx.recv_timeout(Duration::from_millis(100)).is_err(), + "server shutdown passed an in-flight retirement owner" + ); + retirement_gate.release(); + let shutdown = shutdown_rx.recv_timeout(Duration::from_secs(1)).unwrap(); assert!(close.join().unwrap()); + shutdown_worker.join().unwrap(); owned.set_server_shutdown_failure_for_test(false); mux.close_all_surfaces_for_shutdown().unwrap(); assert!(mux.shutdown_owners.is_empty()); @@ -24702,18 +24738,26 @@ mod tests { .map(|surface| state.surfaces[surface].clone()) .collect::>() }); + let shutdown_gate = Arc::new(crate::surface::TestLifecycleGate::default()); for surface in &surfaces { - surface.set_server_shutdown_delay_for_test(Duration::from_millis(50)); + surface.set_server_shutdown_gate_for_test(shutdown_gate.clone()); } - let started = Instant::now(); - assert!(mux.close_pane(pane).unwrap()); - + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let close = std::thread::spawn({ + let mux = mux.clone(); + move || result_tx.send(mux.close_pane(pane)).unwrap() + }); assert!( - started.elapsed() < Duration::from_millis(500), - "bulk close multiplied the per-surface shutdown timeout: {:?}", - started.elapsed() + shutdown_gate.wait_until_entered( + SHUTDOWN_FANOUT_WORKERS, + Instant::now() + Duration::from_secs(1) + ), + "bulk close did not use the bounded parallel shutdown fanout" ); + shutdown_gate.release(); + assert!(result_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap()); + close.join().unwrap(); } #[cfg(unix)] @@ -24951,11 +24995,10 @@ mod tests { )) .unwrap(); server.join().unwrap(); - let disconnect_deadline = Instant::now() + Duration::from_secs(1); - while !runtime.is_closed() && Instant::now() < disconnect_deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(runtime.is_closed(), "test browser runtime did not observe disconnect"); + assert!( + runtime.wait_until_closed(Instant::now() + Duration::from_secs(1)), + "test browser runtime did not observe disconnect" + ); let mux = test_mux(); let options = mux.surface_options.lock().unwrap().clone(); let surface = browser::new_surface( @@ -25087,10 +25130,10 @@ mod tests { shutdown_done_tx.send(()).unwrap(); } }); - let deadline = Instant::now() + Duration::from_secs(1); - while attempts.load(Ordering::Acquire) == 0 && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(5)); - } + assert!( + attempts.wait_until(1, Instant::now() + Duration::from_secs(1)), + "daemon shutdown did not attempt local process termination" + ); let retained = mux.shutdown_owners.len(); failing.store(false, Ordering::Release); @@ -25152,15 +25195,10 @@ mod tests { let slot_available = mux.browser_runtime.lock_available_for_test(); release_connect_tx.send(()).unwrap(); - let deadline = Instant::now() + Duration::from_secs(1); - loop { - if mux.async_surface_creations.inner.state.lock().unwrap().active == 0 - || Instant::now() >= deadline - { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } + assert!( + mux.async_surface_creations.wait_until_idle(Instant::now() + Duration::from_secs(1)), + "browser surface creation did not finish" + ); mux.request_daemon_shutdown(); mux.shutdown().unwrap(); @@ -25229,25 +25267,39 @@ mod tests { #[test] fn shutdown_fanout_does_not_claim_another_batch_after_the_deadline() { let gate = Arc::new((Mutex::new(false), Condvar::new())); + let expired = Arc::new(AtomicBool::new(false)); let (started_tx, started_rx) = std::sync::mpsc::sync_channel(SHUTDOWN_FANOUT_WORKERS + 1); let deadline = Instant::now() + Duration::from_millis(250); let worker = std::thread::spawn({ let gate = gate.clone(); + let worker_expired = expired.clone(); move || { - bounded_shutdown_fanout(&[(); SHUTDOWN_FANOUT_WORKERS + 1], deadline, |_, _| { - started_tx.send(()).unwrap(); - let (released, wake) = &*gate; - let released = - wake.wait_while(released.lock().unwrap(), |released| !*released).unwrap(); - drop(released); - }); + bounded_shutdown_fanout_with_clock( + &[(); SHUTDOWN_FANOUT_WORKERS + 1], + deadline, + || { + if worker_expired.load(Ordering::Acquire) { + deadline + } else { + deadline - Duration::from_millis(1) + } + }, + |_, _| { + started_tx.send(()).unwrap(); + let (released, wake) = &*gate; + let released = wake + .wait_while(released.lock().unwrap(), |released| !*released) + .unwrap(); + drop(released); + }, + ); } }); for _ in 0..SHUTDOWN_FANOUT_WORKERS { started_rx.recv_timeout(Duration::from_millis(200)).unwrap(); } - std::thread::sleep(deadline.saturating_duration_since(Instant::now())); + expired.store(true, Ordering::Release); let (released, wake) = &*gate; *released.lock().unwrap() = true; wake.notify_all(); diff --git a/cmux-tui/crates/cmux-tui-core/src/platform.rs b/cmux-tui/crates/cmux-tui-core/src/platform.rs index 19b1e94b6c53..9ba40bcd97fa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/platform.rs +++ b/cmux-tui/crates/cmux-tui-core/src/platform.rs @@ -829,6 +829,7 @@ pub mod transport { #[cfg(target_os = "macos")] #[test] fn deadline_connect_excludes_concurrent_browser_process_creation() { + use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::PermissionsExt as _; use std::sync::Arc; use std::sync::mpsc; @@ -847,6 +848,10 @@ pub mod transport { let marker_path = root.join("descriptor-state"); let browser_path = root.join("browser"); let profile_path = root.join("profile"); + let hold_path = root.join("browser-hold"); + let hold_path_c = std::ffi::CString::new(hold_path.as_os_str().as_bytes()).unwrap(); + // SAFETY: hold_path_c is a valid, NUL-terminated path in the test directory. + assert_eq!(unsafe { libc::mkfifo(hold_path_c.as_ptr(), 0o600) }, 0); let (descriptor_sender, descriptor_receiver) = mpsc::sync_channel(1); let (release_sender, release_receiver) = mpsc::sync_channel(1); let release_receiver = Arc::new(std::sync::Mutex::new(release_receiver)); @@ -886,9 +891,10 @@ pub mod transport { printf closed > {}\n\ fi\n\ printf 'DevTools listening on ws://127.0.0.1:1/devtools/browser/test\\n' >&2\n\ - exec /bin/sleep 30\n", + exec /bin/cat < {}\n", marker_path.display(), - marker_path.display() + marker_path.display(), + hold_path.display() ), ) .unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 4d2e3cc5dced..154ea1b2919e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -62,6 +62,20 @@ pub(crate) fn reserved_child_reaper_active_for_test() -> usize { slot.as_ref().map_or(0, |reaper| reaper.activity.active.load(Ordering::Acquire)) } +#[cfg(test)] +pub(crate) fn wait_for_reserved_child_reaper_active_for_test( + expected: usize, + deadline: Instant, +) -> bool { + let Some(slot) = NATURAL_REAPER.get() else { return expected == 0 }; + let activity = { + let slot = slot.lock().unwrap(); + let Some(reaper) = slot.as_ref() else { return expected == 0 }; + reaper.activity.clone() + }; + activity.wait_for_active(expected, deadline) +} + #[cfg(test)] pub(crate) fn set_process_session_preflight_failure_for_test(enabled: bool) { FORCE_PROCESS_SESSION_PREFLIGHT_FAILURE.set(enabled); @@ -85,24 +99,51 @@ impl Drop for ForcedPostSpawnFailure { pub(crate) fn force_post_spawn_failure_for_test( marker: std::path::PathBuf, ) -> ForcedPostSpawnFailure { + use std::os::unix::ffi::OsStrExt as _; + + let marker_name = std::ffi::CString::new(marker.as_os_str().as_bytes()) + .expect("post-spawn failure signal path contains a null byte"); + // SAFETY: marker_name is a valid, null-terminated path owned for this call. + let result = unsafe { libc::mkfifo(marker_name.as_ptr(), 0o600) }; + assert_eq!( + result, + 0, + "failed to create post-spawn failure signal FIFO: {}", + io::Error::last_os_error() + ); let previous = POST_SPAWN_FAILURE_MARKER.replace(Some(marker)); ForcedPostSpawnFailure { previous } } #[cfg(test)] pub(crate) fn fail_after_pty_spawn_for_test() -> io::Result<()> { + use std::io::Read as _; + use std::os::fd::AsRawFd as _; + let marker = POST_SPAWN_FAILURE_MARKER.with_borrow(Clone::clone); let Some(marker) = marker else { return Ok(()) }; + let mut signal = std::fs::OpenOptions::new().read(true).write(true).open(marker)?; let deadline = Instant::now() + Duration::from_secs(2); - let marker_is_ready = || marker.metadata().is_ok_and(|metadata| metadata.len() > 0); - while !marker_is_ready() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(5)); - } - if !marker_is_ready() { - return Err(io::Error::new( - io::ErrorKind::TimedOut, - "forced post-spawn failure marker was not fully published", - )); + loop { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "forced post-spawn failure signal was not published", + )); + }; + let timeout = remaining.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: signal.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for this call. + let ready = unsafe { libc::poll(&raw mut descriptor, 1, timeout) }; + if ready > 0 && descriptor.revents & libc::POLLIN != 0 { + let mut byte = [0_u8; 1]; + signal.read_exact(&mut byte)?; + break; + } + if ready < 0 && io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + return Err(io::Error::last_os_error()); + } } Err(io::Error::other("forced post-spawn PTY initialization failure")) } @@ -190,14 +231,14 @@ impl NaturalReaperActivity { fn release(&self, owner_attached: bool) { self.active.fetch_sub(1, Ordering::AcqRel); + let mut state = self.state.lock().unwrap(); if !owner_attached { - let mut state = self.state.lock().unwrap(); state.unpublished = state .unpublished .checked_sub(1) .expect("unpublished PTY child reaper reservation is balanced"); - self.changed.notify_all(); } + self.changed.notify_all(); } fn set_degraded(&self, owner_attached: bool, degraded: bool) { @@ -243,6 +284,22 @@ impl NaturalReaperActivity { } Ok(()) } + + #[cfg(test)] + fn wait_for_active(&self, expected: usize, deadline: Instant) -> bool { + let mut state = self.state.lock().unwrap(); + while self.active.load(Ordering::Acquire) != expected { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.changed.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && self.active.load(Ordering::Acquire) != expected { + return false; + } + } + true + } } fn unpublished_child_reaper_deadline_error(state: &NaturalReaperActivityState) -> io::Error { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 43b8787fc55a..fb3b206bb194 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4283,6 +4283,9 @@ pub fn serve_websocket( continue; } }; + if stream.set_nonblocking(false).is_err() { + continue; + } #[cfg(all(test, target_os = "macos"))] thread_accepted_stream_mode.store( if tcp_stream_is_nonblocking_for_test(&stream) { 2 } else { 1 }, @@ -10701,52 +10704,56 @@ mod tests { const DESCRIPTOR_ENV: &str = "CMUX_TUI_TEST_PUBLICATION_GUARD_FD"; const TEST_NAME: &str = "server::tests::publication_guard_transfers_across_exec"; - fn wait_for_marker(child: &mut std::process::Child, marker: &Path) -> Result<(), String> { - let deadline = Instant::now() + Duration::from_secs(5); - loop { - if marker.exists() { - return Ok(()); - } - if let Some(status) = child.try_wait().map_err(|error| error.to_string())? { - return Err(format!( - "publication child exited before {}: {status}", - marker.display() - )); - } - if Instant::now() >= deadline { - return Err(format!("publication child did not create {}", marker.display())); - } - std::thread::sleep(Duration::from_millis(10)); + struct TestSignal(std::fs::File); + + impl TestSignal { + fn open(path: &Path) -> Self { + Self(std::fs::OpenOptions::new().read(true).write(true).open(path).unwrap()) } - } - fn wait_for_child_exit( - child: &mut std::process::Child, - ) -> Result { - let deadline = Instant::now() + Duration::from_secs(5); - loop { - match child.try_wait() { - Ok(Some(status)) => return Ok(status), - Ok(None) if Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(10)); - } - Ok(None) => { - let _ = child.kill(); - let _ = child.wait(); - return Err("publication child did not exit".to_string()); - } - Err(error) => { - let _ = child.kill(); - let _ = child.wait(); - return Err(error.to_string()); - } + fn create(path: &Path) -> Self { + use std::os::unix::ffi::OsStrExt as _; + + let path_name = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: path_name is a valid, null-terminated path owned for this call. + assert_eq!(unsafe { libc::mkfifo(path_name.as_ptr(), 0o600) }, 0); + Self::open(path) + } + + fn send(&mut self) { + use std::io::Write as _; + + self.0.write_all(b"ready").unwrap(); + } + + fn receive_until(&mut self, deadline: Instant) -> Result<(), String> { + use std::io::Read as _; + use std::os::fd::AsRawFd as _; + + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(|| "publication signal deadline expired".to_string())?; + let timeout = remaining.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: self.0.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for this call. + let ready = unsafe { libc::poll(&raw mut descriptor, 1, timeout) }; + if ready != 1 || descriptor.revents & libc::POLLIN == 0 { + return Err("publication signal was not received before its deadline".into()); } + let mut byte = [0_u8; 1]; + self.0.read_exact(&mut byte).map_err(|error| error.to_string()) } } if let Some(directory) = std::env::var_os(CHILD_ENV) { let directory = PathBuf::from(directory); let path = directory.join("server.sock"); + let mut ready = TestSignal::open(&directory.join("ready")); + let mut release = TestSignal::open(&directory.join("release")); + let mut dropped = TestSignal::open(&directory.join("dropped")); + let mut finish = TestSignal::open(&directory.join("finish")); + let mut exiting = TestSignal::open(&directory.join("exiting")); let descriptor = std::env::var(DESCRIPTOR_ENV).unwrap().parse::().unwrap(); // SAFETY: the parent passed the unique descriptor copy prepared @@ -10759,19 +10766,12 @@ mod tests { ) } .unwrap(); - std::fs::write(directory.join("ready"), []).unwrap(); - let deadline = Instant::now() + Duration::from_secs(5); - while !directory.join("release").exists() { - assert!(Instant::now() < deadline, "parent did not release publication child"); - std::thread::sleep(Duration::from_millis(10)); - } + ready.send(); + release.receive_until(Instant::now() + Duration::from_secs(5)).unwrap(); drop(guard); - std::fs::write(directory.join("dropped"), []).unwrap(); - let deadline = Instant::now() + Duration::from_secs(5); - while !directory.join("finish").exists() { - assert!(Instant::now() < deadline, "parent did not finish publication child"); - std::thread::sleep(Duration::from_millis(10)); - } + dropped.send(); + finish.receive_until(Instant::now() + Duration::from_secs(5)).unwrap(); + exiting.send(); return; } @@ -10780,6 +10780,11 @@ mod tests { .join(format!("cmux-publication-transfer-{}-{sequence:x}", std::process::id())); std::fs::create_dir(&directory).unwrap(); let path = directory.join("server.sock"); + let mut ready = TestSignal::create(&directory.join("ready")); + let mut release = TestSignal::create(&directory.join("release")); + let mut dropped = TestSignal::create(&directory.join("dropped")); + let mut finish = TestSignal::create(&directory.join("finish")); + let mut exiting = TestSignal::create(&directory.join("exiting")); let guard = PublicationGuard::acquire(&path, Instant::now() + Duration::from_secs(1)).unwrap(); let mut command = std::process::Command::new(std::env::current_exe().unwrap()); @@ -10793,7 +10798,7 @@ mod tests { .stderr(std::process::Stdio::null()); let mut child = cmux_tui_process::spawn(&mut command).unwrap(); - let ready = wait_for_marker(&mut child, &directory.join("ready")); + let ready_result = ready.receive_until(Instant::now() + Duration::from_secs(5)); drop(guard); let competing = PublicationGuard::acquire(&path, Instant::now() + Duration::from_millis(150)); @@ -10801,22 +10806,24 @@ mod tests { matches!(&competing, Err(error) if error.kind() == std::io::ErrorKind::TimedOut); drop(competing); - std::fs::write(directory.join("release"), []).unwrap(); - let dropped = wait_for_marker(&mut child, &directory.join("dropped")); + release.send(); + let dropped_result = dropped.receive_until(Instant::now() + Duration::from_secs(5)); let reacquired = PublicationGuard::acquire(&path, Instant::now() + Duration::from_secs(1)); let child_released_lock = reacquired.is_ok(); drop(reacquired); - std::fs::write(directory.join("finish"), []).unwrap(); - let status = wait_for_child_exit(&mut child); + finish.send(); + let exiting_result = exiting.receive_until(Instant::now() + Duration::from_secs(5)); + let status = child.wait(); - for marker in ["ready", "release", "dropped", "finish"] { + for marker in ["ready", "release", "dropped", "finish", "exiting"] { let _ = std::fs::remove_file(directory.join(marker)); } let _ = std::fs::remove_file(crate::publication::publication_lock_path(&path).unwrap()); std::fs::remove_dir(&directory).unwrap(); - ready.unwrap(); - dropped.unwrap(); + ready_result.unwrap(); + dropped_result.unwrap(); + exiting_result.unwrap(); let status = status.unwrap(); assert!(status.success(), "publication child failed: {status}"); assert!(child_retained_lock, "parent release dropped the child's publication lock"); diff --git a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs index 14aea8e721c4..f4ee734f78fc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs +++ b/cmux-tui/crates/cmux-tui-core/src/spawned_pty_child.rs @@ -33,11 +33,29 @@ impl Drop for ReservedPortableChildReaperLease { struct PortableChildReapRequest { child: Box, - _lease: ReservedPortableChildReaperLease, + lease: Option, next_attempt: Instant, retry_delay: Duration, } +#[cfg(test)] +static PORTABLE_REAPER_TEST_SENDER: Mutex>> = + Mutex::new(None); + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PortableReaperTestEvent { + Attempt, + Complete, +} + +#[cfg(test)] +fn publish_portable_reaper_test_event(event: PortableReaperTestEvent) { + if let Some(sender) = PORTABLE_REAPER_TEST_SENDER.lock().unwrap().as_ref() { + let _ = sender.send(event); + } +} + struct StrandedPortableChild { _child: Box, _lease: Option, @@ -100,12 +118,12 @@ fn enqueue_portable_child_cleanup( let sender = lease.sender.clone(); let request = PortableChildReapRequest { child, - _lease: lease, + lease: Some(lease), next_attempt: Instant::now(), retry_delay: PORTABLE_CHILD_REAP_RETRY_INITIAL, }; if let Err(mpsc::SendError(request)) = sender.send(request) { - retain_stranded_portable_child(request.child, Some(request._lease)); + retain_stranded_portable_child(request.child, request.lease); } } @@ -160,8 +178,14 @@ fn run_portable_child_reaper(receiver: mpsc::Receiver) } let mut request = pending.swap_remove(index); let _ = request.child.kill(); + #[cfg(test)] + publish_portable_reaper_test_event(PortableReaperTestEvent::Attempt); match request.child.try_wait() { - Ok(Some(_)) => {} + Ok(Some(_)) => { + drop(request.lease.take()); + #[cfg(test)] + publish_portable_reaper_test_event(PortableReaperTestEvent::Complete); + } Ok(None) | Err(_) => { request.schedule_retry(); pending.push(request); @@ -336,8 +360,25 @@ impl Drop for SpawnedPtyChild { mod tests { use super::*; use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - use std::time::{Duration, Instant}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + + struct PortableReaperTestObserver; + + impl PortableReaperTestObserver { + fn install() -> (Self, mpsc::Receiver) { + let (sender, receiver) = mpsc::channel(); + let previous = PORTABLE_REAPER_TEST_SENDER.lock().unwrap().replace(sender); + assert!(previous.is_none(), "portable reaper observer was already installed"); + (Self, receiver) + } + } + + impl Drop for PortableReaperTestObserver { + fn drop(&mut self) { + PORTABLE_REAPER_TEST_SENDER.lock().unwrap().take(); + } + } #[derive(Debug)] struct FailedKiller; @@ -357,7 +398,7 @@ mod tests { #[derive(Debug)] struct BlockingWaitChild { - try_waits: Arc, + try_waits: usize, wait_called: Arc, } @@ -376,8 +417,8 @@ mod tests { impl portable_pty::Child for BlockingWaitChild { fn try_wait(&mut self) -> std::io::Result> { - let attempt = self.try_waits.fetch_add(1, Ordering::AcqRel) + 1; - Ok((attempt >= 2).then(|| portable_pty::ExitStatus::with_exit_code(1))) + self.try_waits = self.try_waits.saturating_add(1); + Ok((self.try_waits >= 2).then(|| portable_pty::ExitStatus::with_exit_code(1))) } fn wait(&mut self) -> std::io::Result { @@ -412,10 +453,10 @@ mod tests { return; } - let try_waits = Arc::new(AtomicUsize::new(0)); + let (_observer, events) = PortableReaperTestObserver::install(); let wait_called = Arc::new(AtomicBool::new(false)); let child = SpawnedPtyChild::new(Box::new(BlockingWaitChild { - try_waits: try_waits.clone(), + try_waits: 0, wait_called: wait_called.clone(), })); let (dropped_sender, dropped_receiver) = mpsc::sync_channel(1); @@ -427,14 +468,18 @@ mod tests { dropped_receiver .recv_timeout(Duration::from_millis(250)) .expect("spawned PTY child Drop blocked after kill failed"); - let deadline = Instant::now() + Duration::from_millis(500); - while (try_waits.load(Ordering::Acquire) < 2 - || portable_child_reaper_active_for_test() != 0) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(5)); - } - assert!(try_waits.load(Ordering::Acquire) >= 2, "cleanup owner did not retry the child"); + assert_eq!( + events.recv_timeout(Duration::from_millis(500)).unwrap(), + PortableReaperTestEvent::Attempt + ); + assert_eq!( + events.recv_timeout(Duration::from_secs(1)).unwrap(), + PortableReaperTestEvent::Attempt + ); + assert_eq!( + events.recv_timeout(Duration::from_secs(1)).unwrap(), + PortableReaperTestEvent::Complete + ); assert_eq!( portable_child_reaper_active_for_test(), 0, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 70d3a6a2bb88..6dc51bff3476 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1337,9 +1337,7 @@ struct LocalPtyProcess { #[cfg(all(unix, test))] normal_cleanup_attempts: AtomicUsize, #[cfg(all(unix, test))] - normal_cleanup_delay_ms: AtomicU64, - #[cfg(all(unix, test))] - normal_cleanup_started: AtomicBool, + normal_cleanup_gate: Mutex>>, } #[cfg(unix)] @@ -1449,9 +1447,7 @@ impl LocalPtyProcess { #[cfg(all(unix, test))] normal_cleanup_attempts: AtomicUsize::new(0), #[cfg(all(unix, test))] - normal_cleanup_delay_ms: AtomicU64::new(0), - #[cfg(all(unix, test))] - normal_cleanup_started: AtomicBool::new(false), + normal_cleanup_gate: Mutex::new(None), }) } @@ -1703,6 +1699,34 @@ impl LocalPtyProcess { .unwrap(); } + #[cfg(all(unix, test))] + fn wait_for_child_reaped_until(&self, deadline: Instant) -> bool { + let state = self.exited.0.lock().unwrap(); + let (state, _) = self + .exited + .1 + .wait_timeout_while(state, deadline.saturating_duration_since(Instant::now()), |_| { + !self.child_reaped.load(Ordering::Acquire) + }) + .unwrap(); + drop(state); + self.child_reaped.load(Ordering::Acquire) + } + + #[cfg(all(unix, test))] + fn wait_for_normal_cleanup_attempts_until(&self, expected: usize, deadline: Instant) -> bool { + let state = self.exited.0.lock().unwrap(); + let (state, _) = self + .exited + .1 + .wait_timeout_while(state, deadline.saturating_duration_since(Instant::now()), |_| { + self.normal_cleanup_attempts.load(Ordering::Acquire) < expected + }) + .unwrap(); + drop(state); + self.normal_cleanup_attempts.load(Ordering::Acquire) >= expected + } + #[cfg(unix)] fn signal_terminal_process_session( &self, @@ -1793,10 +1817,9 @@ impl LocalPtyProcess { #[cfg(test)] { self.normal_cleanup_attempts.fetch_add(1, Ordering::AcqRel); - self.normal_cleanup_started.store(true, Ordering::Release); - let delay_ms = self.normal_cleanup_delay_ms.load(Ordering::Acquire); - if delay_ms > 0 { - std::thread::sleep(Duration::from_millis(delay_ms)); + self.exited.1.notify_all(); + if let Some(gate) = self.normal_cleanup_gate.lock().unwrap().clone() { + gate.enter_and_wait(); } if self .normal_cleanup_failures @@ -3983,19 +4006,19 @@ impl Surface { } #[cfg(test)] - pub(crate) fn set_server_shutdown_delay_for_test(&self, delay: Duration) { + pub(crate) fn set_server_shutdown_gate_for_test(&self, gate: Arc) { let Self::Pty(pty) = self else { return }; let Some(process) = &pty.local_process else { return }; let LocalProcess::Untracked(killer) = process.as_ref() else { return }; - *killer.lock().unwrap() = Box::new(TestSlowFailingChildKiller(delay)); + *killer.lock().unwrap() = Box::new(TestGatedFailingChildKiller(gate)); } #[cfg(test)] pub(crate) fn set_recovering_server_shutdown_for_test( &self, - ) -> (Arc, Arc) { + ) -> (Arc, Arc) { let failing = Arc::new(AtomicBool::new(true)); - let attempts = Arc::new(AtomicUsize::new(0)); + let attempts = Arc::new(TestAttemptCounter::default()); let Self::Pty(pty) = self else { return (failing, attempts) }; let Some(process) = &pty.local_process else { return (failing, attempts) }; let LocalProcess::Untracked(killer) = process.as_ref() else { @@ -5733,19 +5756,59 @@ impl ChildKiller for TestFailingChildKiller { } } +#[cfg(test)] +#[derive(Debug, Default)] +pub(crate) struct TestLifecycleGate { + state: Mutex<(usize, bool)>, + changed: Condvar, +} + +#[cfg(test)] +impl TestLifecycleGate { + fn enter_and_wait(&self) { + let mut state = self.state.lock().unwrap(); + state.0 = state.0.saturating_add(1); + self.changed.notify_all(); + while !state.1 { + state = self.changed.wait(state).unwrap(); + } + } + + pub(crate) fn wait_until_entered(&self, expected: usize, deadline: Instant) -> bool { + let mut state = self.state.lock().unwrap(); + while state.0 < expected { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.changed.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.0 < expected { + return false; + } + } + true + } + + pub(crate) fn release(&self) { + let mut state = self.state.lock().unwrap(); + state.1 = true; + self.changed.notify_all(); + } +} + #[cfg(test)] #[derive(Debug)] -struct TestSlowFailingChildKiller(Duration); +struct TestGatedFailingChildKiller(Arc); #[cfg(test)] -impl ChildKiller for TestSlowFailingChildKiller { +impl ChildKiller for TestGatedFailingChildKiller { fn kill(&mut self) -> std::io::Result<()> { - std::thread::sleep(self.0); - Err(std::io::Error::other("forced delayed shutdown failure")) + self.0.enter_and_wait(); + Err(std::io::Error::other("forced gated shutdown failure")) } fn clone_killer(&self) -> Box { - Box::new(Self(self.0)) + Box::new(Self(self.0.clone())) } } @@ -5753,13 +5816,48 @@ impl ChildKiller for TestSlowFailingChildKiller { #[derive(Debug)] struct TestRecoveringChildKiller { failing: Arc, - attempts: Arc, + attempts: Arc, +} + +#[cfg(test)] +#[derive(Debug, Default)] +pub(crate) struct TestAttemptCounter { + count: Mutex, + changed: Condvar, +} + +#[cfg(test)] +impl TestAttemptCounter { + fn increment(&self) { + let mut count = self.count.lock().unwrap(); + *count = count.saturating_add(1); + self.changed.notify_all(); + } + + pub(crate) fn load(&self) -> usize { + *self.count.lock().unwrap() + } + + pub(crate) fn wait_until(&self, expected: usize, deadline: Instant) -> bool { + let mut count = self.count.lock().unwrap(); + while *count < expected { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.changed.wait_timeout(count, remaining).unwrap(); + count = next; + if timeout.timed_out() && *count < expected { + return false; + } + } + true + } } #[cfg(test)] impl ChildKiller for TestRecoveringChildKiller { fn kill(&mut self) -> std::io::Result<()> { - self.attempts.fetch_add(1, Ordering::AcqRel); + self.attempts.increment(); if self.failing.load(Ordering::Acquire) { Err(std::io::Error::other("forced transient shutdown failure")) } else { @@ -6346,6 +6444,38 @@ mod tests { use super::*; + #[cfg(unix)] + fn create_test_fifo(path: &std::path::Path) { + use std::os::unix::ffi::OsStrExt as _; + + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: path is a valid, null-terminated path owned for this call. + assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0); + } + + #[cfg(unix)] + fn wait_for_test_fifo(path: &std::path::Path) { + use std::io::Read as _; + use std::os::fd::AsRawFd as _; + + let mut signal = std::fs::OpenOptions::new().read(true).write(true).open(path).unwrap(); + let mut descriptor = + libc::pollfd { fd: signal.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for this call. + assert_eq!(unsafe { libc::poll(&raw mut descriptor, 1, 2_000) }, 1); + assert_ne!(descriptor.revents & libc::POLLIN, 0); + let mut byte = [0_u8; 1]; + signal.read_exact(&mut byte).unwrap(); + } + + #[cfg(unix)] + fn signal_test_fifo(path: &std::path::Path) { + use std::io::Write as _; + + let mut signal = std::fs::OpenOptions::new().read(true).write(true).open(path).unwrap(); + signal.write_all(b"ready").unwrap(); + } + fn append_disabled_kitty_replay_state(payload: &mut Vec) { for _ in 0..4 { payload.extend_from_slice(&0u64.to_le_bytes()); @@ -6399,20 +6529,21 @@ mod tests { let baseline = descriptor_count(); let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); let (started_sender, started_receiver) = sync_channel(1); + let (result_sender, result_receiver) = sync_channel(1); let worker = std::thread::spawn(move || { started_sender.send(()).unwrap(); - Surface::spawn(1, options, Arc::downgrade(&mux)) + result_sender.send(Surface::spawn(1, options, Arc::downgrade(&mux))).unwrap(); }); started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); - let deadline = Instant::now() + Duration::from_millis(250); - let mut observed = baseline; - while observed == baseline && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(5)); - observed = descriptor_count(); - } + assert!( + result_receiver.recv_timeout(Duration::from_millis(250)).is_err(), + "PTY spawn completed while the process barrier was held" + ); + let observed = descriptor_count(); drop(process_barrier); - let surface = worker.join().unwrap().unwrap(); + let surface = result_receiver.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(); + worker.join().unwrap(); let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); assert_eq!(observed, baseline, "PTY descriptors were created outside the process barrier"); @@ -6439,6 +6570,10 @@ mod tests { .join(format!("cmux-local-launcher-env-{}", crate::workspace_registry::new_uuid_v4())); std::fs::create_dir_all(&root).unwrap(); let result = root.join("launcher.txt"); + let ready = root.join("ready"); + let hold = root.join("hold"); + create_test_fifo(&ready); + create_test_fifo(&hold); let mux = Mux::new_for_test("local-launcher-env", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ @@ -6446,17 +6581,20 @@ mod tests { "-c".into(), "result=\"$CMUX_TUI_TEST_RESULT\"; \ printf '%s' \"${CMUX_TUI_LAUNCHER_COMMAND-unset}\" > \"$result.tmp\"; \ - mv \"$result.tmp\" \"$result\"; exec /bin/sleep 60" + mv \"$result.tmp\" \"$result\"; \ + printf ready > \"$CMUX_TUI_TEST_READY\"; \ + exec /bin/cat \"$CMUX_TUI_TEST_HOLD\"" .into(), ]), - extra_env: vec![("CMUX_TUI_TEST_RESULT".into(), result.to_string_lossy().into_owned())], + extra_env: vec![ + ("CMUX_TUI_TEST_RESULT".into(), result.to_string_lossy().into_owned()), + ("CMUX_TUI_TEST_READY".into(), ready.to_string_lossy().into_owned()), + ("CMUX_TUI_TEST_HOLD".into(), hold.to_string_lossy().into_owned()), + ], ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); - let deadline = Instant::now() + Duration::from_secs(2); - while !result.exists() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + wait_for_test_fifo(&ready); let inherited = std::fs::read_to_string(&result).unwrap(); let _ = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); let _ = std::fs::remove_dir_all(root); @@ -6470,9 +6608,14 @@ mod tests { let root = std::env::temp_dir() .join(format!("cmux-local-spawn-failure-{}", crate::workspace_registry::new_uuid_v4())); std::fs::create_dir_all(&root).unwrap(); - let ready = root.join("ready"); + let ready = root.join("leader.pid"); let descendant = root.join("descendant"); - let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); + let signal = root.join("spawn-ready"); + let leader_hold = root.join("leader-hold"); + let descendant_hold = root.join("descendant-hold"); + create_test_fifo(&leader_hold); + create_test_fifo(&descendant_hold); + let _failure = crate::process_session::force_post_spawn_failure_for_test(signal.clone()); let mux = Mux::new_for_test("local-spawn-failure", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ @@ -6480,11 +6623,15 @@ mod tests { "-c".into(), format!( "trap '' HUP TERM; \ - (trap '' HUP TERM; exec /bin/sleep 60) & \ + (trap '' HUP TERM; exec /bin/cat {}) & \ echo $! > {}; \ - echo $$ > {}; exec /bin/sleep 60", + echo $$ > {}; \ + printf ready > {}; exec /bin/cat {}", + descendant_hold.display(), descendant.display(), - ready.display() + ready.display(), + signal.display(), + leader_hold.display() ), ]), ..SurfaceOptions::default() @@ -6507,19 +6654,9 @@ mod tests { libc::waitpid(pid, &raw mut status, 0); } } - let descendant_deadline = Instant::now() + Duration::from_secs(2); - let descendant_gone = loop { - // SAFETY: signal zero performs a non-mutating liveness probe. - if unsafe { libc::kill(descendant_pid, 0) } < 0 - && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) - { - break true; - } - if Instant::now() >= descendant_deadline { - break false; - } - std::thread::sleep(Duration::from_millis(10)); - }; + // SAFETY: signal zero performs a non-mutating liveness probe. + let descendant_gone = unsafe { libc::kill(descendant_pid, 0) } < 0 + && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH); if !descendant_gone { // SAFETY: the marker names the test's retained-session descendant. unsafe { @@ -6556,9 +6693,16 @@ mod tests { drop(reserve_local_child_reaper().unwrap()); let baseline = crate::process_session::reserved_child_reaper_active_for_test(); failure.store(true, Ordering::Release); + let root = std::env::temp_dir().join(format!( + "cmux-local-worker-start-failure-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let hold = root.join("hold"); + create_test_fifo(&hold); let mux = Mux::new_for_test("local-worker-start-failure", SurfaceOptions::default()); let options = SurfaceOptions { - command: Some(vec!["/bin/sleep".into(), "60".into()]), + command: Some(vec!["/bin/cat".into(), hold.to_string_lossy().into_owned()]), ..SurfaceOptions::default() }; @@ -6567,16 +6711,13 @@ mod tests { assert!(error.to_string().contains(expected_error), "unexpected spawn error: {error:#}"); let deadline = Instant::now() + Duration::from_secs(4); - while crate::process_session::reserved_child_reaper_active_for_test() != baseline - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - assert_eq!( - crate::process_session::reserved_child_reaper_active_for_test(), - baseline, + assert!( + crate::process_session::wait_for_reserved_child_reaper_active_for_test( + baseline, deadline + ), "failed local worker startup retained a child-reaper reservation" ); + let _ = std::fs::remove_dir_all(root); } #[cfg(unix)] @@ -6720,13 +6861,16 @@ mod tests { #[cfg(unix)] #[test] fn local_session_kill_observes_the_waitable_leader_under_its_owner_lock() { + let root = std::env::temp_dir().join(format!( + "cmux-local-owner-observation-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + let hold = root.join("hold"); + create_test_fifo(&hold); let mux = Mux::new_for_test("local-owner-observation", SurfaceOptions::default()); let options = SurfaceOptions { - command: Some(vec![ - "/bin/sh".into(), - "-c".into(), - "trap '' HUP TERM; while :; do sleep 60; done".into(), - ]), + command: Some(vec!["/bin/cat".into(), hold.to_string_lossy().into_owned()]), ..SurfaceOptions::default() }; let surface = Surface::spawn(1, options, Arc::downgrade(&mux)).unwrap(); @@ -6736,26 +6880,27 @@ mod tests { }; process.termination_started.store(true, Ordering::Release); - let started = Instant::now(); - let killed = process - .kill_terminal_process_session_until(Instant::now() + Duration::from_secs(1), None) + let kill_process = process.clone(); + let (killed_tx, killed_rx) = sync_channel(1); + let killer = std::thread::spawn(move || { + let killed = kill_process + .kill_terminal_process_session_until(Instant::now() + Duration::from_secs(1), None); + killed_tx.send(killed).unwrap(); + }); + let killed = killed_rx + .recv_timeout(Duration::from_millis(500)) + .expect("session cleanup waited on the reaper while retaining its owner lock") .unwrap(); - let elapsed = started.elapsed(); + killer.join().unwrap(); process.group_escalation_complete.store(true, Ordering::Release); process.exited.1.notify_all(); process.wake_reserved_child_reaper(); let reap_deadline = Instant::now() + Duration::from_secs(2); - while !process.child_reaped.load(Ordering::Acquire) && Instant::now() < reap_deadline { - std::thread::sleep(Duration::from_millis(10)); - } assert!(killed, "session cleanup could not observe its SIGKILLed leader"); - assert!( - elapsed < Duration::from_millis(500), - "session cleanup waited on the reaper while retaining its owner lock: {elapsed:?}" - ); - assert!(process.child_reaped.load(Ordering::Acquire)); + assert!(process.wait_for_child_reaped_until(reap_deadline)); + let _ = std::fs::remove_dir_all(root); } #[cfg(unix)] @@ -6822,6 +6967,10 @@ mod tests { let pid_path = root.join("descendant.pid"); let ready_path = root.join("descendant.ready"); let release_path = root.join("leader.release"); + let descendant_hold = root.join("descendant.hold"); + create_test_fifo(&ready_path); + create_test_fifo(&release_path); + create_test_fifo(&descendant_hold); let mux = Mux::new_for_test("local-reap-descendants", SurfaceOptions::default()); let options = SurfaceOptions { command: Some(vec![ @@ -6829,9 +6978,10 @@ mod tests { "-c".into(), format!( "trap '' HUP TERM; terminal=$(tty); \ - sleep 30 3<>\"$terminal\" & child=$!; \ - : > {ready}; echo $child > {pid}; \ - while [ ! -e {release} ]; do sleep 0.01; done; exit 0", + /bin/cat {hold} 3<>\"$terminal\" & child=$!; \ + echo $child > {pid}; printf ready > {ready}; \ + /bin/cat {release}; exit 0", + hold = descendant_hold.display(), ready = ready_path.display(), pid = pid_path.display(), release = release_path.display(), @@ -6844,34 +6994,21 @@ mod tests { LocalProcess::Owned(process) => process.clone(), LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), }; - let start_deadline = Instant::now() + Duration::from_secs(1); - while !pid_path.exists() && Instant::now() < start_deadline { - std::thread::sleep(Duration::from_millis(10)); - } + wait_for_test_fifo(&ready_path); let descendant = std::fs::read_to_string(&pid_path).unwrap().trim().parse::().unwrap(); // SAFETY: signal 0 only probes the test-owned PID. assert_eq!(unsafe { libc::kill(descendant, 0) }, 0); - std::fs::write(&release_path, b"ready").unwrap(); + signal_test_fifo(&release_path); let deadline = Instant::now() + Duration::from_secs(3); - while !process.child_reaped.load(Ordering::Acquire) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let naturally_reaped = process.child_reaped.load(Ordering::Acquire); + let naturally_reaped = process.wait_for_child_reaped_until(deadline); // SAFETY: signal 0 only probes the test-owned PID. let descendant_alive_before_cleanup = unsafe { libc::kill(descendant, 0) } == 0; let terminated = surface.terminate_for_server_shutdown(Instant::now() + Duration::from_secs(1)); - let mut descendant_alive = true; - let exit_deadline = Instant::now() + Duration::from_secs(1); - while descendant_alive && Instant::now() < exit_deadline { - // SAFETY: signal 0 only probes the test-owned PID. - descendant_alive = unsafe { libc::kill(descendant, 0) } == 0; - if descendant_alive { - std::thread::sleep(Duration::from_millis(10)); - } - } + // SAFETY: signal 0 only probes the test-owned PID. + let descendant_alive = unsafe { libc::kill(descendant, 0) } == 0; if descendant_alive { // SAFETY: this PID was written by the test-owned descendant. unsafe { @@ -6916,14 +7053,8 @@ mod tests { std::fs::write(&release_path, b"ready").unwrap(); let retry_deadline = Instant::now() + Duration::from_secs(3); - while (!process.child_reaped.load(Ordering::Acquire) - || process.normal_cleanup_attempts.load(Ordering::Acquire) < 2) - && Instant::now() < retry_deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - let retried = process.normal_cleanup_attempts.load(Ordering::Acquire) >= 2; - let reaped = process.child_reaped.load(Ordering::Acquire); + let retried = process.wait_for_normal_cleanup_attempts_until(2, retry_deadline); + let reaped = process.wait_for_child_reaped_until(retry_deadline); let _ = process.terminate_and_wait(Instant::now() + Duration::from_secs(1)); let _ = std::fs::remove_dir_all(root); @@ -6973,13 +7104,9 @@ mod tests { std::fs::write(&release_path, b"ready").unwrap(); let deadline = Instant::now() + Duration::from_secs(3); - while processes + let all_attempted = processes .iter() - .any(|process| process.normal_cleanup_attempts.load(Ordering::Acquire) == 0) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } + .all(|process| process.wait_for_normal_cleanup_attempts_until(1, deadline)); let dedicated = crate::process_session::dedicated_natural_reap_workers_for_test() .saturating_sub(baseline); @@ -6997,6 +7124,7 @@ mod tests { dedicated_observers, 0, "local child observation bypassed the reserved shared reaper" ); + assert!(all_attempted, "a reserved child did not reach the shared cleanup owner"); } #[cfg(unix)] @@ -7020,31 +7148,28 @@ mod tests { LocalProcess::Owned(process) => process.clone(), LocalProcess::Untracked(_) => unreachable!("real PTY process must be tracked"), }; - process.normal_cleanup_delay_ms.store( - crate::test_timeout(Duration::from_millis(300)).as_millis() as u64, - Ordering::Release, - ); + let cleanup_gate = Arc::new(TestLifecycleGate::default()); + *process.normal_cleanup_gate.lock().unwrap() = Some(cleanup_gate.clone()); std::fs::write(&release_path, b"ready").unwrap(); let sweep_deadline = Instant::now() + crate::test_timeout(Duration::from_secs(1)); - while !process.normal_cleanup_started.load(Ordering::Acquire) - && Instant::now() < sweep_deadline - { - std::thread::sleep(Duration::from_millis(5)); - } - assert!(process.normal_cleanup_started.load(Ordering::Acquire)); + assert!(cleanup_gate.wait_until_entered(1, sweep_deadline)); - let started = Instant::now(); - let _ = - process.terminate_and_wait(started + crate::test_timeout(Duration::from_millis(50))); - let elapsed = started.elapsed(); + let shutdown_process = process.clone(); + let (shutdown_tx, shutdown_rx) = sync_channel(1); + let shutdown = std::thread::spawn(move || { + let result = shutdown_process.terminate_and_wait( + Instant::now() + crate::test_timeout(Duration::from_millis(50)), + ); + shutdown_tx.send(result).unwrap(); + }); + shutdown_rx + .recv_timeout(crate::test_timeout(Duration::from_millis(150))) + .expect("explicit shutdown waited behind the natural cleanup sweep"); + cleanup_gate.release(); + shutdown.join().unwrap(); let _ = process .terminate_and_wait(Instant::now() + crate::test_timeout(Duration::from_secs(1))); let _ = std::fs::remove_dir_all(root); - - assert!( - elapsed < crate::test_timeout(Duration::from_millis(150)), - "explicit shutdown waited behind the natural cleanup sweep: {elapsed:?}" - ); } #[test] @@ -7054,19 +7179,29 @@ mod tests { Surface::spawn_for_test(1, SurfaceOptions::default(), Arc::downgrade(&mux)).unwrap(); let locked = surface.clone(); let (held_tx, held_rx) = sync_channel(1); + let (release_tx, release_rx) = sync_channel(1); let holder = std::thread::spawn(move || { let pty = locked.as_pty().unwrap(); let _runtime = pty.runtime.lock().unwrap(); held_tx.send(()).unwrap(); - std::thread::sleep(Duration::from_millis(150)); + release_rx.recv().unwrap(); }); held_rx.recv_timeout(Duration::from_secs(1)).unwrap(); - let started = Instant::now(); - assert!(surface.terminate_for_server_shutdown(started + Duration::from_millis(25))); - assert!(started.elapsed() < Duration::from_millis(100)); + let (shutdown_tx, shutdown_rx) = sync_channel(1); + let shutdown = std::thread::spawn(move || { + shutdown_tx + .send( + surface + .terminate_for_server_shutdown(Instant::now() + Duration::from_millis(25)), + ) + .unwrap(); + }); + assert!(shutdown_rx.recv_timeout(Duration::from_millis(100)).unwrap()); + release_tx.send(()).unwrap(); holder.join().unwrap(); + shutdown.join().unwrap(); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index ea5db2b2d7e2..4b209e753336 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -4655,25 +4655,36 @@ mod unix { } } + #[cfg(debug_assertions)] + fn wait_for_test_host_gate(signal_name: &str, gate_name: &str) -> anyhow::Result<()> { + if let Some(signal) = std::env::var_os(signal_name) { + fs::OpenOptions::new().write(true).open(signal)?.write_all(b"1")?; + } + if let Some(gate) = std::env::var_os(gate_name) { + let mut gate = fs::File::open(gate)?; + let mut release = [0_u8; 1]; + gate.read_exact(&mut release)?; + } + Ok(()) + } + pub fn serve_terminal_host_stdio( args: &[String], reader: &mut impl Read, writer: &mut impl Write, ) -> anyhow::Result<()> { let publication_descriptor = terminal_host_publication_descriptor(args)?; - if let Ok(delay) = std::env::var("CMUX_TUI_TEST_BOOTSTRAP_READY_DELAY_MS") - && let Ok(delay) = delay.parse::() - && delay > 0 - { - thread::sleep(Duration::from_millis(delay.min(5_000))); - } + #[cfg(debug_assertions)] + wait_for_test_host_gate( + "CMUX_TUI_TEST_BOOTSTRAP_READY_STARTED_SIGNAL", + "CMUX_TUI_TEST_BOOTSTRAP_READY_GATE", + )?; let bootstrapped = crate::terminal_host::bootstrap_stdio_once(reader, writer)?; - if let Ok(delay) = std::env::var("CMUX_TUI_TEST_STALL_AFTER_BOOTSTRAP_READY_MS") - && let Ok(delay) = delay.parse::() - && delay > 0 - { - thread::sleep(Duration::from_millis(delay.min(5_000))); - } + #[cfg(debug_assertions)] + wait_for_test_host_gate( + "CMUX_TUI_TEST_LAUNCH_READ_STARTED_SIGNAL", + "CMUX_TUI_TEST_LAUNCH_READ_GATE", + )?; let Some(launch_frame) = read_frame(reader, MAX_LAUNCH_PAYLOAD)? else { // Keep the one-frame bootstrap probe useful for compatibility and // packaging diagnostics. Production launchers always follow it @@ -6290,7 +6301,9 @@ mod unix { fs::create_dir_all(&root).unwrap(); let ready = root.join("ready"); let descendant = root.join("descendant"); - let _failure = crate::process_session::force_post_spawn_failure_for_test(ready.clone()); + let signal = root.join("spawn-ready"); + let _failure = + crate::process_session::force_post_spawn_failure_for_test(signal.clone()); let bootstrap = HostBootstrap { min_version: PROTOCOL_VERSION, max_version: PROTOCOL_VERSION, @@ -6320,9 +6333,11 @@ mod unix { "trap '' HUP TERM; \ (trap '' HUP TERM; exec /bin/sleep 60) & \ echo $! > {}; \ - echo $$ > {}; exec /bin/sleep 60", + echo $$ > {}; \ + printf ready > {}; exec /bin/sleep 60", descendant.display(), - ready.display() + ready.display(), + signal.display() ), ], extra_env: Vec::new(), diff --git a/cmux-tui/crates/cmux-tui-process/src/lib.rs b/cmux-tui/crates/cmux-tui-process/src/lib.rs index 5a0529714434..9f000336357d 100644 --- a/cmux-tui/crates/cmux-tui-process/src/lib.rs +++ b/cmux-tui/crates/cmux-tui-process/src/lib.rs @@ -470,10 +470,16 @@ mod tests { .recv_timeout(Duration::from_secs(1)) .expect("process creation barrier was not acquired"); - let connect = - tokio::spawn(async move { super::tokio_net::connect_tcp_stream(&[address]).await }); - tokio::time::sleep(Duration::from_millis(250)).await; - let completed_while_barrier_held = connect.is_finished(); + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let mut connect = tokio::spawn(async move { + started_sender.send(()).unwrap(); + super::tokio_net::connect_tcp_stream(&[address]).await + }); + started_receiver.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(250), &mut connect).await.is_err(), + "TCP connect created a socket while a concurrent process could inherit it" + ); release_sender.send(()).unwrap(); holder.join().unwrap(); @@ -485,9 +491,5 @@ mod tests { let (peer, _) = listener.accept().unwrap(); drop(peer); drop(stream); - assert!( - !completed_while_barrier_held, - "TCP connect created a socket while a concurrent process could inherit it" - ); } } diff --git a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs index c9e21a2cc2df..bfe60261c476 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_runtime.rs @@ -565,26 +565,27 @@ mod tests { let baseline = descriptor_count(); let process_barrier = cmux_tui_process::ProcessCreationGuard::acquire(); let (started_sender, started_receiver) = std::sync::mpsc::sync_channel(1); + let (result_sender, result_receiver) = std::sync::mpsc::sync_channel(1); let worker = thread::spawn(move || { let mut command = Command::new("/usr/bin/true"); command.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()); started_sender.send(()).unwrap(); - spawn_transport_process(&mut command) + result_sender.send(spawn_transport_process(&mut command)).unwrap(); }); started_receiver.recv_timeout(Duration::from_secs(1)).unwrap(); - let deadline = Instant::now() + Duration::from_millis(250); - let mut observed = baseline; - while observed == baseline && Instant::now() < deadline { - thread::sleep(Duration::from_millis(5)); - observed = descriptor_count(); - } - + assert!( + result_receiver.recv_timeout(Duration::from_millis(250)).is_err(), + "transport spawn completed while the process barrier was held" + ); assert_eq!( - observed, baseline, + descriptor_count(), + baseline, "UnixStream::pair created inheritable descriptors outside the process barrier" ); drop(process_barrier); - let (stdin, stdout, process) = worker.join().unwrap().unwrap(); + let (stdin, stdout, process) = + result_receiver.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(); + worker.join().unwrap(); drop(stdin); drop(stdout); drop(process); diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index ed629f0e0c69..5e93b30a52fa 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1827,7 +1827,14 @@ impl ServerProcessShutdownGuard { #[cfg(debug_assertions)] if let Some(marker) = std::env::var_os("CMUX_TUI_TEST_WATCHDOG_START_FAILURE") { std::fs::write(marker, b"starting")?; - std::thread::sleep(std::time::Duration::from_millis(500)); + if let Some(release) = std::env::var_os("CMUX_TUI_TEST_WATCHDOG_START_FAILURE_RELEASE") + { + use std::io::Read as _; + + let mut release = std::fs::File::open(release)?; + let mut signal = [0_u8; 1]; + release.read_exact(&mut signal)?; + } return Err(io::Error::other("forced watchdog start failure")); } let shutdown_watch = mux.watch_shutdown_request(); @@ -2002,7 +2009,12 @@ fn finish_server_process( } else { server_process.abandon_unpublished(); } - combine_server_and_remote_shutdown_results(result, shutdown_result) + let result = combine_server_and_remote_shutdown_results(result, shutdown_result); + #[cfg(debug_assertions)] + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_SERVER_EXIT_READY_SIGNAL") { + let _ = std::fs::write(signal, b"1"); + } + result } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index a22bf1baf7dd..73ebb7791199 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -7,9 +7,9 @@ use std::fs::{self, OpenOptions}; use std::future::Future; use std::net::SocketAddr; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc; +use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::Duration; @@ -337,6 +337,48 @@ pub struct DaemonRuntimeHandle { info: DaemonRuntimeInfo, shutdown: watch::Sender, thread: Option>>, + completion: Arc, +} + +#[derive(Default)] +struct RuntimeThreadCompletion { + finished: Mutex, + changed: Condvar, +} + +impl RuntimeThreadCompletion { + fn mark_finished(&self) { + *self.finished.lock().unwrap() = true; + self.changed.notify_all(); + } + + fn is_finished(&self) -> bool { + *self.finished.lock().unwrap() + } + + fn wait_until(&self, deadline: std::time::Instant) -> bool { + let finished = self.finished.lock().unwrap(); + if *finished { + return true; + } + let (finished, _) = self + .changed + .wait_timeout_while( + finished, + deadline.saturating_duration_since(std::time::Instant::now()), + |finished| !*finished, + ) + .unwrap(); + *finished + } +} + +struct RuntimeThreadCompletionGuard(Arc); + +impl Drop for RuntimeThreadCompletionGuard { + fn drop(&mut self) { + self.0.mark_finished(); + } } pub(crate) enum DaemonRuntimeShutdown { @@ -350,7 +392,7 @@ impl DaemonRuntimeHandle { } pub fn is_finished(&self) -> bool { - self.thread.as_ref().is_some_and(thread::JoinHandle::is_finished) + self.completion.is_finished() } fn request_shutdown(&self) { @@ -359,19 +401,15 @@ impl DaemonRuntimeHandle { pub(crate) fn shutdown_until(&mut self, deadline: std::time::Instant) -> DaemonRuntimeShutdown { self.request_shutdown(); - loop { - let Some(worker) = self.thread.as_ref() else { - return DaemonRuntimeShutdown::Complete(Err(anyhow!( - "remote daemon runtime thread is absent" - ))); - }; - if worker.is_finished() { - return DaemonRuntimeShutdown::Complete(self.join_runtime_thread()); - } - let Some(remaining) = deadline.checked_duration_since(std::time::Instant::now()) else { - return DaemonRuntimeShutdown::Pending; - }; - thread::sleep(remaining.min(Duration::from_millis(10))); + if self.thread.is_none() { + return DaemonRuntimeShutdown::Complete(Err(anyhow!( + "remote daemon runtime thread is absent" + ))); + } + if self.completion.wait_until(deadline) { + DaemonRuntimeShutdown::Complete(self.join_runtime_thread()) + } else { + DaemonRuntimeShutdown::Pending } } @@ -390,6 +428,15 @@ impl DaemonRuntimeHandle { #[cfg(test)] pub(crate) fn from_test_thread(thread: thread::JoinHandle>) -> Self { let (shutdown, _) = watch::channel(false); + let completion = Arc::new(RuntimeThreadCompletion::default()); + let worker_completion = completion.clone(); + let thread = thread::spawn(move || { + let _completion = RuntimeThreadCompletionGuard(worker_completion); + match thread.join() { + Ok(result) => result, + Err(_) => Err(anyhow!("remote daemon runtime thread panicked")), + } + }); Self { info: DaemonRuntimeInfo { session: "test".into(), @@ -405,28 +452,30 @@ impl DaemonRuntimeHandle { }, shutdown, thread: Some(thread), + completion, } } pub fn shutdown(mut self) -> anyhow::Result<()> { #[cfg(debug_assertions)] - let shutdown_marker = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER"); + let shutdown_started = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_STARTED_SIGNAL"); #[cfg(debug_assertions)] - if let Some(marker) = shutdown_marker.as_ref() { - let _ = fs::write(marker, b"started"); + if let Some(signal) = shutdown_started.as_ref() { + fs::write(signal, b"1")?; } #[cfg(debug_assertions)] - if let Some(milliseconds) = std::env::var("CMUX_TUI_TEST_REMOTE_SHUTDOWN_DELAY_MS") - .ok() - .and_then(|value| value.parse::().ok()) - { - thread::sleep(Duration::from_millis(milliseconds)); + if let Some(gate) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_GATE") { + use std::io::Read as _; + + let mut gate = fs::File::open(gate)?; + let mut signal = [0_u8; 1]; + gate.read_exact(&mut signal)?; } self.request_shutdown(); let result = self.join_runtime_thread(); #[cfg(debug_assertions)] - if let Some(marker) = shutdown_marker { - let _ = fs::write(marker, b"complete"); + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_COMPLETE_SIGNAL") { + fs::write(signal, b"1")?; } result } @@ -674,6 +723,8 @@ pub fn start_client_runtime(options: ClientRuntimeOptions) -> anyhow::Result usize { .map_or(0, |reaper| reaper.active.load(Ordering::Acquire)) } +#[cfg(all(unix, test))] +fn wait_for_active_legacy_helper_reapers_for_test(expected: usize, deadline: Instant) -> bool { + let Some(reaper) = LEGACY_HELPER_REAPER.get().and_then(Option::as_ref) else { + return expected == 0; + }; + let mut pending = reaper.pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + while reaper.active.load(Ordering::Acquire) != expected { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = reaper + .pending + .1 + .wait_timeout(pending, remaining) + .unwrap_or_else(std::sync::PoisonError::into_inner); + pending = next; + if timeout.timed_out() && reaper.active.load(Ordering::Acquire) != expected { + return false; + } + } + true +} + #[cfg(unix)] fn run_legacy_helper_reaper(pending: Arc<(Mutex>, Condvar)>) { let mut pending_children = pending.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner); @@ -2110,7 +2135,8 @@ mod tests { let (listener, quarantine, path) = quarantined_test_listener("cq-reap"); let mut helper = Command::new("/bin/sh") .arg("-c") - .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") + .arg("trap '' TERM; printf 'ready\\n'; read _") + .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() @@ -2123,6 +2149,7 @@ mod tests { let logical_now = Arc::new(Mutex::new(logical_start)); let read_clock = logical_now.clone(); let advance_clock = logical_now.clone(); + let baseline = active_legacy_helper_reapers_for_test(); let reaper = reserve_legacy_helper_reaper().unwrap(); let error = wait_for_child_until_with( @@ -2141,7 +2168,10 @@ mod tests { unsafe { libc::kill(helper_pid, libc::SIGKILL); } - std::thread::sleep(Duration::from_millis(250)); + assert!(wait_for_active_legacy_helper_reapers_for_test( + baseline, + Instant::now() + Duration::from_secs(1) + )); let mut status = 0; // SAFETY: this probes whether another owner retained and reaped the // exact test child after wait_for_child_until returned. @@ -2209,7 +2239,8 @@ mod tests { let (listener, quarantine, path) = quarantined_test_listener("cq-help"); let mut helper = Command::new("/bin/sh") .arg("-c") - .arg("trap '' TERM; printf 'ready\\n'; while :; do sleep 1; done") + .arg("trap '' TERM; printf 'ready\\n'; read _") + .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) .spawn() @@ -2233,9 +2264,7 @@ mod tests { libc::kill(helper_pid, libc::SIGKILL); } let deadline = Instant::now() + Duration::from_secs(1); - while active_legacy_helper_reapers_for_test() != baseline && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + assert!(wait_for_active_legacy_helper_reapers_for_test(baseline, deadline)); let restored_after_helper_exit = UnixStream::connect(&path).is_ok(); drop(listener); @@ -2274,9 +2303,7 @@ mod tests { reaper.retain_shutdown_ownership(LegacyHelperChild { child, quarantine }); let deadline = Instant::now() + Duration::from_millis(250); - while active_legacy_helper_reapers_for_test() != baseline && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } + assert!(wait_for_active_legacy_helper_reapers_for_test(baseline, deadline)); assert_eq!( active_legacy_helper_reapers_for_test(), diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index d7e676b37d85..32ec2fc933be 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -222,10 +222,8 @@ pub(super) fn terminate_process_tree_until( process: ProcessIdentity, deadline: Instant, ) -> io::Result<()> { - retry_process_tree_termination(process, deadline, |_| { - let tree = FrozenProcessTree::freeze(process, deadline)?; - tree.terminate_until(deadline) - }) + let tree = FrozenProcessTree::freeze(process, deadline)?; + tree.terminate_until(deadline) } #[cfg(test)] @@ -253,9 +251,6 @@ fn retry_process_tree_termination( if Instant::now() >= deadline { return Err(error); } - std::thread::sleep( - deadline.saturating_duration_since(Instant::now()).min(PROCESS_TREE_RETRY_INTERVAL), - ); } } @@ -973,20 +968,11 @@ mod tests { } } drop(tree); - let deadline = Instant::now() + Duration::from_millis(250); - let mut resumed = false; - while Instant::now() < deadline { - match stdout.read(&mut output) { - Ok(0) => break, - Ok(_) => { - resumed = true; - break; - } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} - Err(error) => panic!("failed to read resumed child output: {error}"), - } - std::thread::sleep(Duration::from_millis(5)); - } + let mut poll_descriptor = libc::pollfd { fd: descriptor, events: libc::POLLIN, revents: 0 }; + // SAFETY: poll_descriptor points to one initialized pollfd for this call. + assert_eq!(unsafe { libc::poll(&raw mut poll_descriptor, 1, 250) }, 1); + assert_ne!(poll_descriptor.revents & libc::POLLIN, 0); + let resumed = stdout.read(&mut output).unwrap() != 0; STABLE_HANDLE_CAPTURE_BUDGET.set(None); // SAFETY: only the exact test-owned child can still use this unreaped PID. @@ -1050,8 +1036,8 @@ mod tests { #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn captured_session_waits_for_an_unavailable_exact_member_to_exit() { - let mut command = Command::new("sleep"); - command.arg("60").stdout(Stdio::null()).stderr(Stdio::null()); + let mut command = Command::new("cat"); + command.stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::null()); unsafe { command.pre_exec(|| { if libc::setsid() < 0 { @@ -1066,14 +1052,27 @@ mod tests { let unavailable = ProcessIdentity { pid: process.pid, started_at: process.started_at.wrapping_add(1) }; let session = CapturedSession { id: pid, members: vec![unavailable] }; + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); let reaper = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(50)); + release_rx.recv().unwrap(); child.kill().unwrap(); child.wait().unwrap() }); - let result = session.kill_until_empty(Instant::now() + Duration::from_secs(1)); + let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let waiter = std::thread::spawn(move || { + result_tx + .send(session.kill_until_empty(Instant::now() + Duration::from_secs(1))) + .unwrap(); + }); + assert!( + result_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "captured session completed while its exact member remained alive" + ); + release_tx.send(()).unwrap(); + let result = result_rx.recv_timeout(Duration::from_secs(1)).unwrap(); reaper.join().unwrap(); + waiter.join().unwrap(); assert!(result.unwrap()); } diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index 920606aa4d75..fa7eafc54ef5 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -1725,6 +1725,10 @@ impl RemoteSession { client_info["capabilities"] = json!([GUARDED_BROWSER_POINTER_CAPABILITY]); } self.request(client_info)?; + #[cfg(debug_assertions)] + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_CLIENT_REGISTERED_SIGNAL") { + std::fs::write(signal, b"1")?; + } if subscribe { self.prime_local_subscription(); if let Err(error) = self.request(self.subscription_request()) { diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 75bd4b9f6d35..b7f7adea780b 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -21,6 +21,72 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use cmux_tui_core::platform::transport; +#[cfg(unix)] +struct TestFifoSignal(File); + +#[cfg(unix)] +impl TestFifoSignal { + fn create(path: &std::path::Path) -> Self { + let path_name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: path_name is a valid, NUL-terminated path in the test directory. + assert_eq!(unsafe { libc::mkfifo(path_name.as_ptr(), 0o600) }, 0); + Self(fs::OpenOptions::new().read(true).write(true).open(path).unwrap()) + } + + fn send(&mut self) { + self.0.write_all(b"1").unwrap(); + } + + fn receive(&mut self, timeout: Duration) { + let timeout = timeout.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: self.0.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for this call. + assert_eq!(unsafe { libc::poll(&raw mut descriptor, 1, timeout) }, 1); + assert_ne!(descriptor.revents & libc::POLLIN, 0); + let mut byte = [0_u8; 1]; + self.0.read_exact(&mut byte).unwrap(); + } +} + +#[cfg(unix)] +fn wait_for_child_exit( + child: &mut Child, + timeout: Duration, + failure: &str, +) -> std::process::ExitStatus { + let pid = child.id(); + let (exited_tx, exited_rx) = mpsc::sync_channel(1); + let observer = std::thread::spawn(move || { + let mut status = std::mem::MaybeUninit::::zeroed(); + // SAFETY: status points to writable siginfo storage. WNOWAIT observes + // this exact child transition without consuming its exit status. + let result = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + status.as_mut_ptr(), + libc::WEXITED | libc::WNOWAIT, + ) + }; + let result = if result == 0 { Ok(()) } else { Err(std::io::Error::last_os_error()) }; + exited_tx.send(result).unwrap(); + }); + match exited_rx.recv_timeout(timeout) { + Ok(Ok(())) => {} + Ok(Err(error)) => panic!("waitid failed: {error}"), + Err(_) => { + let _ = child.kill(); + let _ = exited_rx.recv_timeout(Duration::from_secs(1)); + let _ = child.wait(); + observer.join().unwrap(); + panic!("{failure}"); + } + } + observer.join().unwrap(); + child.wait().unwrap() +} + struct HeadlessServer { child: Child, socket: PathBuf, @@ -175,37 +241,40 @@ fn server_establishes_shutdown_ownership_before_publishing_its_listener() { fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let marker = dir.join("watchdog-starting"); + let release = dir.join("watchdog-release"); + for path in [&marker, &release] { + let path = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: path is a valid, NUL-terminated path in the test directory. + assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0); + } + let (ready_sender, ready_receiver) = std::sync::mpsc::sync_channel(1); + let ready_marker = marker.clone(); + let ready_reader = std::thread::spawn(move || { + use std::io::Read as _; + + let mut marker = fs::File::open(ready_marker).unwrap(); + let mut signal = [0_u8; 1]; + marker.read_exact(&mut signal).unwrap(); + ready_sender.send(()).unwrap(); + }); let mut child = Command::new(bin()) .args(["--headless", "--ephemeral", "--socket"]) .arg(&socket) .env("CMUX_TUI_TEST_WATCHDOG_START_FAILURE", &marker) + .env("CMUX_TUI_TEST_WATCHDOG_START_FAILURE_RELEASE", &release) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() .unwrap(); - let marker_deadline = Instant::now() + Duration::from_secs(15); - while !marker.exists() && Instant::now() < marker_deadline { - assert!( - child.try_wait().unwrap().is_none(), - "server exited before entering the watchdog failure hook" - ); - std::thread::sleep(Duration::from_millis(10)); - } - assert!(marker.exists(), "watchdog failure hook was not reached"); + ready_receiver + .recv_timeout(Duration::from_secs(15)) + .expect("watchdog failure hook was not reached"); + ready_reader.join().unwrap(); let listener_was_published = socket.exists(); - let exit_deadline = Instant::now() + Duration::from_secs(5); - let status = loop { - if let Some(status) = child.try_wait().unwrap() { - break status; - } - if Instant::now() >= exit_deadline { - child.kill().unwrap(); - break child.wait().unwrap(); - } - std::thread::sleep(Duration::from_millis(10)); - }; + fs::write(&release, b"release").unwrap(); + let status = child.wait().unwrap(); let _ = fs::remove_file(&socket); let _ = fs::remove_dir_all(&dir); @@ -251,7 +320,26 @@ fn process_is_active(pid: u32) -> bool { #[cfg(all(unix, not(target_os = "linux")))] fn process_is_active(pid: u32) -> bool { - process_exists(pid) + let output = Command::new("ps") + .args(["-o", "stat=", "-p", &pid.to_string()]) + .output() + .expect("inspect process state"); + output.status.success() + && output.stdout.iter().copied().find(|byte| !byte.is_ascii_whitespace()) != Some(b'Z') +} + +#[cfg(unix)] +fn process_group_is_active(group: u32) -> bool { + let output = Command::new("ps") + .args(["-axo", "pgid=,stat="]) + .output() + .expect("inspect process-group state"); + let group = group.to_string(); + String::from_utf8_lossy(&output.stdout).lines().any(|line| { + let mut fields = line.split_whitespace(); + fields.next() == Some(group.as_str()) + && fields.next().is_some_and(|state| !state.starts_with('Z')) + }) } #[cfg(unix)] @@ -277,16 +365,117 @@ fn process_group_exists(_pid: u32) -> bool { #[cfg(unix)] fn wait_for_pid_file(path: &std::path::Path, timeout: Duration) -> u32 { + fn read_pid(path: &std::path::Path) -> Option { + fs::read_to_string(path).ok()?.trim().parse().ok() + } + + if let Some(pid) = read_pid(path) { + return pid; + } let deadline = Instant::now() + timeout; - loop { - if let Some(pid) = - fs::read_to_string(path).ok().and_then(|value| value.trim().parse::().ok()) - { - return pid; + + #[cfg(target_os = "linux")] + { + let parent = path.parent().unwrap(); + // SAFETY: inotify_init1 returns a new owned descriptor on success. + let descriptor = unsafe { libc::inotify_init1(libc::IN_CLOEXEC) }; + assert!(descriptor >= 0, "could not create PID publication observer"); + // SAFETY: descriptor is newly owned by this test. + let observer = unsafe { File::from_raw_fd(descriptor) }; + let parent = std::ffi::CString::new(parent.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: parent is a valid, NUL-terminated directory path. + assert!( + unsafe { + libc::inotify_add_watch( + observer.as_raw_fd(), + parent.as_ptr(), + libc::IN_CREATE | libc::IN_CLOSE_WRITE | libc::IN_MOVED_TO, + ) + } >= 0 + ); + loop { + if let Some(pid) = read_pid(path) { + return pid; + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("PID file was not published before its deadline"); + let timeout = remaining.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut poll_descriptor = + libc::pollfd { fd: observer.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: poll_descriptor points to one initialized pollfd for this call. + assert_eq!(unsafe { libc::poll(&raw mut poll_descriptor, 1, timeout) }, 1); + let mut events = [0_u8; 4_096]; + // SAFETY: events is writable for its full length. + assert!( + unsafe { + libc::read(observer.as_raw_fd(), events.as_mut_ptr().cast(), events.len()) + } > 0 + ); } - assert!(Instant::now() < deadline, "PID file was not ready: {}", path.display()); - std::thread::sleep(Duration::from_millis(10)); } + + #[cfg(target_os = "macos")] + { + let directory = File::open(path.parent().unwrap()).unwrap(); + // SAFETY: kqueue returns a new owned descriptor on success. + let descriptor = unsafe { libc::kqueue() }; + assert!(descriptor >= 0, "could not create PID publication observer"); + // SAFETY: descriptor is newly owned by this test. + let observer = unsafe { File::from_raw_fd(descriptor) }; + let change = libc::kevent { + ident: directory.as_raw_fd() as libc::uintptr_t, + filter: libc::EVFILT_VNODE, + flags: (libc::EV_ADD | libc::EV_ENABLE | libc::EV_CLEAR) as u16, + fflags: libc::NOTE_WRITE | libc::NOTE_EXTEND | libc::NOTE_RENAME, + data: 0, + udata: std::ptr::null_mut(), + }; + // SAFETY: change describes the live directory descriptor for this call. + assert_eq!( + unsafe { + libc::kevent( + observer.as_raw_fd(), + &raw const change, + 1, + std::ptr::null_mut(), + 0, + std::ptr::null(), + ) + }, + 0 + ); + loop { + if let Some(pid) = read_pid(path) { + return pid; + } + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("PID file was not published before its deadline"); + let timeout = libc::timespec { + tv_sec: remaining.as_secs() as _, + tv_nsec: remaining.subsec_nanos() as _, + }; + let mut event = std::mem::MaybeUninit::::uninit(); + // SAFETY: event points to one writable kevent and timeout lives for this call. + assert_eq!( + unsafe { + libc::kevent( + observer.as_raw_fd(), + std::ptr::null(), + 0, + event.as_mut_ptr(), + 1, + &raw const timeout, + ) + }, + 1 + ); + } + } + + #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] + panic!("PID publication observer is unavailable on this test platform") } #[cfg(unix)] @@ -313,12 +502,15 @@ impl LegacyServerProcess { fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let descendant_pid_file = dir.join("descendant.pid"); + let ready_path = dir.join("ready"); + let mut ready = TestFifoSignal::create(&ready_path); let mut command = Command::new(std::env::current_exe().unwrap()); command .args(["--ignored", "--exact", "legacy_server_process_helper"]) .env("CMUX_TUI_TEST_LEGACY_SOCKET", &socket) .env("CMUX_TUI_TEST_LEGACY_SCENARIO", scenario) .env("CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE", &descendant_pid_file) + .env("CMUX_TUI_TEST_LEGACY_READY_SIGNAL", &ready_path) .stdout(Stdio::null()) .stderr(Stdio::piped()); if let Some(reported_pid) = reported_pid { @@ -328,20 +520,8 @@ impl LegacyServerProcess { command.env("CMUX_TUI_TEST_LEGACY_SURFACE_PID", surface_pid.to_string()); } let child = command.spawn().unwrap(); - let mut server = Self { child, socket, dir, descendant_pid_file }; - let deadline = Instant::now() + Duration::from_secs(15); - while Instant::now() < deadline { - if server.socket.exists() { - return server; - } - if let Some(status) = server.child.try_wait().unwrap() { - let mut stderr = String::new(); - server.child.stderr.take().unwrap().read_to_string(&mut stderr).unwrap(); - panic!("legacy server helper exited with {status}: {stderr}"); - } - std::thread::sleep(Duration::from_millis(25)); - } - panic!("legacy server helper did not create socket at {}", server.socket.display()); + ready.receive(Duration::from_secs(15)); + Self { child, socket, dir, descendant_pid_file } } fn descendant_pid(&self) -> Option { @@ -417,18 +597,20 @@ fn legacy_server_process_helper() { ) .unwrap(); drop(descendant); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - let status = Command::new("ps") - .args(["-o", "stat=", "-p", &descendant_pid.to_string()]) - .output() - .unwrap(); - if String::from_utf8_lossy(&status.stdout).trim_start().starts_with('Z') { - break; - } - assert!(Instant::now() < deadline, "descendant did not become a zombie"); - std::thread::sleep(Duration::from_millis(10)); - } + let mut status = std::mem::MaybeUninit::::zeroed(); + // SAFETY: status points to writable siginfo storage. WNOWAIT observes + // the exact child exit without consuming its zombie status. + assert_eq!( + unsafe { + libc::waitid( + libc::P_PID, + descendant_pid as libc::id_t, + status.as_mut_ptr(), + libc::WEXITED | libc::WNOWAIT, + ) + }, + 0 + ); } else if scenario == "dead-owned-surface" { let mut command = Command::new("/bin/bash"); command @@ -483,6 +665,7 @@ fn legacy_server_process_helper() { reparent_on_close = Some(command.spawn().unwrap()); } let listener = transport::listen(&socket).unwrap(); + fs::write(std::env::var_os("CMUX_TUI_TEST_LEGACY_READY_SIGNAL").unwrap(), b"1").unwrap(); let serve_identify = |stream: &mut Box| { let mut reader = BufReader::new(stream.try_clone_box().unwrap()); let mut request = String::new(); @@ -1496,29 +1679,27 @@ fi #[test] fn plain_launch_attaches_to_existing_local_session() { let server = HeadlessServer::start("plain-launch-attach"); - let mut tui = PtyChild::start(&["--socket", server.socket.to_str().unwrap()]); - let deadline = Instant::now() + Duration::from_secs(10); + let registered_path = server.dir.join("tui-client-registered"); + let mut registered = TestFifoSignal::create(®istered_path); + let mut tui = PtyChild::start_with_env( + &["--socket", server.socket.to_str().unwrap()], + &[("CMUX_TUI_TEST_CLIENT_REGISTERED_SIGNAL", registered_path.as_os_str())], + ); - while Instant::now() < deadline { - if let Some(status) = tui.child.try_wait().unwrap() { - panic!("plain launch exited instead of attaching: {status}"); - } - let clients = json_cli(&server, &["client", "list"]); - if clients.status.success() { - let clients = json_output(&clients); - if clients - .as_array() - .unwrap() - .iter() - .any(|client| client["client_kind"].as_str() == Some("tui")) - { - return; - } - } - std::thread::sleep(Duration::from_millis(50)); + registered.receive(Duration::from_secs(10)); + if let Some(status) = tui.child.try_wait().unwrap() { + panic!("plain launch exited instead of attaching: {status}"); } - - panic!("plain launch never attached as a TUI client"); + let clients = json_cli(&server, &["client", "list"]); + assert_success(&clients); + assert!( + json_output(&clients) + .as_array() + .unwrap() + .iter() + .any(|client| client["client_kind"].as_str() == Some("tui")), + "plain launch registration signal preceded server client state" + ); } #[cfg(target_os = "linux")] @@ -1552,18 +1733,11 @@ fn plain_launch_preserves_the_existing_session_connect_deadline() { .stderr(Stdio::piped()) .spawn() .unwrap(); - let deadline = Instant::now() + Duration::from_secs(3); - let status = loop { - if let Some(status) = launch.try_wait().unwrap() { - break status; - } - if Instant::now() >= deadline { - let _ = launch.kill(); - let _ = launch.wait(); - panic!("plain launch exceeded the existing-session connection deadline"); - } - std::thread::sleep(Duration::from_millis(10)); - }; + let status = wait_for_child_exit( + &mut launch, + Duration::from_secs(3), + "plain launch exceeded the existing-session connection deadline", + ); assert!(!status.success(), "plain launch replaced a live saturated session"); assert!(socket.exists(), "plain launch unlinked the live saturated session socket"); @@ -1843,22 +2017,14 @@ fn signal_termination_completes_server_shutdown() { wait_for_socket_path(&socket); assert_eq!(unsafe { libc::kill(server.id() as libc::pid_t, libc::SIGTERM) }, 0); - let deadline = Instant::now() + Duration::from_secs(3); - let status = loop { - if let Some(status) = server.try_wait().unwrap() { - break Some(status); - } - if Instant::now() >= deadline { - server.kill().unwrap(); - let _ = server.wait(); - break None; - } - std::thread::sleep(Duration::from_millis(10)); - }; + let status = wait_for_child_exit( + &mut server, + Duration::from_secs(3), + "SIGTERM left the server waiting on an unrequested mux shutdown", + ); let socket_remained = socket.exists(); fs::remove_dir_all(dir).unwrap(); - let status = status.expect("SIGTERM left the server waiting on an unrequested mux shutdown"); assert!(status.success(), "signal-driven shutdown exited with {status}"); assert!(!socket_remained, "signal-driven shutdown retained its control socket"); } @@ -1869,7 +2035,8 @@ fn invalid_websocket_startup_returns_after_publishing_the_local_socket() { let dir = unique_temp_dir("invalid-websocket-startup"); fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); - let published = dir.join("published"); + let published_path = dir.join("published"); + let mut published = TestFifoSignal::create(&published_path); let ghostty = dir.join("ghostty"); fs::write(&ghostty, "#!/bin/sh\nprintf 'background = #272822\\nforeground = #fdfff1\\n'\n") .unwrap(); @@ -1879,36 +2046,23 @@ fn invalid_websocket_startup_returns_after_publishing_the_local_socket() { .arg(&socket) .args(["--ws", "not-an-address"]) .env("GHOSTTY_BIN", &ghostty) - .env("CMUX_TUI_TEST_LOCAL_SOCKET_PUBLISHED_MARKER", &published) + .env("CMUX_TUI_TEST_LOCAL_SOCKET_PUBLISHED_MARKER", &published_path) .stdout(Stdio::null()) .stderr(Stdio::piped()) .spawn() .unwrap(); - let publication_deadline = Instant::now() + Duration::from_secs(15); - while !published.exists() && Instant::now() < publication_deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(published.exists(), "invalid WebSocket setup did not publish its local socket"); - - let deadline = Instant::now() + Duration::from_secs(3); - let status = loop { - if let Some(status) = server.try_wait().unwrap() { - break Some(status); - } - if Instant::now() >= deadline { - server.kill().unwrap(); - let _ = server.wait(); - break None; - } - std::thread::sleep(Duration::from_millis(10)); - }; + published.receive(Duration::from_secs(15)); + let status = wait_for_child_exit( + &mut server, + Duration::from_secs(3), + "invalid WebSocket setup entered the running-server shutdown wait", + ); let socket_remained = socket.exists(); let mut stderr = String::new(); server.stderr.take().unwrap().read_to_string(&mut stderr).unwrap(); fs::remove_dir_all(dir).unwrap(); - let status = status.expect("invalid WebSocket setup entered the running-server shutdown wait"); assert!(!status.success(), "invalid WebSocket setup unexpectedly succeeded"); assert!(stderr.contains("invalid WebSocket address"), "{stderr}"); assert!(!socket_remained, "failed startup retained its local control socket"); @@ -1933,14 +2087,11 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { let response = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "shutdown"})); assert_eq!(response, serde_json::json!({})); - let deadline = Instant::now() + Duration::from_secs(3); - let status = loop { - if let Some(status) = server.child.try_wait().unwrap() { - break status; - } - assert!(Instant::now() < deadline, "server remained alive after acknowledging shutdown"); - std::thread::sleep(Duration::from_millis(10)); - }; + let status = wait_for_child_exit( + &mut server.child, + Duration::from_secs(3), + "server remained alive after acknowledging shutdown", + ); assert!(status.success(), "server exited with {status}"); assert!(transport::connect(&socket).is_err()); drop(server); @@ -1955,8 +2106,16 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { let socket = dir.join("mux.sock"); let state = dir.join("state"); let remote_state = dir.join("remote-state"); - let remote_started = dir.join("remote-started"); - let remote_shutdown = dir.join("remote-shutdown"); + let remote_started_path = dir.join("remote-started"); + let remote_shutdown_started_path = dir.join("remote-shutdown-started"); + let remote_shutdown_release_path = dir.join("remote-shutdown-release"); + let remote_shutdown_complete_path = dir.join("remote-shutdown-complete"); + let server_exit_ready_path = dir.join("server-exit-ready"); + let mut remote_started = TestFifoSignal::create(&remote_started_path); + let mut remote_shutdown_started = TestFifoSignal::create(&remote_shutdown_started_path); + let mut remote_shutdown_release = TestFifoSignal::create(&remote_shutdown_release_path); + let mut remote_shutdown_complete = TestFifoSignal::create(&remote_shutdown_complete_path); + let mut server_exit_ready = TestFifoSignal::create(&server_exit_ready_path); let mut server = PtyChild::start_with_env( &[ "--socket", @@ -1970,44 +2129,34 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { &[ ("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER", std::ffi::OsStr::new("1")), ("CMUX_TUI_TEST_SHUTDOWN_EXIT_GRACE_MS", std::ffi::OsStr::new("100")), - ("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER", remote_started.as_os_str()), - ("CMUX_TUI_TEST_REMOTE_SHUTDOWN_MARKER", remote_shutdown.as_os_str()), - ("CMUX_TUI_TEST_REMOTE_SHUTDOWN_DELAY_MS", std::ffi::OsStr::new("500")), + ("CMUX_TUI_TEST_REMOTE_RUNTIME_STARTED_MARKER", remote_started_path.as_os_str()), + ( + "CMUX_TUI_TEST_REMOTE_SHUTDOWN_STARTED_SIGNAL", + remote_shutdown_started_path.as_os_str(), + ), + ("CMUX_TUI_TEST_REMOTE_SHUTDOWN_GATE", remote_shutdown_release_path.as_os_str()), + ( + "CMUX_TUI_TEST_REMOTE_SHUTDOWN_COMPLETE_SIGNAL", + remote_shutdown_complete_path.as_os_str(), + ), + ("CMUX_TUI_TEST_SERVER_EXIT_READY_SIGNAL", server_exit_ready_path.as_os_str()), ], ); wait_for_socket_path(&socket); - let startup_deadline = Instant::now() + Duration::from_secs(10); - while !remote_started.exists() { - assert!( - server.child.try_wait().unwrap().is_none(), - "server exited before starting its remote runtime" - ); - assert!(Instant::now() < startup_deadline, "remote runtime did not start"); - std::thread::sleep(Duration::from_millis(10)); - } + remote_started.receive(Duration::from_secs(10)); let response = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "shutdown"})); assert_eq!(response, serde_json::json!({})); - std::thread::sleep(Duration::from_millis(250)); - assert_eq!( - fs::read(&remote_shutdown).ok().as_deref(), - Some(b"started".as_slice()), - "remote runtime cleanup did not enter the process completion fence" - ); + remote_shutdown_started.receive(Duration::from_secs(3)); assert!( server.child.try_wait().unwrap().is_none(), "server forced exit before remote runtime cleanup completed" ); - let deadline = Instant::now() + Duration::from_secs(5); - let status = loop { - if let Some(status) = server.child.try_wait().unwrap() { - break status; - } - assert!(Instant::now() < deadline, "server remained alive after remote cleanup"); - std::thread::sleep(Duration::from_millis(10)); - }; - assert_eq!(fs::read(&remote_shutdown).unwrap(), b"complete"); + remote_shutdown_release.send(); + remote_shutdown_complete.receive(Duration::from_secs(5)); + server_exit_ready.receive(Duration::from_secs(5)); + let status = server.child.wait().unwrap(); assert!(status.success(), "server exited with {status}"); drop(server); fs::remove_dir_all(dir).unwrap(); @@ -2056,26 +2205,14 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ ); assert_eq!(response["accepted"], true); - let deadline = Instant::now() + Duration::from_secs(3); - let status = loop { - if let Some(status) = server.child.try_wait().unwrap() { - break status; - } - assert!( - Instant::now() < deadline, - "server remained alive after acknowledging daemon handoff" - ); - std::thread::sleep(Duration::from_millis(10)); - }; + let status = wait_for_child_exit( + &mut server.child, + Duration::from_secs(3), + "server remained alive after acknowledging daemon handoff", + ); assert!(status.success(), "server exited with {status}"); assert!(transport::connect(&socket).is_err()); - let cleanup_deadline = Instant::now() + Duration::from_secs(3); - while (process_exists(local_pid) || process_group_exists(local_pid)) - && Instant::now() < cleanup_deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - let cleaned = !process_exists(local_pid) && !process_group_exists(local_pid); + let cleaned = !process_is_active(local_pid) && !process_group_is_active(local_pid); if !cleaned { let local_pid = libc::pid_t::try_from(local_pid).unwrap(); // SAFETY: local_pid names the isolated PTY session created by this test. @@ -2092,10 +2229,19 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ #[cfg(unix)] #[test] fn server_stop_cancels_a_blocked_terminal_host_launch() { + let blocked_path = unique_temp_dir("server-stop-blocked-launch").with_extension("blocked"); + let gate_path = unique_temp_dir("server-stop-blocked-launch").with_extension("gate"); + let mut blocked = TestFifoSignal::create(&blocked_path); + let _gate = TestFifoSignal::create(&gate_path); + let blocked_value = blocked_path.to_string_lossy().into_owned(); + let gate_value = gate_path.to_string_lossy().into_owned(); let mut server = HeadlessServer::start_with( "server-stop-blocked-launch", false, - &[("CMUX_TUI_TEST_BOOTSTRAP_READY_DELAY_MS", "3000")], + &[ + ("CMUX_TUI_TEST_BOOTSTRAP_READY_STARTED_SIGNAL", &blocked_value), + ("CMUX_TUI_TEST_BOOTSTRAP_READY_GATE", &gate_value), + ], ); let mut create = raw_cli_command( &server.socket, @@ -2106,7 +2252,7 @@ fn server_stop_cancels_a_blocked_terminal_host_launch() { .spawn() .unwrap(); let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); - std::thread::sleep(Duration::from_millis(250)); + blocked.receive(Duration::from_secs(5)); assert!(create.try_wait().unwrap().is_none(), "terminal creation did not remain in flight"); let stop_started = Instant::now(); @@ -2121,15 +2267,27 @@ fn server_stop_cancels_a_blocked_terminal_host_launch() { assert!(server.child.wait().unwrap().success()); assert!(!create.wait().unwrap().success()); assert!(terminal_host_pids(&host_root).is_empty()); + fs::remove_file(blocked_path).unwrap(); + fs::remove_file(gate_path).unwrap(); } #[cfg(unix)] #[test] fn server_stop_cancels_a_blocked_terminal_host_launch_write() { + let blocked_path = + unique_temp_dir("server-stop-blocked-launch-write").with_extension("blocked"); + let gate_path = unique_temp_dir("server-stop-blocked-launch-write").with_extension("gate"); + let mut blocked = TestFifoSignal::create(&blocked_path); + let _gate = TestFifoSignal::create(&gate_path); + let blocked_value = blocked_path.to_string_lossy().into_owned(); + let gate_value = gate_path.to_string_lossy().into_owned(); let mut server = HeadlessServer::start_with( "server-stop-blocked-launch-write", false, - &[("CMUX_TUI_TEST_STALL_AFTER_BOOTSTRAP_READY_MS", "3000")], + &[ + ("CMUX_TUI_TEST_LAUNCH_READ_STARTED_SIGNAL", &blocked_value), + ("CMUX_TUI_TEST_LAUNCH_READ_GATE", &gate_value), + ], ); // Linux caps one execve argument below 128 KiB even when the aggregate // ARG_MAX is larger. This still exceeds the bootstrap pipe capacity, so @@ -2149,7 +2307,7 @@ fn server_stop_cancels_a_blocked_terminal_host_launch_write() { .spawn() .unwrap(); let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); - std::thread::sleep(Duration::from_millis(250)); + blocked.receive(Duration::from_secs(5)); assert!(create.try_wait().unwrap().is_none(), "terminal creation did not block on Launch"); let stop_started = Instant::now(); @@ -2164,12 +2322,15 @@ fn server_stop_cancels_a_blocked_terminal_host_launch_write() { stop_elapsed < Duration::from_secs(2), "shutdown waited for the blocked Launch write: {stop_elapsed:?}" ); + fs::remove_file(blocked_path).unwrap(); + fs::remove_file(gate_path).unwrap(); } #[cfg(unix)] #[test] fn cancelled_published_host_is_terminated_through_its_record() { let barrier = unique_temp_dir("server-stop-published-launch").with_extension("barrier"); + let mut published = TestFifoSignal::create(&barrier); let barrier_value = barrier.to_string_lossy().into_owned(); let mut server = HeadlessServer::start_with( "server-stop-published-launch", @@ -2197,17 +2358,10 @@ fn cancelled_published_host_is_terminated_through_its_record() { .spawn() .unwrap(); let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); - let deadline = Instant::now() + Duration::from_secs(5); - while (!direct_pid_file.exists() - || !descendant_pid_file.exists() - || terminal_host_pids(&host_root).is_empty() - || !barrier.exists()) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } + published.receive(Duration::from_secs(5)); let direct_pid = wait_for_pid_file(&direct_pid_file, Duration::from_secs(5)); let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + assert!(!terminal_host_pids(&host_root).is_empty()); let stop = cli(&server, &["--json", "server", "stop"]); @@ -2215,17 +2369,9 @@ fn cancelled_published_host_is_terminated_through_its_record() { assert!(server.child.wait().unwrap().success()); assert!(!create.wait().unwrap().success()); let _ = fs::remove_file(&barrier); - let deadline = Instant::now() + Duration::from_secs(5); - while (process_exists(direct_pid) - || process_group_exists(direct_pid) - || process_exists(descendant_pid)) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(direct_pid)); - assert!(!process_group_exists(direct_pid)); - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(direct_pid)); + assert!(!process_group_is_active(direct_pid)); + assert!(!process_is_active(descendant_pid)); assert!(terminal_host_pids(&host_root).is_empty()); } @@ -2254,17 +2400,9 @@ fn server_stop_kills_ephemeral_pty_process_groups() { assert_success(&stop); assert!(server.child.wait().unwrap().success()); - let deadline = Instant::now() + Duration::from_secs(5); - while (process_exists(direct_pid) - || process_group_exists(direct_pid) - || process_exists(descendant_pid)) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(direct_pid)); - assert!(!process_group_exists(direct_pid)); - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(direct_pid)); + assert!(!process_group_is_active(direct_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2309,11 +2447,7 @@ fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { assert_success(&stop); assert!(server.child.wait().unwrap().success()); - let deadline = Instant::now() + Duration::from_secs(1); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let descendant_remained = process_exists(descendant_pid); + let descendant_remained = process_is_active(descendant_pid); if descendant_remained { // SAFETY: the live process and group belong to this isolated fixture. let _ = unsafe { libc::killpg(descendant_group, libc::SIGKILL) }; @@ -2377,17 +2511,9 @@ fn server_stop_drains_many_hosted_panes_before_acknowledging() { assert_success(&stop); assert!(stop_started.elapsed() < Duration::from_secs(5)); assert!(server.child.wait().unwrap().success()); - let deadline = Instant::now() + Duration::from_secs(10); - while Instant::now() < deadline - && (host_pids.iter().copied().any(process_exists) - || process_exists(first_terminal_pid) - || process_group_exists(first_terminal_pid)) - { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!host_pids.iter().copied().any(process_exists)); - assert!(!process_exists(first_terminal_pid)); - assert!(!process_group_exists(first_terminal_pid)); + assert!(!host_pids.iter().copied().any(process_is_active)); + assert!(!process_is_active(first_terminal_pid)); + assert!(!process_group_is_active(first_terminal_pid)); assert!(!server.socket.exists()); } @@ -2407,11 +2533,7 @@ fn server_stop_falls_back_when_an_older_server_lacks_shutdown_capability() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2455,11 +2577,7 @@ fn server_stop_waits_for_the_initiating_legacy_client_to_drain() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2528,18 +2646,9 @@ fn published_host_launch_errors_reap_the_complete_pty_session() { let direct_pid = wait_for_pid_file(&direct_pid_file, Duration::from_secs(5)); let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); - let deadline = Instant::now() + Duration::from_secs(5); - while (process_exists(direct_pid) - || process_group_exists(direct_pid) - || process_exists(descendant_pid) - || !terminal_host_pids(&host_root).is_empty()) - && Instant::now() < deadline - { - std::thread::sleep(Duration::from_millis(10)); - } - let cleaned = !process_exists(direct_pid) - && !process_group_exists(direct_pid) - && !process_exists(descendant_pid) + let cleaned = !process_is_active(direct_pid) + && !process_group_is_active(direct_pid) + && !process_is_active(descendant_pid) && terminal_host_pids(&host_root).is_empty(); let stop = cli(&server, &["--json", "server", "stop"]); @@ -2572,12 +2681,8 @@ fn legacy_stop_retains_pty_ownership_captured_before_surface_close() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } assert!( - !process_exists(descendant_pid), + !process_is_active(descendant_pid), "surface close reparented a captured PTY job outside the final server tree" ); } @@ -2598,11 +2703,7 @@ fn server_stop_handles_an_exited_unreaped_legacy_descendant() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2625,11 +2726,7 @@ fn server_stop_reconciles_an_applied_legacy_close_that_returned_an_error() { assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2652,11 +2749,7 @@ fn server_stop_reconnects_after_an_applied_legacy_close_drops_the_control_stream assert_success(&output); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] @@ -2675,10 +2768,6 @@ fn server_stop_completes_when_the_last_legacy_surface_exits_with_its_server() { assert_success(&output); assert!(server.child.wait().unwrap().success()); - let deadline = Instant::now() + Duration::from_secs(5); - while process_is_active(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } assert!( !process_is_active(descendant_pid), "last-surface server exit leaked its captured PTY owner" @@ -2728,11 +2817,7 @@ fn detached_legacy_stop_survives_the_calling_pane_process_exit() { assert_eq!(output.status.signal(), Some(libc::SIGKILL)); let status = server.child.wait().unwrap(); assert_eq!(status.signal(), Some(libc::SIGKILL)); - let deadline = Instant::now() + Duration::from_secs(5); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - assert!(!process_exists(descendant_pid)); + assert!(!process_is_active(descendant_pid)); } #[cfg(unix)] diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index ef9f8143d4ac..18fc8d8966eb 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -1,7 +1,8 @@ #![cfg(unix)] -use std::fs; -use std::io::{BufRead, BufReader, Write}; +use std::fs::{self, File}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::os::fd::AsRawFd; use std::os::unix::net::UnixStream; use std::os::unix::process::CommandExt; use std::path::{Path, PathBuf}; @@ -26,6 +27,28 @@ use cmux_tui_core::terminal_host_runtime::{ }; use ghostty_vt::{Rgb, TerminalColorOverrides}; +struct TestFifoSignal(File); + +impl TestFifoSignal { + fn create(path: &Path) -> Self { + let path_name = std::ffi::CString::new(path.as_os_str().as_encoded_bytes()).unwrap(); + // SAFETY: path_name is a valid, NUL-terminated path in the test directory. + assert_eq!(unsafe { libc::mkfifo(path_name.as_ptr(), 0o600) }, 0); + Self(fs::OpenOptions::new().read(true).write(true).open(path).unwrap()) + } + + fn receive(&mut self, timeout: Duration) { + let timeout = timeout.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut descriptor = + libc::pollfd { fd: self.0.as_raw_fd(), events: libc::POLLIN, revents: 0 }; + // SAFETY: descriptor points to one initialized pollfd for this call. + assert_eq!(unsafe { libc::poll(&raw mut descriptor, 1, timeout) }, 1); + assert_ne!(descriptor.revents & libc::POLLIN, 0); + let mut byte = [0_u8; 1]; + self.0.read_exact(&mut byte).unwrap(); + } +} + fn test_timeout_scale() -> u32 { std::env::var("CMUX_TEST_TIMEOUT_SCALE") .ok() @@ -134,6 +157,14 @@ impl RecoveryHarness { wait_for_socket(&self.socket); } + fn restart_with_adoption_signal(&mut self, signal: &Path) { + assert!(self.child.is_none()); + let mut command = self.daemon_command(); + command.env("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL", signal); + self.child = Some(command.spawn().unwrap()); + wait_for_socket(&self.socket); + } + fn daemon_command(&self) -> Command { let mut command = Command::new(bin()); command @@ -663,11 +694,7 @@ fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { host.terminate().unwrap(); host.disconnect(); wait_for_no_host_records(&harness.host_root()); - let deadline = Instant::now() + Duration::from_secs(1); - while process_exists(descendant_pid) && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); - } - let descendant_remained = process_exists(descendant_pid); + let descendant_remained = process_is_active(descendant_pid); if descendant_remained { // SAFETY: the live process and group belong to this isolated fixture. let _ = unsafe { libc::killpg(descendant_group, libc::SIGKILL) }; @@ -1649,8 +1676,11 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { assert!(endpoint.exists(), "terminal host socket disappeared with the daemon"); assert_eq!(wait_for_host_records(&harness.host_root(), 1)[0].1.terminal_id, terminal_id); - harness.restart(); - let adopted_surface = wait_for_adopted_terminal( + let adoption_signal_path = harness.dir.join("terminal-adopted"); + let mut adoption_signal = TestFifoSignal::create(&adoption_signal_path); + harness.restart_with_adoption_signal(&adoption_signal_path); + adoption_signal.receive(Duration::from_secs(15)); + let adopted_surface = adopted_terminal( &harness.socket, &terminal_id, &incarnation, @@ -1717,8 +1747,11 @@ fn terminal_host_survives_sigkill_and_is_adopted_with_io_and_size() { // A second crash proves the resize landed in the PTY-owning process, not // only in the disposable daemon-side Ghostty mirror. harness.sigkill(); - harness.restart(); - let resized_surface = wait_for_adopted_terminal( + let adoption_signal_path = harness.dir.join("resized-terminal-adopted"); + let mut adoption_signal = TestFifoSignal::create(&adoption_signal_path); + harness.restart_with_adoption_signal(&adoption_signal_path); + adoption_signal.receive(Duration::from_secs(15)); + let resized_surface = adopted_terminal( &harness.socket, &terminal_id, &incarnation, @@ -2099,6 +2132,7 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { .unwrap(); let mut renderer_writer = renderer.stream.try_clone().unwrap(); let drained = Arc::new(AtomicUsize::new(0)); + let (drain_progress_tx, drain_progress_rx) = std::sync::mpsc::channel(); let drain_count = drained.clone(); let drain = std::thread::spawn(move || { let mut renderer = renderer; @@ -2110,7 +2144,9 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { renderer.next_sequence = renderer.next_sequence.wrapping_add(1); } if frame.kind == MessageKind::Output { - drain_count.fetch_add(frame.payload.len(), Ordering::AcqRel); + let drained = drain_count.fetch_add(frame.payload.len(), Ordering::AcqRel) + + frame.payload.len(); + drain_progress_tx.send(drained).unwrap(); } } Ok(None) | Err(ProtocolError::Truncated { .. }) | Err(ProtocolError::Io(_)) => { @@ -2133,8 +2169,11 @@ fn daemon_admin_backpressure_reconnects_without_restarting_host_or_renderer() { .unwrap(); let expected = chunk * 1_000_000; let deadline = Instant::now() + scaled_timeout(Duration::from_secs(15)); - while drained.load(Ordering::Acquire) < expected && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(10)); + while drained.load(Ordering::Acquire) < expected { + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("renderer did not drain output before its deadline"); + drain_progress_rx.recv_timeout(remaining).unwrap(); } assert!( drained.load(Ordering::Acquire) >= expected, @@ -3064,58 +3103,31 @@ fn wait_for_screen(path: &Path, surface: u64, marker: &str) -> String { } fn wait_for_stream_disconnect(stream: &mut UnixStream, timeout: Duration) -> bool { - let deadline = Instant::now() + timeout; - let previous_timeout = stream.write_timeout().ok().flatten(); - let disconnected = loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - break false; - } - if stream.set_write_timeout(Some(remaining.min(Duration::from_millis(50)))).is_err() { - break true; - } - // ReleaseViewer is valid and idempotent for this renderer but produces - // no host-to-client frame, so it probes only the input half of the - // connection while output remains deliberately unread. - match write_frame(stream, &Frame::new(MessageKind::ReleaseViewer, Vec::new())) { - Ok(()) => {} - Err(ProtocolError::Io(error)) - if matches!( - error.kind(), - std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut - ) => {} - Err(_) => break true, - } - std::thread::sleep(remaining.min(Duration::from_millis(10))); - }; - let _ = stream.set_write_timeout(previous_timeout); - disconnected + let timeout = timeout.as_millis().clamp(1, i32::MAX as u128) as i32; + let mut descriptor = libc::pollfd { fd: stream.as_raw_fd(), events: 0, revents: 0 }; + // SAFETY: descriptor points to the live renderer socket for this call. + match unsafe { libc::poll(&raw mut descriptor, 1, timeout) } { + 1 => descriptor.revents & (libc::POLLERR | libc::POLLHUP | libc::POLLNVAL) != 0, + 0 => false, + _ => panic!("renderer disconnect observer failed: {}", std::io::Error::last_os_error()), + } } -fn wait_for_adopted_terminal( - path: &Path, - terminal_id: &str, - incarnation: &str, - failure: &str, -) -> u64 { - let deadline = Instant::now() + Duration::from_secs(15); - loop { - let resolved = request( - path, - serde_json::json!({ - "cmd": "resolve-terminal", - "terminal_id": terminal_id, - }), - ); - if resolved["lifecycle"] == "running" - && resolved["terminal_incarnation"].as_str() == Some(incarnation) - && let Some(surface) = resolved["surface"].as_u64() - { - return surface; - } - assert!(Instant::now() < deadline, "{failure}: {resolved}"); - std::thread::sleep(Duration::from_millis(50)); - } +fn adopted_terminal(path: &Path, terminal_id: &str, incarnation: &str, failure: &str) -> u64 { + let resolved = request( + path, + serde_json::json!({ + "cmd": "resolve-terminal", + "terminal_id": terminal_id, + }), + ); + assert_eq!(resolved["lifecycle"], "running", "{failure}: {resolved}"); + assert_eq!( + resolved["terminal_incarnation"].as_str(), + Some(incarnation), + "{failure}: {resolved}" + ); + resolved["surface"].as_u64().unwrap_or_else(|| panic!("{failure}: {resolved}")) } fn wait_for_host_records(root: &Path, expected: usize) -> Vec<(PathBuf, TerminalHostRecord)> { @@ -3213,6 +3225,15 @@ fn process_exists(pid: libc::pid_t) -> bool { || std::io::Error::last_os_error().kind() == std::io::ErrorKind::PermissionDenied } +fn process_is_active(pid: libc::pid_t) -> bool { + let output = Command::new("ps") + .args(["-o", "stat=", "-p", &pid.to_string()]) + .output() + .expect("inspect process state"); + output.status.success() + && output.stdout.iter().copied().find(|byte| !byte.is_ascii_whitespace()) != Some(b'Z') +} + fn wait_for_host_size(root: &Path, cols: u16, rows: u16) { let deadline = Instant::now() + Duration::from_secs(10); loop { From 845a4022c25c3d26673062fae7b195824b560042 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:35:29 -0700 Subject: [PATCH 349/521] ci(tui): run Linux tests on Blacksmith --- .github/workflows/cmux-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 5667c507ea6b..e8c5cf76de11 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -286,7 +286,7 @@ jobs: test: name: test (${{ matrix.os }}) - runs-on: ${{ matrix.os == 'macos' && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || 'ubuntu-latest' }} + runs-on: ${{ matrix.os == 'macos' && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || (vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }} timeout-minutes: 40 strategy: fail-fast: false From f52c5b263fbe4b0f5b84092ea8f51e21e542f9ef Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:36:09 -0700 Subject: [PATCH 350/521] ci(tui): pin Blacksmith test runners --- .github/workflows/cmux-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index e8c5cf76de11..939675ed1c8e 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -286,7 +286,7 @@ jobs: test: name: test (${{ matrix.os }}) - runs-on: ${{ matrix.os == 'macos' && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || (vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }} + runs-on: ${{ matrix.os == 'macos' && 'blacksmith-6vcpu-macos-15' || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 40 strategy: fail-fast: false From d2c87a7221b5e16d274675223d622eb78432358f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:36:35 -0700 Subject: [PATCH 351/521] ci(tui): preserve macOS runner routing --- .github/workflows/cmux-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 939675ed1c8e..e8c5cf76de11 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -286,7 +286,7 @@ jobs: test: name: test (${{ matrix.os }}) - runs-on: ${{ matrix.os == 'macos' && 'blacksmith-6vcpu-macos-15' || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ matrix.os == 'macos' && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || (vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }} timeout-minutes: 40 strategy: fail-fast: false From 702109ad92673dede9587f71b93d5b9ca967bf1b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:38:03 -0700 Subject: [PATCH 352/521] ci(tui): allow Blacksmith macOS dispatch --- .github/workflows/cmux-tui.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index e8c5cf76de11..771847ea9bf2 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -3,6 +3,12 @@ name: cmux-tui on: # Temporarily manual-only beginning 2026-07-13 to pause automatic CI. workflow_dispatch: + inputs: + use_blacksmith_macos: + description: "Run the macOS test matrix on Blacksmith" + required: false + type: boolean + default: false concurrency: group: cmux-tui-${{ github.workflow }}-${{ github.ref }} @@ -286,7 +292,7 @@ jobs: test: name: test (${{ matrix.os }}) - runs-on: ${{ matrix.os == 'macos' && (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') || (vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }} + runs-on: ${{ matrix.os == 'macos' && (inputs.use_blacksmith_macos && 'blacksmith-6vcpu-macos-15' || (vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15')) || (vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }} timeout-minutes: 40 strategy: fail-fast: false From 4c46cebeb717e97f3fd84020dd3abadd50425b9e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:41:30 -0700 Subject: [PATCH 353/521] ci(tui): check the Valgrind execution job --- tests/test_tui_workflow_toolchain.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py index 0e8565205386..c7f13c2088f4 100644 --- a/tests/test_tui_workflow_toolchain.py +++ b/tests/test_tui_workflow_toolchain.py @@ -5,8 +5,9 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "cmux-tui.yml" VALGRIND_PIDFD_SKIPS = ROOT / "cmux-tui" / "dist" / "valgrind-pidfd-skips.txt" +VALGRIND_EXECUTION_JOB = "valgrind-leak-check-shard" RUST_JOBS = ( - "valgrind-leak-check", + VALGRIND_EXECUTION_JOB, "test", "bindings-e2e", "windows-experimental", @@ -75,7 +76,7 @@ def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: test_name for scope, test_name in entries if scope == "cmux_tui_core" } - valgrind_job = job_block(WORKFLOW.read_text(), "valgrind-leak-check") + valgrind_job = job_block(WORKFLOW.read_text(), VALGRIND_EXECUTION_JOB) assert "Skipping pidfd-dependent test binary" not in valgrind_job assert "dist/valgrind-pidfd-skips.txt" in valgrind_job assert "test_scope=browser_runtime" in valgrind_job From dd79103fc434df2e9c31387d6f2effbc6fce89e1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 19:58:15 -0700 Subject: [PATCH 354/521] fix(tui): restore cross-platform compilation Resolve the shared hosted compile diagnostics without changing lifecycle ownership or adding timing-based synchronization. --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + .../cmux-tui-core/src/mux/resource_topology.rs | 12 ++++++------ cmux-tui/crates/cmux-tui-core/src/surface.rs | 2 +- .../cmux-tui-core/src/terminal_host_runtime.rs | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fe608b5d60fb..f7691c708849 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -4159,6 +4159,7 @@ impl Mux { Ok(binding) => binding, Err(_) => return false, }; + #[cfg(test)] let id = restored_binding .as_ref() .and_then(|binding| binding.placements.first().map(|(slot, _)| *slot)) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 21c136ab2cf5..b349e9e2fb95 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2456,6 +2456,11 @@ impl Mux { terminal_host_cleanup_receipt(&workspace_host_identities), ); } + let closed_workspace_terminals = if closed_workspace_key.is_some() { + Self::terminal_public_ids_for_hosted(®istry, &workspace_host_identities)? + } else { + Vec::new() + }; let cleanup_identities = if closed_workspace_key.is_some() { workspace_host_identities } else { @@ -2475,16 +2480,11 @@ impl Mux { TERMINAL_HOST_CLEANUP_RECEIPT_FIELD.to_string(), terminal_host_cleanup_receipt(&cleanup_identities), ); - let closed_workspace_terminals = if closed_workspace_key.is_some() { - Self::terminal_public_ids_for_hosted(®istry, &workspace_host_identities)? - } else { - Vec::new() - }; #[cfg(test)] if let Some(hook) = self.resource_projection_before_commit.lock().unwrap().clone() { hook(); } - let mut close = registry.commit_resource_close_patch( + let close = registry.commit_resource_close_patch( idempotency_key, operation_name, fingerprint, diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index f6ad090c8fc9..bcefff8d0c88 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -34,9 +34,9 @@ use crate::MuxEvent; use crate::mux::ResourceWaitWake; use crate::platform; use crate::resource::{ContentPublicId, TabResourceIdentity, TerminalPublicId}; +use crate::terminal_host_protocol::TerminalExit; #[cfg(not(unix))] use crate::terminal_host_protocol::TerminalExitOutcome; -use crate::terminal_host_protocol::{TerminalExit, wait_for_native_child_status}; use crate::{Mux, SurfaceId}; pub use crate::browser::{ diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 16624956311e..03123ae6f77f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -33,7 +33,7 @@ use crate::terminal_host_protocol::{ KITTY_IMAGE_ALIAS_ENCODED_LEN, MAX_FRAME_PAYLOAD, MAX_KITTY_IMAGE_ALIASES, MessageKind, PROTOCOL_VERSION, RESIZE_ACK_CANONICAL_CHANGED, TerminalExit, decode_host_launch_failure, decode_terminal_exit, encode_host_launch_failure, encode_terminal_exit, read_frame, - wait_for_native_child_status, write_frame, + write_frame, }; const HOST_RECORD_VERSION: u32 = 3; From 899a7c4903fe42cf0e67c551712a4ac7812b80f5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:15:14 -0700 Subject: [PATCH 355/521] fix(tui): repair exact-head review and CI findings Move blocking native teardown onto bounded dedicated threads, update SDK inventory checks, and reconcile hosted clippy diagnostics without timing-based synchronization. --- ...nalSurfaceRuntimeTeardownCoordinator.swift | 26 ++-- ...lSurfaceRuntimeTeardownExecutionLane.swift | 23 ++- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 2 - ...rfaceRuntimeTeardownCoordinatorTests.swift | 135 ++++++++++++++++++ cmux-tui/bindings/ERGONOMICS.md | 4 +- .../bindings/cpp/tests/test_generated.cpp | 2 +- cmux-tui/bindings/go/raw/README.md | 2 +- cmux-tui/bindings/go/raw/client_test.go | 4 +- .../com/cmux/raw/GeneratedCoverageTest.java | 2 +- .../bindings/python/tests/test_protocol.py | 2 +- .../typescript/test/generated.test.ts | 2 +- cmux-tui/bindings/zig/examples/watch.zig | 2 +- cmux-tui/bindings/zig/src/raw.zig | 2 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 82 +---------- .../src/mux/public_projections.rs | 2 +- .../crates/cmux-tui-core/src/resource_api.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/server.rs | 20 +-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 23 ++- .../src/terminal_host_protocol.rs | 1 + .../src/terminal_host_runtime.rs | 5 +- 20 files changed, 214 insertions(+), 129 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 707546119f51..5809a5166b3f 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -421,16 +421,22 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? ) { - let prepared = executionLane.prepare { - self.freeNativeSurface(request) - await self.finishTeardown( - request, - executionLane: executionLane, - executionSlot: executionSlot, - isolatedHibernationReservation: - isolatedHibernationReservation - ) - } + // This method is reachable only after one of two close slots or two + // isolated-hibernation slots is admitted, which caps worker threads at 4. + let prepared = executionLane.prepare( + operation: { + self.freeNativeSurface(request) + }, + completion: { + await self.finishTeardown( + request, + executionLane: executionLane, + executionSlot: executionSlot, + isolatedHibernationReservation: + isolatedHibernationReservation + ) + } + ) let active = TerminalSurfaceRuntimeActiveTeardown( ticketID: request.ticketID, task: prepared.task diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift index c8d569436798..540fade6667b 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -1,3 +1,4 @@ +internal import Foundation internal import os /// Selects the ownership boundary for a native surface free. @@ -13,13 +14,26 @@ nonisolated enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { /// The checked-continuation gate makes storage happen before execution. It /// intentionally does not treat Task cancellation as permission to skip a /// native free, because the request owns the surface until free completes. + /// The coordinator calls this only after slot admission, so buffered + /// submissions never create threads. Each admitted Task creates one Thread, + /// runs one blocking free, resumes once, and lets that Thread return. nonisolated func prepare( - operation: @escaping @Sendable () async -> Void + operation: @escaping @Sendable () -> Void, + completion: @escaping @Sendable () async -> Void ) -> TerminalSurfaceRuntimePreparedTeardownExecution { let startGate = TerminalSurfaceRuntimeTeardownStartGate() - let task = Task.detached(priority: .utility) { + let task = Task(priority: .utility) { await startGate.wait() - await operation() + await withCheckedContinuation { continuation in + let thread = Thread { + operation() + continuation.resume() + } + thread.name = "com.cmux.runtime-teardown" + thread.qualityOfService = .utility + thread.start() + } + await completion() } return TerminalSurfaceRuntimePreparedTeardownExecution( task: task, @@ -46,6 +60,9 @@ nonisolated struct TerminalSurfaceRuntimePreparedTeardownExecution: Sendable { } } +/// Safety: the lock protects every state transition and the stored continuation. +/// The single waiter and idempotent start move that continuation out at most +/// once, and resume it only after releasing the lock. private final class TerminalSurfaceRuntimeTeardownStartGate: @unchecked Sendable { private enum State { case pending diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 5a534dcefd9d..f07ca3618117 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -192,7 +192,6 @@ import CmuxTerminalCore @Test func runtimeCreationRetriesAfterTeardownAdmissionRecovers() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( - closeTeardownTimeout: .milliseconds(50), maximumRuntimeSurfaceOwnerCount: 4 ) let retainedSurfaces = (0..<2).map { _ in @@ -267,7 +266,6 @@ import CmuxTerminalCore @Test func restoredCreationReentersPacingAfterAdmissionRecovers() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( - closeTeardownTimeout: .milliseconds(50), maximumRuntimeSurfaceOwnerCount: 4 ) let retainedSurfaces = (0..<2).map { _ in diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index a9d89789caeb..b016087290ac 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -109,6 +109,141 @@ private func requireTeardownTicket( #expect(recorder.freed == [UInt(bitPattern: surface)]) } + @MainActor + @Test func nativeFreeWorkersUseDedicatedThreadsAndStayBoundedByAdmittedSlots() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let surfaces = (0..<5).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + defer { for surface in surfaces { surface.deallocate() } } + let runtimeReservations = try (0...makeStream() + let releaseWorkers = (0..() + while firstStartedIndices.count < 4 { + firstStartedIndices.insert( + try #require(await workerStartedIterator.next()) + ) + } + #expect(firstStartedIndices == Set(0..<4)) + #expect(await coordinator.reserveIsolatedHibernationTeardown() == nil) + + let fifthTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.dedicatedWorker.4", + surface: surfaces[4], + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[4], + freeSurface: { _ in + let hasCurrentTask = withUnsafeCurrentTask { $0 != nil } + workerState.withLock { state in + state.active += 1 + state.maximumActive = max( + state.maximumActive, + state.active + ) + state.observedCurrentTask = + state.observedCurrentTask || hasCurrentTask + state.startsByIndex[4, default: 0] += 1 + } + workerStarted.continuation.yield(4) + releaseWorkers[4].wait() + workerState.withLock { $0.active -= 1 } + } + ) + ) + tickets.append(fifthTicket) + + releaseWorkers[0].signal() + let fifthStartedIndex = try #require( + await workerStartedIterator.next() + ) + #expect(fifthStartedIndex == 4) + for index in 1..); static_assert(std::is_move_constructible_v); -constexpr std::size_t kExpectedRawCommandCount = 97U; +constexpr std::size_t kExpectedRawCommandCount = 98U; constexpr std::array kViewportHistoryCommandNames{ "clear-history", "new-pane-right", diff --git a/cmux-tui/bindings/go/raw/README.md b/cmux-tui/bindings/go/raw/README.md index 2a672fc17741..d207a021ac50 100644 --- a/cmux-tui/bindings/go/raw/README.md +++ b/cmux-tui/bindings/go/raw/README.md @@ -1,6 +1,6 @@ # cmux Go SDK -The Go SDK covers all 97 protocol-11 commands and 45 event shapes using only +The Go SDK covers all 98 protocol-11 commands and 46 event shapes using only the Go standard library. ```go diff --git a/cmux-tui/bindings/go/raw/client_test.go b/cmux-tui/bindings/go/raw/client_test.go index bcd6fb695119..54fbd91a7f94 100644 --- a/cmux-tui/bindings/go/raw/client_test.go +++ b/cmux-tui/bindings/go/raw/client_test.go @@ -20,8 +20,8 @@ import ( func TestGeneratedInventoryHasTypedMethodForEveryCommand(t *testing.T) { commands := AllCommandMetadata() - if len(commands) != 97 { - t.Fatalf("generated commands = %d, want 97", len(commands)) + if len(commands) != 98 { + t.Fatalf("generated commands = %d, want 98", len(commands)) } clientType := reflect.TypeOf((*Client)(nil)) commandNames := make(map[string]struct{}, len(commands)) diff --git a/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java b/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java index 90106e6aee0b..ef1b4b80e3f3 100644 --- a/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java +++ b/cmux-tui/bindings/java/tests/com/cmux/raw/GeneratedCoverageTest.java @@ -20,7 +20,7 @@ public final class GeneratedCoverageTest { public static void main(String[] args) throws Exception { check(Protocol.VERSION == 11, "protocol version"); check("1.0.0".equals(Protocol.SDK_VERSION), "SDK release version"); - check(Commands.ALL.size() == 97, "all 97 commands generated"); + check(Commands.ALL.size() == 98, "all 98 commands generated"); check(Events.ALL.size() == 46, "all 46 events generated"); Map methods = Arrays.stream(GeneratedCmuxClient.class.getDeclaredMethods()) diff --git a/cmux-tui/bindings/python/tests/test_protocol.py b/cmux-tui/bindings/python/tests/test_protocol.py index 020cd8d1287f..818eca2a1b96 100644 --- a/cmux-tui/bindings/python/tests/test_protocol.py +++ b/cmux-tui/bindings/python/tests/test_protocol.py @@ -22,7 +22,7 @@ class GeneratedProtocolTests(unittest.TestCase): def test_protocol_inventory_is_exhaustive(self) -> None: self.assertEqual(MUX_PROTOCOL, 11) - self.assertEqual(len(COMMANDS), 97) + self.assertEqual(len(COMMANDS), 98) self.assertEqual(set(COMMANDS), set(SCHEMA["commands"])) self.assertEqual(set(EVENTS), set(SCHEMA["events"])) self.assertEqual(len(IR_SHA256), 64) diff --git a/cmux-tui/bindings/typescript/test/generated.test.ts b/cmux-tui/bindings/typescript/test/generated.test.ts index 150a3267738d..7fc3e10fa4cc 100644 --- a/cmux-tui/bindings/typescript/test/generated.test.ts +++ b/cmux-tui/bindings/typescript/test/generated.test.ts @@ -12,7 +12,7 @@ import { test("generated protocol coverage matches the canonical v11 IR", () => { assert.equal(MUX_PROTOCOL_VERSION, 11); assert.equal(SDK_SCHEMA_VERSION, 2); - assert.equal(Object.keys(COMMAND_METADATA).length, 97); + assert.equal(Object.keys(COMMAND_METADATA).length, 98); assert.equal(Object.keys(EVENT_METADATA).length, 46); assert.equal(SDK_IR_SHA256.length, 64); assert.deepEqual(Object.keys(PROFILES).sort(), [ diff --git a/cmux-tui/bindings/zig/examples/watch.zig b/cmux-tui/bindings/zig/examples/watch.zig index 5707965de4a7..c2abc29d72b9 100644 --- a/cmux-tui/bindings/zig/examples/watch.zig +++ b/cmux-tui/bindings/zig/examples/watch.zig @@ -126,7 +126,7 @@ test "package consumer imports handwritten root and generated raw module" { wheel.pointer_frame_seq, ); try std.testing.expectEqual( - @as(usize, 97), + @as(usize, 98), cmux.raw.protocol.command_count, ); try std.testing.expectEqual( diff --git a/cmux-tui/bindings/zig/src/raw.zig b/cmux-tui/bindings/zig/src/raw.zig index 083e58a22679..6b0238027daa 100644 --- a/cmux-tui/bindings/zig/src/raw.zig +++ b/cmux-tui/bindings/zig/src/raw.zig @@ -44,7 +44,7 @@ test { _ = @import("raw/wire_presence_test.zig"); _ = @import("raw/generated/presence_test.zig"); std.testing.refAllDecls(protocol); - try std.testing.expectEqual(@as(usize, 97), protocol.command_count); + try std.testing.expectEqual(@as(usize, 98), protocol.command_count); for ([_][]const u8{ "browser-frame-presented", "browser-key-press", diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f7691c708849..93017c489e33 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2762,6 +2762,7 @@ pub struct Mux { discovered_terminal_termination_requests: Mutex)>>, #[cfg(test)] terminal_host_cleanup_hook: Mutex bool + Send + Sync>>>, + #[cfg(test)] resource_close_cleanup: Mutex>>, browser_runtime: BrowserRuntimeSlot, active_render_attachments: Arc, @@ -3132,6 +3133,7 @@ impl Mux { discovered_terminal_termination_requests: Mutex::new(Vec::new()), #[cfg(test)] terminal_host_cleanup_hook: Mutex::new(None), + #[cfg(test)] resource_close_cleanup: Mutex::new(None), browser_runtime: BrowserRuntimeSlot::default(), active_render_attachments: Arc::new(AtomicUsize::new(0)), @@ -8369,14 +8371,6 @@ impl Mux { std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) } - fn prepare_terminal_host_cleanup( - &self, - terminals: &[(String, Option)], - ) -> anyhow::Result { - let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - self.prepare_terminal_host_cleanup_at_root(root.as_deref(), terminals) - } - fn prepare_terminal_host_cleanup_at_root( &self, root: Option<&Path>, @@ -16236,65 +16230,6 @@ fn acknowledge_exact_terminal_host_exit( } } -#[cfg(unix)] -fn acknowledge_terminal_exit_sidecar( - record_path: &Path, - terminal_id: &str, - incarnation: Option<&str>, -) -> bool { - let exit_path = record_path.with_extension("exit"); - let exit = match crate::terminal_host_runtime::terminal_host_exit_record(record_path) { - Ok(Some((_, exit))) => exit, - Ok(None) => return !exit_path.exists(), - Err(_) => return false, - }; - if exit.terminal_id != terminal_id - || incarnation.is_some_and(|expected| exit.incarnation != expected) - { - return false; - } - match crate::terminal_host_runtime::acknowledge_terminal_host_exit_record(&exit_path, &exit) { - Ok(true) => true, - Ok(false) => !exit_path.exists(), - Err(_) => false, - } -} - -#[cfg(unix)] -fn schedule_terminal_host_record_cleanup( - record: crate::terminal_host_runtime::TerminalHostRecord, - record_path: std::path::PathBuf, -) { - let fallback_record = record.clone(); - let fallback_path = record_path.clone(); - let name = format!("terminal-clean-{}", record.terminal_id); - if std::thread::Builder::new() - .name(name) - .spawn(move || retry_terminal_host_record_cleanup(record, record_path)) - .is_err() - { - // Thread exhaustion cannot turn a durable close into a permanent - // orphan. The request may remain blocked, but the exact host keeps - // being reconciled until it accepts termination or proves itself dead. - retry_terminal_host_record_cleanup(fallback_record, fallback_path); - } -} - -#[cfg(unix)] -fn retry_terminal_host_record_cleanup( - record: crate::terminal_host_runtime::TerminalHostRecord, - record_path: std::path::PathBuf, -) { - let mut delay = Duration::from_millis(25); - loop { - std::thread::sleep(delay); - if cleanup_terminal_host_record(&record, &record_path) { - return; - } - delay = (delay * 2).min(Duration::from_secs(5)); - } -} - #[cfg(unix)] fn terminal_host_record_liveness( record_path: &Path, @@ -21796,7 +21731,7 @@ mod tests { } assert_eq!(surface.size(), (80, 40)); mux.set_client_resize_before_apply(None); - mux.shutdown(); + let _ = mux.shutdown(); } fn projected_terminal_view(mux: &Arc, source: &Arc) -> Arc { @@ -22060,7 +21995,7 @@ mod tests { exited.insert(surface); } } - mux.shutdown(); + let _ = mux.shutdown(); } #[test] @@ -26398,10 +26333,7 @@ mod tests { } let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); - let close = std::thread::spawn({ - let mux = mux.clone(); - move || result_tx.send(mux.close_pane(pane)).unwrap() - }); + let close = std::thread::spawn(move || result_tx.send(mux.close_pane(pane)).unwrap()); assert!( shutdown_gate.wait_until_entered( SHUTDOWN_FANOUT_WORKERS, @@ -28455,7 +28387,7 @@ mod tests { && change["resource"] == "terminal" && change["id"] == terminal_public_id.as_str() })); - mux.shutdown(); + let _ = mux.shutdown(); drop(mux); let reopened = Mux::open_persistent(session, options, &root).unwrap(); @@ -28555,7 +28487,7 @@ mod tests { assert_eq!(exited.surface, None); assert_eq!(exited.terminal.lifecycle, TerminalLifecycle::Exited); assert_eq!(exited.terminal.exit.unwrap()["outcome"]["reason"], "persisted-exit"); - reopened.shutdown(); + let _ = reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); } diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs index c7303a7632c3..fbabf933ab99 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/public_projections.rs @@ -187,7 +187,7 @@ mod tests { assert_eq!(restored.notification_ledger[0].terminal_id.as_ref(), Some(&terminal)); assert_eq!(restored.notification_ledger[0].surface, Some(runtime.id)); assert!(restored.terminal_notifications[&terminal].unread); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_api.rs b/cmux-tui/crates/cmux-tui-core/src/resource_api.rs index b1de70c84977..7a8001700d65 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_api.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_api.rs @@ -1001,7 +1001,7 @@ mod tests { assert_eq!(terminal["tab_id"], Value::Null); assert_eq!(terminal["tab_ids"], json!([])); assert!(terminal["exit"].is_object()); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 216ca6e79f58..8de8bd389e3e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4704,10 +4704,10 @@ impl Drop for WebSocketServer { fn drop(&mut self) { self.shutdown.store(true, Ordering::Release); let _ = self.try_close_connections(); - if let Some(thread) = self.thread.take() { - if thread.is_finished() { - let _ = thread.join(); - } + if let Some(thread) = self.thread.take() + && thread.is_finished() + { + let _ = thread.join(); } } } @@ -12736,7 +12736,7 @@ mod tests { assert!(response.get("result").is_none()); disconnect_client(&mux, client, false); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] @@ -13339,7 +13339,7 @@ mod tests { assert!(disconnect_client(&first, client, false)); drop(scheduler); drop(writer); - first.shutdown(); + let _ = first.shutdown(); let shutdown_deadline = Instant::now() + Duration::from_secs(10); while Arc::strong_count(&first) > 1 && Instant::now() < shutdown_deadline { std::thread::sleep(Duration::from_millis(10)); @@ -13372,7 +13372,7 @@ mod tests { assert_eq!(response["result"]["exited_at"], "4567890"); assert!(disconnect_client(&reopened, client, false)); - reopened.shutdown(); + let _ = reopened.shutdown(); drop(scheduler); drop(writer); drop(reopened); @@ -16801,7 +16801,7 @@ mod tests { 1 ); assert!(disconnect_client(&mux, second_client, true)); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] @@ -16878,7 +16878,7 @@ mod tests { assert_eq!(replay["surface"].as_u64(), Some(created)); assert!(disconnect_client(&mux, replay_client, true)); mux.close_surface(created).unwrap(); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] @@ -16968,7 +16968,7 @@ mod tests { assert_eq!(response["outcome"], "superseded"); assert_eq!(response["accepted"], false); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index bcefff8d0c88..453655b52cb3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -2730,7 +2730,6 @@ impl Surface { mux, None, None, - None, PtyLifetime::DaemonOwned, cell_pixels, ) @@ -2747,7 +2746,6 @@ impl Surface { opts, mux, None, - None, Some(TabResourceIdentity::terminal(None)?), PtyLifetime::SessionOwned, cell_pixels, @@ -2767,8 +2765,7 @@ impl Surface { id, opts, mux, - Some(terminal_id), - Some(publication_guard), + Some((terminal_id, publication_guard)), identity, PtyLifetime::SessionOwned, cell_pixels, @@ -2789,7 +2786,6 @@ impl Surface { opts, mux, None, - None, resource_identity, PtyLifetime::SessionOwned, cell_pixels, @@ -2800,8 +2796,10 @@ impl Surface { id: SurfaceId, opts: SurfaceOptions, mux: Weak, - terminal_id: Option, - publication_guard: Option, + terminal_host_ownership: Option<( + crate::terminal_host::TerminalId, + crate::PublicationGuard, + )>, resource_identity: Option, lifetime: PtyLifetime, cell_pixels: (u16, u16), @@ -2829,8 +2827,8 @@ impl Surface { { let default_colors = mux.upgrade().map(|mux| mux.default_colors()).unwrap_or_default(); let launch_cancelled = || mux.upgrade().is_none_or(|mux| mux.is_shutting_down()); - let attachment = match (terminal_id, publication_guard) { - (Some(terminal_id), Some(publication_guard)) => { + let attachment = match terminal_host_ownership { + Some((terminal_id, publication_guard)) => { crate::terminal_host_runtime::launch_terminal_host_with_identity_cancellable( &opts, &root, @@ -2841,7 +2839,7 @@ impl Surface { &launch_cancelled, )? } - (None, None) => crate::terminal_host_runtime::launch_terminal_host_cancellable( + None => crate::terminal_host_runtime::launch_terminal_host_cancellable( &opts, &root, default_colors, @@ -2849,7 +2847,6 @@ impl Surface { initial_kitty_limits, &launch_cancelled, )?, - _ => anyhow::bail!("terminal-host identity and publication ownership diverged"), }; return Self::spawn_hosted( id, @@ -2865,7 +2862,7 @@ impl Surface { }, ); } - let _ = (terminal_id, publication_guard); + let _ = terminal_host_ownership; let reaper = reserve_local_child_reaper().context("reserve bounded PTY session cleanup")?; let initial_geometry = PtyGeometry { cols: opts.cols, @@ -5929,7 +5926,7 @@ impl Surface { if let Some(mux) = pty.mux.upgrade() { #[cfg(unix)] if let Some(identity) = terminate_fallback { - mux.terminate_discovered_terminal_host( + let _ = mux.terminate_discovered_terminal_host( &identity.terminal_id, Some(&identity.incarnation), ); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs index 1b070256795a..db1974c62475 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_protocol.rs @@ -149,6 +149,7 @@ impl TerminalExit { /// /// cmux-pty's Unix backend returns `std::process::Child`, so failure to downcast /// is an alternate backend and becomes an explicit unknown outcome. +#[cfg(all(test, unix))] pub(crate) fn wait_for_native_child_status( child: &mut (dyn cmux_pty::Child + Send + Sync), ) -> TerminalExit { diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 03123ae6f77f..0f2ab1f5578f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -52,8 +52,6 @@ const MAX_ENV: usize = 1024; const MAX_RENDERER_CAPABILITY_TTL: std::time::Duration = std::time::Duration::from_secs(60); pub(crate) const CONTROL_RESPONSE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); const HOST_HANDSHAKE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); -const HOST_CONNECT_RETRY_WINDOW: std::time::Duration = std::time::Duration::from_secs(1); -const HOST_CONNECT_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); const TERMINAL_HOST_PUBLICATION_LOCK_FILE: &str = ".publication.lock"; // Keep live PTY backpressure independent from the extra headroom needed by // one maximum Resized + Colors + targeted acknowledgement transition. @@ -630,7 +628,6 @@ mod unix { .expect("terminal-host process reaper remains alive while registered"); } const HOST_CLIENT_WRITE_TIMEOUT: Duration = Duration::from_secs(2); - const HOST_HANDSHAKE_TRANSIENT_RETRIES: usize = 1; const HOST_EXIT_PERSIST_RETRY_MIN: Duration = Duration::from_millis(100); const HOST_EXIT_PERSIST_RETRY_MAX: Duration = Duration::from_secs(5); const HOST_EXIT_PERSIST_REPORT_INTERVAL: Duration = Duration::from_secs(60); @@ -4104,6 +4101,7 @@ mod unix { normal_cleanup_failures: AtomicUsize, #[cfg(test)] normal_cleanup_attempts: AtomicUsize, + #[cfg(test)] fail_next_resize_publication: AtomicBool, } @@ -6075,6 +6073,7 @@ mod unix { ), #[cfg(test)] normal_cleanup_attempts: AtomicUsize::new(0), + #[cfg(test)] fail_next_resize_publication: AtomicBool::new(false), }); HostShared::start_exit_publisher(&shared, exit_publish_receiver)?; From 5600f2894e4822d0f010b0d1bbd8ae244c2d7a80 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:22:26 -0700 Subject: [PATCH 356/521] fix(tui): clear remaining hosted lints --- cmux-tui/crates/cmux-remote/src/daemon.rs | 4 ++-- cmux-tui/crates/cmux-remote/src/workspace/process.rs | 2 +- cmux-tui/crates/cmux-tui-core/tests/pty.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/src/daemon.rs b/cmux-tui/crates/cmux-remote/src/daemon.rs index cd0c31b3f995..27bde7d5d77e 100644 --- a/cmux-tui/crates/cmux-remote/src/daemon.rs +++ b/cmux-tui/crates/cmux-remote/src/daemon.rs @@ -1720,7 +1720,7 @@ mod tests { let (daemon, _accepted) = RemoteDaemon::new(auth, SessionLimits::default()); let (release, holder) = hold_process_creation_barrier(); - let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (started_sender, started_receiver) = oneshot::channel(); let mut bind = tokio::spawn(async move { started_sender.send(()).unwrap(); serve_direct_websocket(daemon, "127.0.0.1:0".parse().unwrap(), 128 * 1024, false).await @@ -1751,7 +1751,7 @@ mod tests { LimitedTcpListener { inner: listener, permits: Arc::new(Semaphore::new(1)) }; let (release, holder) = hold_process_creation_barrier(); - let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (started_sender, started_receiver) = oneshot::channel(); let mut accept = tokio::spawn(async move { started_sender.send(()).unwrap(); listener.accept().await diff --git a/cmux-tui/crates/cmux-remote/src/workspace/process.rs b/cmux-tui/crates/cmux-remote/src/workspace/process.rs index 4e2583943692..eee557fae614 100644 --- a/cmux-tui/crates/cmux-remote/src/workspace/process.rs +++ b/cmux-tui/crates/cmux-remote/src/workspace/process.rs @@ -2827,7 +2827,7 @@ mod tests { ); options.env.insert("PIDFILE".into(), pid_file.to_string_lossy().into_owned()); let spawn_manager = manager.clone(); - let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let (started_sender, started_receiver) = oneshot::channel(); let mut spawn = tokio::spawn(async move { started_sender.send(()).unwrap(); spawn_manager.spawn(root, options).await diff --git a/cmux-tui/crates/cmux-tui-core/tests/pty.rs b/cmux-tui/crates/cmux-tui-core/tests/pty.rs index d3fea28ef4ae..e14aec6c2b94 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/pty.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/pty.rs @@ -328,7 +328,7 @@ fn surface_exit_detaches_terminal_view_and_emits_event() { assert_eq!(state.workspaces.len(), 1); assert!(state.workspaces[0].screens.is_empty()); }); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] From 21a16894d4cbe068c294fa51a191eba5d04affe6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:44:55 -0700 Subject: [PATCH 357/521] fix(tui): reconcile hosted compile owners --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 ++ .../crates/cmux-tui/src/remote_runtime.rs | 18 +++++----- .../crates/cmux-tui/src/session/remote.rs | 2 +- cmux-tui/crates/cmux-tui/tests/cli.rs | 35 ++++++++++--------- .../cmux-tui/tests/terminal_host_recovery.rs | 2 +- 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 93017c489e33..ee9ccf476320 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -13557,6 +13557,7 @@ impl Mux { surface.kill(); } } + #[cfg(unix)] let runtime_identities = terminal_runtimes .iter() .filter_map(|runtime| self.resource_terminal_host_identity(runtime)) @@ -13566,6 +13567,7 @@ impl Mux { self.purge_terminal_runtime_side_tables(&runtime); self.terminate_terminal_runtime(&runtime); } + #[cfg(unix)] let host_cleanup = host_cleanup .into_iter() .filter(|(_, record)| { diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index ad1783165d39..63d837e0e9e6 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -83,16 +83,14 @@ struct DaemonCleanupPause { expected_state_dir: PathBuf, expected_phase: DaemonCleanupPausePhase, reached: mpsc::SyncSender<()>, - resume: std::sync::Mutex>, + resume: Mutex>, } #[cfg(test)] -static DAEMON_CLEANUP_PAUSES: std::sync::Mutex>> = - std::sync::Mutex::new(Vec::new()); +static DAEMON_CLEANUP_PAUSES: Mutex>> = Mutex::new(Vec::new()); #[cfg(test)] -static STATE_DIRECTORY_SYNC_FAILURES: std::sync::Mutex> = - std::sync::Mutex::new(Vec::new()); +static STATE_DIRECTORY_SYNC_FAILURES: Mutex> = Mutex::new(Vec::new()); #[cfg(test)] struct DaemonCleanupPauseHandle { @@ -110,7 +108,7 @@ impl DaemonCleanupPauseHandle { expected_state_dir, expected_phase, reached: reached_tx, - resume: std::sync::Mutex::new(resume_rx), + resume: Mutex::new(resume_rx), }); let mut installed = DAEMON_CLEANUP_PAUSES.lock().unwrap_or_else(std::sync::PoisonError::into_inner); @@ -726,8 +724,6 @@ pub fn start_client_runtime(options: ClientRuntimeOptions) -> anyhow::Result>, + requests: Mutex>, } #[async_trait] @@ -3224,7 +3222,7 @@ mod tests { #[derive(Default)] struct AuthRecordingReconnectRouteAttempt { - auth: std::sync::Mutex>, + auth: Mutex>, } #[async_trait] diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index 389da64c8424..bcee6b46d773 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -1892,7 +1892,7 @@ impl RemoteSession { self.request(client_info)?; #[cfg(debug_assertions)] if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_CLIENT_REGISTERED_SIGNAL") { - std::fs::write(signal, b"1")?; + fs::write(signal, b"1")?; } if subscribe { self.prime_local_subscription(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 53d778345c51..760d14ee91c5 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -2,6 +2,8 @@ use std::ffi::CStr; use std::fs; #[cfg(unix)] +use std::fs::File; +#[cfg(unix)] use std::io::{BufRead, BufReader, Read, Write}; #[cfg(unix)] use std::os::fd::AsRawFd; @@ -86,6 +88,12 @@ fn wait_for_child_exit( child.wait().unwrap() } +#[cfg(unix)] +fn native_pty_child(child: &mut (dyn cmux_pty::Child + Send + Sync)) -> &mut Child { + let child: &mut dyn cmux_pty::Child = child; + child.downcast_mut::().expect("cmux-pty Unix test child is a native process") +} + struct HeadlessServer { child: Child, socket: PathBuf, @@ -249,18 +257,6 @@ impl HeadlessServer { } } -#[cfg(unix)] -fn wait_for_child_exit(child: &mut Child, timeout: Duration) -> bool { - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if child.try_wait().unwrap().is_some() { - return true; - } - std::thread::sleep(Duration::from_millis(10)); - } - false -} - #[cfg(unix)] fn wait_for_processes_to_exit(pids: &[u32], timeout: Duration) -> bool { let deadline = Instant::now() + timeout; @@ -326,7 +322,7 @@ fn server_establishes_shutdown_ownership_before_publishing_its_listener() { // SAFETY: path is a valid, NUL-terminated path in the test directory. assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), 0o600) }, 0); } - let (ready_sender, ready_receiver) = std::sync::mpsc::sync_channel(1); + let (ready_sender, ready_receiver) = mpsc::sync_channel(1); let ready_marker = marker.clone(); let ready_reader = std::thread::spawn(move || { use std::io::Read as _; @@ -2905,7 +2901,7 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { assert_eq!(response, serde_json::json!({})); let status = wait_for_child_exit( - &mut server.child, + native_pty_child(server.child.as_mut()), Duration::from_secs(3), "server remained alive after acknowledging shutdown", ); @@ -3023,7 +3019,7 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ assert_eq!(response["accepted"], true); let status = wait_for_child_exit( - &mut server.child, + native_pty_child(server.child.as_mut()), Duration::from_secs(3), "server remained alive after acknowledging daemon handoff", ); @@ -3928,7 +3924,11 @@ fn graceful_shutdown_stops_server_owned_sidebar_process() { let server_pid = libc::pid_t::try_from(server.child.id()).unwrap(); // SAFETY: this PID is the live child owned by the test fixture. assert_eq!(unsafe { libc::kill(server_pid, libc::SIGINT) }, 0); - let server_stopped = wait_for_child_exit(&mut server.child, Duration::from_secs(10)); + let server_status = wait_for_child_exit( + &mut server.child, + Duration::from_secs(10), + "SIGINT did not complete graceful server shutdown", + ); let owned_processes_stopped = wait_for_processes_to_exit(&owned_pids, Duration::from_secs(5)); // Keep lifecycle regressions leak-free. Every captured process group and @@ -3954,7 +3954,7 @@ fn graceful_shutdown_stops_server_owned_sidebar_process() { } } - assert!(server_stopped, "SIGINT did not complete graceful server shutdown"); + assert!(server_status.success(), "SIGINT server shutdown exited with {server_status}"); assert!( !used_durable_host, "server-owned sidebar process entered the durable terminal-host registry" @@ -4931,6 +4931,7 @@ fn create_live_terminal_host_record(root: &std::path::Path) -> fs::File { host_start_nonce: host_start_nonce.clone(), workspace_key: String::new(), supports_set_defaults: true, + supports_terminate_only: false, supports_clear_history: true, supports_terminate_ack: false, }; diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index b4c1ed6ae901..1247eca8c31a 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -730,7 +730,7 @@ fn explicit_terminate_reaps_background_jobs_in_separate_pty_process_groups() { // SAFETY: both fixture processes are live and owned by this test. let descendant_session = unsafe { libc::getsid(descendant_pid) }; - let host = adopt_terminal_host(record, record_path).unwrap(); + let mut host = adopt_terminal_host(record, record_path).unwrap(); host.terminate().unwrap(); host.disconnect(); wait_for_no_host_records(&harness.host_root()); From 35bb2833f6ad6508ed0a0f874b3acd9426cd4882 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:45:09 -0700 Subject: [PATCH 358/521] test(tui): expose unowned runtime launches --- .../crates/cmux-remote/src/ssh_bootstrap.rs | 44 ++++++++++++ cmux-tui/crates/cmux-tui-core/src/server.rs | 69 ++++++++++++++++++- cmux-tui/crates/cmux-tui/src/config.rs | 37 ++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs index cab7a414bba4..a0fd0dbd71e8 100644 --- a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs +++ b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs @@ -1131,4 +1131,48 @@ mod tests { async fn oversized_ssh_stderr_kills_and_reaps_ssh() { assert_oversized_output_is_bounded("stderr").await; } + + #[cfg(target_os = "macos")] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn binary_upload_waits_for_the_process_creation_barrier() { + use std::fs; + + let directory = tempfile::tempdir().unwrap(); + let source = directory.path().join("cmux-tui"); + fs::write(&source, b"test binary").unwrap(); + let mut config = SshBootstrapConfig::defaults("host"); + config.ssh_binary = "/usr/bin/true".into(); + config.timeout = Duration::from_secs(5); + let bootstrap = SshBootstrapper::new(config).unwrap(); + let (held_sender, held_receiver) = std::sync::mpsc::channel(); + let (release_sender, release_receiver) = std::sync::mpsc::channel(); + let holder = std::thread::spawn(move || { + let _barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + held_sender.send(()).unwrap(); + release_receiver.recv().unwrap(); + }); + held_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("process creation barrier was not acquired"); + + let (started_sender, started_receiver) = tokio::sync::oneshot::channel(); + let mut upload = tokio::spawn(async move { + started_sender.send(()).unwrap(); + bootstrap.run_remote_with_input("ignored", &source).await + }); + started_receiver.await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(250), &mut upload).await.is_err(), + "SSH upload spawned a child while descriptor creation held the shared barrier" + ); + release_sender.send(()).unwrap(); + holder.join().unwrap(); + + let output = tokio::time::timeout(Duration::from_secs(5), upload) + .await + .expect("SSH upload did not resume after the process barrier was released") + .unwrap() + .unwrap(); + assert_eq!(output.status, 0); + } } diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 8de8bd389e3e..6345053524a8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4654,6 +4654,8 @@ pub struct WebSocketServer { thread: Option>, #[cfg(test)] accept_attempts: Arc, + #[cfg(test)] + active_connections: Arc, #[cfg(all(test, target_os = "macos"))] accepted_stream_mode: Arc, } @@ -4718,6 +4720,24 @@ pub fn serve_websocket( addr: SocketAddr, token: Option, allow_insecure_bind: bool, +) -> anyhow::Result { + serve_websocket_with_tracking_cloner( + mux, + addr, + token, + allow_insecure_bind, + Arc::new(cmux_tui_process::tcp::clone_stream), + ) +} + +type WebSocketTrackingCloner = Arc std::io::Result + Send + Sync>; + +fn serve_websocket_with_tracking_cloner( + mux: Arc, + addr: SocketAddr, + token: Option, + allow_insecure_bind: bool, + tracking_cloner: WebSocketTrackingCloner, ) -> anyhow::Result { // WebSocket has no TLS here. Remote deployments must explicitly opt in and // should put cmux-tui behind a TLS-terminating reverse proxy. @@ -4747,6 +4767,7 @@ pub fn serve_websocket( let accepted_stream_mode = Arc::new(AtomicU64::new(0)); let thread_shutdown = shutdown.clone(); let thread_connections = connections.clone(); + let thread_active_connections = active_connections.clone(); #[cfg(test)] let thread_accept_attempts = accept_attempts.clone(); #[cfg(all(test, target_os = "macos"))] @@ -4782,9 +4803,9 @@ pub fn serve_websocket( if thread_shutdown.load(Ordering::Acquire) { break; } - let Some(permit) = claim_connection(&active_connections) else { continue }; + let Some(permit) = claim_connection(&thread_active_connections) else { continue }; let id = next_connection.fetch_add(1, Ordering::Relaxed); - if let Ok(tracked) = cmux_tui_process::tcp::clone_stream(&stream) { + if let Ok(tracked) = tracking_cloner(&stream) { thread_connections.lock().unwrap().insert(id, tracked); } let mux = mux.clone(); @@ -4823,6 +4844,8 @@ pub fn serve_websocket( thread: Some(thread), #[cfg(test)] accept_attempts, + #[cfg(test)] + active_connections, #[cfg(all(test, target_os = "macos"))] accepted_stream_mode, }) @@ -11515,6 +11538,7 @@ mod tests { connections: Arc::new(Mutex::new(HashMap::new())), thread: Some(listener_thread), accept_attempts: Arc::new(AtomicU64::new(0)), + active_connections: Arc::new(AtomicU64::new(0)), #[cfg(target_os = "macos")] accepted_stream_mode: Arc::new(AtomicU64::new(0)), }; @@ -11546,6 +11570,7 @@ mod tests { connections: Arc::new(Mutex::new(HashMap::new())), thread: Some(listener_thread), accept_attempts: Arc::new(AtomicU64::new(0)), + active_connections: Arc::new(AtomicU64::new(0)), #[cfg(target_os = "macos")] accepted_stream_mode: Arc::new(AtomicU64::new(0)), }; @@ -11561,6 +11586,46 @@ mod tests { assert!(server.thread.is_none(), "completed WebSocket listener retained its join handle"); } + #[test] + fn websocket_worker_requires_a_registered_shutdown_owner() { + let mux = Mux::new("websocket-tracking-clone-test", SurfaceOptions::default()); + let (clone_attempted, observed_clone_attempt) = std::sync::mpsc::channel(); + let tracking_cloner: WebSocketTrackingCloner = Arc::new(move |_| { + clone_attempted.send(()).unwrap(); + Err(std::io::Error::other("injected tracking clone failure")) + }); + let mut server = serve_websocket_with_tracking_cloner( + mux.clone(), + "127.0.0.1:0".parse().unwrap(), + None, + false, + tracking_cloner, + ) + .unwrap(); + let client = cmux_tui_process::tcp::connect_stream_timeout( + &server.local_addr, + Duration::from_secs(1), + ) + .unwrap(); + observed_clone_attempt + .recv_timeout(Duration::from_secs(1)) + .expect("WebSocket listener did not attempt to register the accepted connection"); + + assert!( + server.shutdown_until(Instant::now() + Duration::from_secs(1)).unwrap(), + "WebSocket listener did not finish after rejecting the untracked connection" + ); + let active_connections = server.active_connections.load(Ordering::Acquire); + drop(client); + drop(server); + let _ = mux.shutdown(); + + assert_eq!( + active_connections, 0, + "WebSocket worker started without a registered shutdown owner" + ); + } + #[cfg(target_os = "macos")] #[test] fn websocket_accept_restores_blocking_mode_before_protocol_reads() { diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 1c53d19ed216..8788824841e8 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -5549,6 +5549,43 @@ mod tests { assert!(!stdout.contains("CMUX_PROVIDER_WORKSPACE_AUTHORITY="), "{stdout}"); } + #[cfg(target_os = "macos")] + #[test] + fn ghostty_config_helper_waits_for_the_process_creation_barrier() { + let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = std::sync::mpsc::channel(); + let (result_sender, result_receiver) = std::sync::mpsc::channel(); + let helper = std::thread::spawn(move || { + let mut command = Command::new("/usr/bin/printf"); + command + .arg("foreground=#010203\nbackground=#040506\n") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + started_sender.send(()).unwrap(); + let result = ghostty_defaults_from_helper_command(command, Duration::from_secs(5)); + result_sender.send(result).unwrap(); + }); + started_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("config helper worker did not reach process creation"); + + assert!( + matches!( + result_receiver.recv_timeout(Duration::from_millis(250)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "Ghostty config helper spawned while descriptor creation held the shared barrier" + ); + drop(barrier); + let result = result_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("config helper did not resume after the process barrier was released"); + helper.join().unwrap(); + + assert!(matches!(result, GhosttyHelperDefaults::Resolved(_))); + } + #[cfg(unix)] #[test] fn ghostty_config_helper_cleanup_reaps_killed_child() { From 882999974e1c0d0e03b8edd56fec09e8a0609213 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:45:26 -0700 Subject: [PATCH 359/521] fix(tui): require tracked runtime launches --- cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs | 7 +++---- cmux-tui/crates/cmux-tui-core/src/server.rs | 8 +++++--- cmux-tui/crates/cmux-tui/src/config.rs | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs index a0fd0dbd71e8..bc2662f49ca4 100644 --- a/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs +++ b/cmux-tui/crates/cmux-remote/src/ssh_bootstrap.rs @@ -448,13 +448,12 @@ impl SshBootstrapper { command.arg("-p").arg(port.to_string()); } command.args(&self.config.extra_args).arg(&self.config.destination).arg(remote_command); - let mut child = command + command .stdin(Stdio::from(source)) .stdout(Stdio::piped()) .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .map_err(BootstrapError::Io)?; + .kill_on_drop(true); + let mut child = crate::process_creation::spawn(&mut command).map_err(BootstrapError::Io)?; let stdout = match child.stdout.take() { Some(stdout) => stdout, None => { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 6345053524a8..1765c2a69ec3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4805,9 +4805,11 @@ fn serve_websocket_with_tracking_cloner( } let Some(permit) = claim_connection(&thread_active_connections) else { continue }; let id = next_connection.fetch_add(1, Ordering::Relaxed); - if let Ok(tracked) = tracking_cloner(&stream) { - thread_connections.lock().unwrap().insert(id, tracked); - } + let tracked = match tracking_cloner(&stream) { + Ok(tracked) => tracked, + Err(_) => continue, + }; + thread_connections.lock().unwrap().insert(id, tracked); let mux = mux.clone(); let token = token.clone(); let render_service = render_service.clone(); diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 8788824841e8..e96e8c52049e 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -3272,7 +3272,7 @@ fn ghostty_defaults_from_helper_command( mut command: Command, parent_deadline: Duration, ) -> GhosttyHelperDefaults { - let Ok(mut child) = command.spawn() else { + let Ok(mut child) = cmux_tui_process::spawn(&mut command) else { return GhosttyHelperDefaults::Unavailable; }; let Some(stdout) = child.stdout.take() else { From dbb61f84ad6dc089eb198b3b9668ea09c09d5917 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:52:28 -0700 Subject: [PATCH 360/521] fix(tui): scope websocket test ownership --- cmux-tui/crates/cmux-tui-core/src/server.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 1765c2a69ec3..8facbfbd13f2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4767,7 +4767,8 @@ fn serve_websocket_with_tracking_cloner( let accepted_stream_mode = Arc::new(AtomicU64::new(0)); let thread_shutdown = shutdown.clone(); let thread_connections = connections.clone(); - let thread_active_connections = active_connections.clone(); + #[cfg(test)] + let observed_active_connections = active_connections.clone(); #[cfg(test)] let thread_accept_attempts = accept_attempts.clone(); #[cfg(all(test, target_os = "macos"))] @@ -4803,7 +4804,7 @@ fn serve_websocket_with_tracking_cloner( if thread_shutdown.load(Ordering::Acquire) { break; } - let Some(permit) = claim_connection(&thread_active_connections) else { continue }; + let Some(permit) = claim_connection(&active_connections) else { continue }; let id = next_connection.fetch_add(1, Ordering::Relaxed); let tracked = match tracking_cloner(&stream) { Ok(tracked) => tracked, @@ -4847,7 +4848,7 @@ fn serve_websocket_with_tracking_cloner( #[cfg(test)] accept_attempts, #[cfg(test)] - active_connections, + active_connections: observed_active_connections, #[cfg(all(test, target_os = "macos"))] accepted_stream_mode, }) From 8a676ba46083bd0592b19c8af8ddf80f630d6416 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 20:57:41 -0700 Subject: [PATCH 361/521] fix(ci): make review gates match lifecycle owners Use the full test-binary inventory for the shared Valgrind pidfd probe, retain artifact-level macOS deployment verification, and remove impossible Swift Task cancellation assertions from Foundation Thread callbacks. --- .github/workflows/cmux-tui.yml | 19 +++++++++---------- ...rfaceRuntimeTeardownCoordinatorTests.swift | 10 ---------- cmux-tui/dist/scripts/test_package_pypi.py | 18 ------------------ 3 files changed, 9 insertions(+), 38 deletions(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 771847ea9bf2..1e79f371f5f6 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -121,6 +121,7 @@ jobs: seen = set() selected = [] + pidfd_probe = None with open("target/cargo-test-binaries.jsonl", "r", encoding="utf-8") as messages: for line in messages: try: @@ -133,6 +134,8 @@ jobs: if not executable or executable in seen: continue seen.add(executable) + if re.fullmatch(r"cmux_tui_core-[0-9a-f]+", os.path.basename(executable)): + pidfd_probe = executable if shard_for(executable) == shard: selected.append(executable) @@ -140,10 +143,14 @@ jobs: raise SystemExit("cargo did not report any test binaries") if not selected: raise SystemExit(f"Valgrind shard {shard} selected no test binaries") + if pidfd_probe is None: + raise SystemExit("cargo did not report the cmux_tui_core pidfd probe binary") with open("target/valgrind-test-binaries.txt", "w", encoding="utf-8") as output: for executable in selected: print(executable, file=output) + with open("target/valgrind-pidfd-probe.txt", "w", encoding="utf-8") as output: + print(pidfd_probe, file=output) print( f"Valgrind shard {shard} selected {len(selected)} of {len(seen)} test binaries", @@ -177,16 +184,8 @@ jobs: -- "$bin" "$@" } - pidfd_probe= - while IFS= read -r bin; do - case "${bin##*/}" in - cmux_tui_core-*) - pidfd_probe="$bin" - break - ;; - esac - done < target/valgrind-test-binaries.txt - test -n "$pidfd_probe" + pidfd_probe="$(< target/valgrind-pidfd-probe.txt)" + test -x "$pidfd_probe" probe_test=process_session::tests::successful_preflight_survives_a_transient_initializer_failure "$pidfd_probe" --exact "$probe_test" >/dev/null valgrind_pidfd_supported=true diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index b016087290ac..4eda03021e49 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -413,7 +413,6 @@ private func requireTeardownTicket( let activeFreesStarted = AsyncStream.makeStream() let cancelledFreeStarted = AsyncStream.makeStream() let releaseActiveFrees = DispatchSemaphore(value: 0) - let observedCancellation = OSAllocatedUnfairLock(initialState: false) defer { releaseActiveFrees.signal() releaseActiveFrees.signal() @@ -454,9 +453,6 @@ private func requireTeardownTicket( byteTeeLease: nil, runtimeOwnershipReservation: reservations[2], freeSurface: { _ in - observedCancellation.withLock { - $0 = Task.isCancelled - } cancelledFreeStarted.continuation.yield() } ) @@ -472,7 +468,6 @@ private func requireTeardownTicket( cancelledFreeStarted.stream.makeAsyncIterator() _ = await cancelledFreeIterator.next() #expect(await cancelledTicket.wait(timeout: nil)) - #expect(observedCancellation.withLock { $0 }) releaseActiveFrees.signal() for ticket in activeTickets { @@ -486,7 +481,6 @@ private func requireTeardownTicket( defer { surface.deallocate() } let freeStarted = AsyncStream.makeStream() let releaseFree = DispatchSemaphore(value: 0) - let observedCancellation = OSAllocatedUnfairLock(initialState: false) defer { releaseFree.signal() freeStarted.continuation.finish() @@ -502,9 +496,6 @@ private func requireTeardownTicket( freeSurface: { _ in freeStarted.continuation.yield() releaseFree.wait() - observedCancellation.withLock { - $0 = Task.isCancelled - } } ) ) @@ -517,7 +508,6 @@ private func requireTeardownTicket( releaseFree.signal() #expect(await ticket.wait(timeout: nil)) - #expect(observedCancellation.withLock { $0 }) #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) } diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 5962346c1278..68b4e4733bc5 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -100,23 +100,5 @@ def test_release_build_and_wheels_share_one_macos_minimum(self) -> None: readme = Path(__file__).resolve().parents[2] / "README.md" self.assertIn("macOS 15 or newer", readme.read_text()) - def test_intel_release_uses_the_apple_linker_deployment_target(self) -> None: - workflow = ( - Path(__file__).resolve().parents[3] - / ".github" - / "workflows" - / "cmux-tui-build-package.yml" - ).read_text() - cross_build = workflow.split("- name: Build cmux-tui (cross)", 1)[1].split( - "- name: Stage binary", 1 - )[0] - - self.assertIn('if [[ "$RUNNER_OS" == macOS ]]; then', cross_build) - self.assertIn( - "cargo build -p cmux-tui --bin cmux-tui --release --locked --target", - cross_build, - ) - - if __name__ == "__main__": unittest.main() From 1acf86041102b2a11537f82b2323d72d843388f5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:31:51 -0700 Subject: [PATCH 362/521] fix(ci): keep hosted platform owners portable Compile the shutdown-result owner on every platform and keep the Valgrind pidfd skip inventory aligned with current process-owning tests. --- cmux-tui/crates/cmux-tui/src/main.rs | 3 --- cmux-tui/dist/valgrind-pidfd-skips.txt | 16 ++++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 57e44035738f..02e2107b5cdb 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -2122,7 +2122,6 @@ fn finish_server_process( result } -#[cfg(unix)] fn combine_server_and_remote_shutdown_results( server_result: anyhow::Result<()>, remote_shutdown_result: anyhow::Result<()>, @@ -3170,7 +3169,6 @@ mod tests { assert!(!socket_remained, "remote startup failure retained the published server socket"); } - #[cfg(unix)] #[test] fn remote_shutdown_failure_preserves_an_earlier_server_error() { let error = combine_server_and_remote_shutdown_results( @@ -3184,7 +3182,6 @@ mod tests { assert!(message.contains("remote shutdown failed")); } - #[cfg(unix)] #[test] fn remote_shutdown_failure_is_returned_after_successful_server_execution() { let error = combine_server_and_remote_shutdown_results( diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index c9c5794b1790..c6c994ace9f1 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -21,7 +21,6 @@ cmux_tui_core mux::tests::daemon_exit_retains_active_local_owner_until_terminati cmux_tui_core mux::tests::daemon_exit_retries_until_async_browser_bootstrap_releases_ownership cmux_tui_core mux::tests::failed_new_pane_attachment_retains_shutdown_ownership cmux_tui_core mux::tests::failed_surface_shutdown_retains_process_ownership_for_retry -cmux_tui_core mux::tests::key_close_reacquires_replacement_workspace_lifecycle cmux_tui_core mux::tests::live_authority_install_and_rotation_preserve_open_pty cmux_tui_core mux::tests::new_browser_tab_materializes_selected_empty_workspace cmux_tui_core mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal @@ -31,7 +30,7 @@ cmux_tui_core mux::tests::persistent_provider_managed_mux_keeps_registry_durable cmux_tui_core mux::tests::provider_ownership_handoff_waits_for_an_entered_ordinary_close cmux_tui_core mux::tests::rejected_shutdown_preflight_preserves_browser_ownership cmux_tui_core mux::tests::restart_marks_reserved_terminal_without_host_record_exited_without_respawn -cmux_tui_core mux::tests::restart_materializes_exited_terminal_until_explicit_close +cmux_tui_core mux::tests::restart_keeps_exited_terminal_as_detached_receipt_until_explicit_close cmux_tui_core mux::tests::run_materializes_active_empty_workspace cmux_tui_core mux::tests::run_new_workspace_accepts_a_stable_caller_key cmux_tui_core mux::tests::server_shutdown_cannot_pass_an_in_flight_surface_retirement @@ -79,8 +78,7 @@ cmux_tui app::tests::active_scrollbar_drag_rebases_after_terminal_output_changes cmux_tui app::tests::alt_n_rejects_a_new_pane_with_no_visible_content cmux_tui app::tests::alt_n_uses_zellij_default_vertical_distribution cmux_tui app::tests::attached_tab_mouse_click_uses_rendered_hit_and_survives_routing_refresh -cmux_tui app::tests::attached_tab_selection_is_published_to_owner_mux -cmux_tui app::tests::attached_workspace_mouse_click_uses_both_rendered_rows_and_survives_routing_refresh +cmux_tui app::tests::attached_workspace_mouse_down_uses_both_rendered_rows_and_survives_routing_refresh cmux_tui app::tests::attached_workspace_release_without_press_is_a_no_op cmux_tui app::tests::browser_drag_retargets_to_the_animated_pane_geometry cmux_tui app::tests::browser_graphics_processing_does_not_block_terminal_pointer_input @@ -97,8 +95,6 @@ cmux_tui app::tests::command_k_reaches_a_kitty_app_on_the_alternate_screen cmux_tui app::tests::contended_terminal_semantics_retain_discrete_press_for_replay cmux_tui app::tests::ctrl_l_reaches_foreground_primary_screen_app cmux_tui app::tests::deferred_click_fails_closed_after_pointer_map_generation_changes -cmux_tui app::tests::deferred_input_does_not_follow_a_later_routing_mutation -cmux_tui app::tests::deferred_input_follows_a_committed_routing_mutation cmux_tui app::tests::deferred_input_is_discarded_when_its_destination_changes cmux_tui app::tests::deferred_scrollbar_press_cannot_reinterpret_changed_thumb_geometry cmux_tui app::tests::deferred_tab_press_cannot_retarget_a_replacement_surface_at_the_same_index @@ -182,6 +178,7 @@ cmux_tui app::tests::split_ratio_samples_coalesce_without_snapshots_before_final cmux_tui app::tests::stale_layout_undo_confirmation_keeps_the_created_pane cmux_tui app::tests::stale_session_events_are_ignored_after_an_in_place_switch cmux_tui app::tests::surface_only_context_menu_omits_client_management +cmux_tui app::tests::tab_selection_is_client_local cmux_tui app::tests::tab_switch_moves_size_lease_without_dropping_hidden_surface cmux_tui app::tests::terminal_geometry_mutations_enter_the_pointer_routing_lane cmux_tui app::tests::terminal_paint_reuses_unchanged_pointer_owner_snapshots @@ -196,7 +193,6 @@ cmux_tui app::tests::workspace_rail_scrollbar_is_visible_clickable_and_draggable cmux_tui config::tests::injected_ghostty_defaults_drive_headless_render_state cmux_tui pty_input::tests::ambiguous_local_press_keeps_its_recovery_release_reservation cmux_tui server_lifecycle::legacy_process::tests::frozen_tree_drop_resumes_with_its_original_process_proof -cmux_tui server_lifecycle::legacy_process::tests::process_identity_never_signals_through_a_reusable_pid cmux_tui server_lifecycle::tests::legacy_stop_terminates_only_the_identified_process cmux_tui tests::failed_server_publication_preserves_live_socket pty attach_stream_orders_resize_between_output_frames @@ -213,17 +209,17 @@ pty control_socket_round_trip pty control_socket_wait_for_matches_one_shot_output_already_on_screen pty create_empty_workspace_is_visible_and_materialized_in_place pty default_colors_apply_to_existing_and_future_surfaces -pty headless_creation_uses_legacy_default_then_latest_client_size +pty headless_creation_uses_explicit_or_authoritative_client_size pty new_tab_on_empty_headless_session_creates_workspace pty read_scrollback_pages_oldest_rows_and_clamps_bounds pty render_attach_headless_fans_one_frame_to_render_and_byte_consumers pty render_attach_resize_is_a_full_replacement_at_the_new_size pty render_attach_snapshot_and_raced_write_have_no_gap_or_duplicate_frame pty send_paste_wraps_only_while_dec_mode_2004_is_enabled -pty shared_surface_uses_smallest_viewer_size_until_that_viewer_detaches -pty surface_exit_reaps_tree_and_emits_event +pty surface_exit_detaches_terminal_view_and_emits_event pty surface_resize_reports_whether_the_size_changed pty surface_runs_command_and_screen_updates +pty terminal_surface_uses_only_its_explicit_geometry_authority pty tree_event_modes_receive_delta_or_exact_coarse_fallback websocket_transport clients_list_identify_resize_and_detach_across_transports websocket_transport websocket_auth_accepts_exact_preamble_and_rejects_missing_or_wrong_tokens From 36edd98cf14d677faa20ca2927d093a347a48a83 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 21:52:40 -0700 Subject: [PATCH 363/521] test(macOS): bound stalled surface recovery waiters --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 4eda03021e49..0b14db3e86c3 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -743,6 +743,52 @@ private func requireTeardownTicket( #expect(admission.debugOwnerCount == 0) } + @MainActor + @Test func recoveryQueueRejectsNewIDsAtOwnershipCapacity() async throws { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + let firstOwner = try #require(admission.reserve()) + let secondOwner = try #require(admission.reserve()) + let recoveries = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(2) + ) + var thirdRecoveryRan = false + + for index in 0..<3 { + let reservation = admission.reserve( + recoveryID: UUID(), + onRecovery: { reservation in + if index == 2 { + thirdRecoveryRan = true + } + admission.release(reservation) + if index < 2 { + recoveries.continuation.yield(index) + if index == 1 { + recoveries.continuation.finish() + } + } + } + ) + #expect(reservation == nil) + } + + admission.release(firstOwner) + var recoveryIterator = recoveries.stream.makeAsyncIterator() + #expect(await recoveryIterator.next() == 0) + #expect(await recoveryIterator.next() == 1) + #expect( + admission.debugOwnerCount == 1, + "a third recovery reservation exceeded the bounded owner count" + ) + #expect( + !thirdRecoveryRan, + "a recovery request beyond the owner capacity was retained" + ) + admission.release(secondOwner) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) From 59b6c93d6de409db6e21f230d9376e7efc114485 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:02:57 -0700 Subject: [PATCH 364/521] fix(macOS): bound surface recovery admission --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 48 +++--- ...timeOwnershipRecoveryAdmissionResult.swift | 11 ++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 2 +- .../TerminalSurface+RuntimeLifecycle.swift | 39 +++-- ...rfaceRuntimeTeardownCoordinatorTests.swift | 152 +++++++++++++++--- 5 files changed, 193 insertions(+), 59 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 8801ecd7b090..05c9006a56be 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -26,31 +26,47 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { self.maximumOwnerCount = maximumOwnerCount } - func reserve( - recoveryID: UUID? = nil, - onRecovery: TerminalSurfaceRuntimeOwnershipRecovery? = nil - ) -> TerminalSurfaceRuntimeOwnershipReservation? { + func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { state.withLock { state in guard !state.closeTeardownDegraded, state.reservationIDs.count < maximumOwnerCount else { - if let recoveryID, let onRecovery { - enqueueRecoveryAction( - onRecovery, - id: recoveryID, - in: &state - ) - } return nil } - if let recoveryID { - _ = removeRecoveryAction(recoveryID, from: &state) - } let reservation = TerminalSurfaceRuntimeOwnershipReservation() state.reservationIDs.insert(reservation.id) return reservation } } + func reserve( + recoveryID: UUID, + onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery + ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { + state.withLock { state in + if !state.closeTeardownDegraded, + state.reservationIDs.count < maximumOwnerCount + { + _ = removeRecoveryAction(recoveryID, from: &state) + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + return .reserved(reservation) + } + if state.recoveryEntriesByID[recoveryID] != nil { + state.recoveryEntriesByID[recoveryID]?.action = onRecovery + return .deferred + } + guard state.recoveryEntriesByID.count < maximumOwnerCount else { + return .rejected + } + enqueueRecoveryAction( + onRecovery, + id: recoveryID, + in: &state + ) + return .deferred + } + } + func contains(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) -> Bool { state.withLock { $0.reservationIDs.contains(reservation.id) } } @@ -104,10 +120,6 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { id recoveryID: UUID, in state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) { - if state.recoveryEntriesByID[recoveryID] != nil { - state.recoveryEntriesByID[recoveryID]?.action = action - return - } let previousID = state.recoveryTailID state.recoveryEntriesByID[recoveryID] = TerminalSurfaceRuntimeOwnershipRecoveryEntry( diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift new file mode 100644 index 000000000000..91d374e90204 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift @@ -0,0 +1,11 @@ +/// Result of synchronous native-surface ownership admission with recovery. +nonisolated enum TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult: Equatable, Sendable { + /// Ownership was admitted immediately. + case reserved(TerminalSurfaceRuntimeOwnershipReservation) + + /// The bounded recovery queue retained the request for a later grant. + case deferred + + /// The bounded recovery queue rejected the request without retaining it. + case rejected +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 5809a5166b3f..d2c88afe7940 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -145,7 +145,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { nonisolated func reserveRuntimeSurfaceOwnership( recoveryID: UUID, onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery - ) -> TerminalSurfaceRuntimeOwnershipReservation? { + ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { runtimeOwnershipAdmission.reserve( recoveryID: recoveryID, onRecovery: onRecovery diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index bc4c4e4d6700..fa8f826de08b 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -756,26 +756,33 @@ extension TerminalSurface { ).promoted(with: source) runtimeSurfaceAdmissionDeferredCreationView = view let teardownCoordinator = runtimeTeardown - guard let reservation = - teardownCoordinator.reserveRuntimeSurfaceOwnership( - recoveryID: id, - onRecovery: { [weak self, teardownCoordinator] reservation in - guard let self else { - teardownCoordinator.cancelRuntimeSurfaceOwnership( - reservation - ) - return - } - self.resumeRuntimeSurfaceCreationAfterAdmissionRecovery( - reservation: reservation + let admissionResult = + teardownCoordinator.reserveRuntimeSurfaceOwnership( + recoveryID: id, + onRecovery: { [weak self, teardownCoordinator] reservation in + guard let self else { + teardownCoordinator.cancelRuntimeSurfaceOwnership( + reservation ) + return } - ) else { + self.resumeRuntimeSurfaceCreationAfterAdmissionRecovery( + reservation: reservation + ) + } + ) + switch admissionResult { + case .reserved(let reservation): + runtimeSurfaceAdmissionDeferredCreationSource = nil + runtimeSurfaceAdmissionDeferredCreationView = nil + return reservation + case .deferred: + return nil + case .rejected: + runtimeSurfaceAdmissionDeferredCreationSource = nil + runtimeSurfaceAdmissionDeferredCreationView = nil return nil } - runtimeSurfaceAdmissionDeferredCreationSource = nil - runtimeSurfaceAdmissionDeferredCreationView = nil - return reservation } @MainActor diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 0b14db3e86c3..3a99d8617907 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -626,7 +626,7 @@ private func requireTeardownTicket( recovery.continuation.yield(reservation) recovery.continuation.finish() } - ) == nil + ) == .deferred ) releaseFrees.signal() @@ -670,7 +670,7 @@ private func requireTeardownTicket( bufferingPolicy: .bufferingNewest(1) ) - let staleReservation = admission.reserve( + let staleResult = admission.reserve( recoveryID: staleRecoveryID, onRecovery: { [weak staleProbe] reservation in guard staleProbe != nil else { @@ -681,7 +681,7 @@ private func requireTeardownTicket( admission.release(reservation) } ) - let nextReservation = admission.reserve( + let nextResult = admission.reserve( recoveryID: nextRecoveryID, onRecovery: { reservation in nextRecoveryRan = true @@ -690,8 +690,8 @@ private func requireTeardownTicket( admission.release(reservation) } ) - #expect(staleReservation == nil) - #expect(nextReservation == nil) + #expect(staleResult == .deferred) + #expect(nextResult == .deferred) admission.release(firstOwner) staleProbe = nil @@ -718,7 +718,7 @@ private func requireTeardownTicket( let recoveries = AsyncStream.makeStream() for _ in 0..<3 { - let reservation = admission.reserve( + let result = admission.reserve( recoveryID: UUID(), onRecovery: { reservation in recoveredCount += 1 @@ -726,7 +726,7 @@ private func requireTeardownTicket( admission.release(reservation) } ) - #expect(reservation == nil) + #expect(result == .deferred) } admission.setCloseTeardownDegraded(false) @@ -744,36 +744,136 @@ private func requireTeardownTicket( } @MainActor - @Test func recoveryQueueRejectsNewIDsAtOwnershipCapacity() async throws { + @Test func recoveryAdmissionReservesImmediatelyWhenCapacityIsAvailable() { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + var recoveryRan = false + + let result = admission.reserve( + recoveryID: UUID(), + onRecovery: { _ in recoveryRan = true } + ) + let reservation: TerminalSurfaceRuntimeOwnershipReservation + switch result { + case .reserved(let admittedReservation): + reservation = admittedReservation + case .deferred, .rejected: + Issue.record("available ownership did not reserve immediately") + return + } + + #expect(admission.contains(reservation)) + #expect(admission.debugOwnerCount == 1) + #expect(!recoveryRan) + admission.release(reservation) + #expect(admission.debugOwnerCount == 0) + } + + @MainActor + @Test func queuedRecoveryRetryReservesImmediatelyWhenCapacityIsAvailable() async throws { let admission = TerminalSurfaceRuntimeOwnershipAdmission( maximumOwnerCount: 2 ) let firstOwner = try #require(admission.reserve()) let secondOwner = try #require(admission.reserve()) - let recoveries = AsyncStream.makeStream( - bufferingPolicy: .bufferingNewest(2) + let leadingRecovery = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(1) ) - var thirdRecoveryRan = false + let recoveryID = UUID() + var queuedRecoveryRan = false + var retryRecoveryRan = false - for index in 0..<3 { - let reservation = admission.reserve( + #expect( + admission.reserve( recoveryID: UUID(), onRecovery: { reservation in - if index == 2 { - thirdRecoveryRan = true - } admission.release(reservation) - if index < 2 { - recoveries.continuation.yield(index) - if index == 1 { - recoveries.continuation.finish() - } - } + leadingRecovery.continuation.yield() + leadingRecovery.continuation.finish() } - ) - #expect(reservation == nil) + ) == .deferred + ) + #expect( + admission.reserve( + recoveryID: recoveryID, + onRecovery: { reservation in + queuedRecoveryRan = true + admission.release(reservation) + } + ) == .deferred + ) + + admission.release(firstOwner) + admission.release(secondOwner) + let retryResult = admission.reserve( + recoveryID: recoveryID, + onRecovery: { _ in retryRecoveryRan = true } + ) + let retryReservation: TerminalSurfaceRuntimeOwnershipReservation + switch retryResult { + case .reserved(let reservation): + retryReservation = reservation + case .deferred, .rejected: + Issue.record("available same-ID retry did not reserve immediately") + return } + #expect(admission.contains(retryReservation)) + #expect(!queuedRecoveryRan) + #expect(!retryRecoveryRan) + admission.release(retryReservation) + var leadingRecoveryIterator = leadingRecovery.stream.makeAsyncIterator() + _ = await leadingRecoveryIterator.next() + #expect(admission.debugOwnerCount == 0) + } + + @MainActor + @Test func recoveryQueueRejectsNewIDsAtOwnershipCapacity() async throws { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + let firstOwner = try #require(admission.reserve()) + let secondOwner = try #require(admission.reserve()) + let recoveries = AsyncStream.makeStream( + bufferingPolicy: .bufferingNewest(2) + ) + let firstRecoveryID = UUID() + let secondRecoveryID = UUID() + var replacedRecoveryRan = false + var thirdRecoveryRan = false + + let firstResult = admission.reserve( + recoveryID: firstRecoveryID, + onRecovery: { _ in replacedRecoveryRan = true } + ) + #expect(firstResult == .deferred) + let replacementResult = admission.reserve( + recoveryID: firstRecoveryID, + onRecovery: { reservation in + admission.release(reservation) + recoveries.continuation.yield(0) + } + ) + #expect(replacementResult == .deferred) + let secondResult = admission.reserve( + recoveryID: secondRecoveryID, + onRecovery: { reservation in + admission.release(reservation) + recoveries.continuation.yield(1) + recoveries.continuation.finish() + } + ) + #expect(secondResult == .deferred) + let thirdResult = admission.reserve( + recoveryID: UUID(), + onRecovery: { reservation in + thirdRecoveryRan = true + admission.release(reservation) + } + ) + #expect(thirdResult == .rejected) + admission.release(firstOwner) var recoveryIterator = recoveries.stream.makeAsyncIterator() #expect(await recoveryIterator.next() == 0) @@ -782,6 +882,10 @@ private func requireTeardownTicket( admission.debugOwnerCount == 1, "a third recovery reservation exceeded the bounded owner count" ) + #expect( + !replacedRecoveryRan, + "same-ID recovery replacement retained the prior action" + ) #expect( !thirdRecoveryRan, "a recovery request beyond the owner capacity was retained" From f4a3f5001a3e24b36dc34fe8034f7891c0e50438 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:39:11 -0700 Subject: [PATCH 365/521] fix(ci): bound hosted lifecycle verification --- .github/workflows/cmux-tui.yml | 16 +++++- cmux-tui/crates/cmux-tui-core/src/mux.rs | 51 ++++++++++++++---- .../cmux-tui-core/src/workspace_registry.rs | 2 +- .../src/workspace_registry/resource_store.rs | 2 +- cmux-tui/crates/cmux-tui/src/app.rs | 3 +- cmux-tui/crates/cmux-tui/src/config.rs | 4 +- cmux-tui/crates/cmux-tui/src/pty_input.rs | 2 +- .../src/server_lifecycle/legacy_process.rs | 4 +- cmux-tui/crates/cmux-tui/tests/cli.rs | 53 ++----------------- cmux-tui/dist/valgrind-pidfd-skips.txt | 4 ++ 10 files changed, 70 insertions(+), 71 deletions(-) diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 1e79f371f5f6..1a2fc3c7b2b4 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -171,6 +171,7 @@ jobs: # Retain the full 128 MiB fairness workload under Valgrind with # explicit instrumentation-only bounds. Normal CI stays strict. CMUX_TEST_PERF_INSTRUMENTED: "valgrind" + VALGRIND_SHARD: ${{ matrix.shard }} run: | run_valgrind() { local bin="$1" @@ -269,12 +270,23 @@ jobs: fi done < target/valgrind-test-binaries.txt if [ "$valgrind_pidfd_supported" = false ]; then - while read -r skip_scope; do + required_skip_scopes=() + case "$VALGRIND_SHARD" in + core) required_skip_scopes=(cmux_tui_core) ;; + tui) required_skip_scopes=(cmux_tui) ;; + remainder) required_skip_scopes=(browser_runtime pty websocket_transport) ;; + remote) ;; + *) + echo "Unknown Valgrind shard: $VALGRIND_SHARD" >&2 + exit 1 + ;; + esac + for skip_scope in "${required_skip_scopes[@]}"; do if [ -z "${seen_skip_scopes[$skip_scope]:-}" ]; then echo "No test binary matched Valgrind skip scope: $skip_scope" >&2 exit 1 fi - done < <(awk '!/^#/ && NF { print $1 }' dist/valgrind-pidfd-skips.txt | sort -u) + done fi valgrind-leak-check: diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ee9ccf476320..dd384d327b8a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -27648,26 +27648,55 @@ mod tests { let mux = test_mux(); let initial = mux.create_empty_workspace(None, None, None).unwrap(); let events = mux.subscribe(); - let close_ready = Arc::new(std::sync::Barrier::new(2)); - let resume_close = Arc::new(std::sync::Barrier::new(2)); + let final_failure_timeout = crate::test_timeout(Duration::from_secs(2)); + let (close_reached_tx, close_reached_rx) = std::sync::mpsc::sync_channel(1); + let (release_close_tx, release_close_rx) = std::sync::mpsc::sync_channel(1); + let release_close_rx = Arc::new(Mutex::new(release_close_rx)); *mux.workspace_close_before_empty_check.lock().unwrap() = Some(Arc::new({ - let close_ready = close_ready.clone(); - let resume_close = resume_close.clone(); + let release_close_rx = Arc::clone(&release_close_rx); move || { - close_ready.wait(); - resume_close.wait(); + close_reached_tx.send(()).unwrap(); + release_close_rx + .lock() + .unwrap() + .recv_timeout(final_failure_timeout) + .expect("close owner was not released before the final failure deadline"); } })); + let (close_done_tx, close_done_rx) = std::sync::mpsc::sync_channel(1); let close_mux = mux.clone(); let close = std::thread::spawn(move || { - close_mux.close_workspace_at_revision(initial.workspace, Some(1)).unwrap() + close_done_tx + .send(close_mux.close_workspace_at_revision(initial.workspace, Some(1))) + .unwrap(); + }); + close_reached_rx.recv_timeout(final_failure_timeout).expect( + "close did not reach the empty-workspace check before the final failure deadline", + ); + + let (replacement_done_tx, replacement_done_rx) = std::sync::mpsc::sync_channel(1); + let replacement_mux = mux.clone(); + let replacement_owner = std::thread::spawn(move || { + replacement_done_tx + .send(replacement_mux.create_empty_workspace(None, None, Some(2))) + .unwrap(); }); - close_ready.wait(); - let replacement = mux.create_empty_workspace(None, None, Some(2)).unwrap(); + let replacement = replacement_done_rx + .recv_timeout(final_failure_timeout) + .expect("replacement creation did not finish before the final failure deadline") + .unwrap(); *mux.workspace_close_before_empty_check.lock().unwrap() = None; - resume_close.wait(); - assert_eq!(close.join().unwrap(), Some(2)); + release_close_tx.send(()).unwrap(); + assert_eq!( + close_done_rx + .recv_timeout(final_failure_timeout) + .expect("close did not finish after release before the final failure deadline") + .unwrap(), + Some(2) + ); + close.join().unwrap(); + replacement_owner.join().unwrap(); let emitted = events.try_iter().collect::>(); assert!(emitted.iter().any(|event| matches!( diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index eea5b1441c8e..7268ad9af469 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -2704,7 +2704,7 @@ impl WorkspaceRegistry { self.terminal_snapshot_count.fetch_add(1, Ordering::Relaxed); let mut statement = self .connection - .prepare("SELECT terminal_id FROM terminal_placements ORDER BY terminal_id ASC")?; + .prepare("SELECT terminal_id FROM terminal_hosts ORDER BY terminal_id ASC")?; statement .query_map([], |row| row.get::<_, String>(0))? .collect::, _>>() diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 56cf3c6e9d63..6277d9fe32ba 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -534,7 +534,7 @@ impl WorkspaceRegistry { validate_workspace_key(workspace_key)?; let mut statement = self.connection.prepare( "SELECT terminal_id, incarnation - FROM terminal_placements + FROM terminal_hosts WHERE workspace_key = ?1 AND lifecycle != 'tombstoned' ORDER BY created_revision ASC, terminal_id ASC", diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index 2960a516b6b8..e6950b6ebfb6 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -12906,6 +12906,7 @@ impl App { } => { if self .pending_pointer_motion + .as_ref() .is_some_and(|pointer| pointer.destination == Some(surface)) { self.pending_pointer_motion = None; @@ -31094,7 +31095,7 @@ mod tests { assert!(!app.session.has_pending_mutations()); assert_eq!(app.deferred_input.len(), 1); assert!(events.try_recv().is_err()); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index e96e8c52049e..cf3d55d44dc4 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -5553,8 +5553,8 @@ mod tests { #[test] fn ghostty_config_helper_waits_for_the_process_creation_barrier() { let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); - let (started_sender, started_receiver) = std::sync::mpsc::channel(); - let (result_sender, result_receiver) = std::sync::mpsc::channel(); + let (started_sender, started_receiver) = mpsc::channel(); + let (result_sender, result_receiver) = mpsc::channel(); let helper = std::thread::spawn(move || { let mut command = Command::new("/usr/bin/printf"); command diff --git a/cmux-tui/crates/cmux-tui/src/pty_input.rs b/cmux-tui/crates/cmux-tui/src/pty_input.rs index 28facce41121..b3396889d46d 100644 --- a/cmux-tui/crates/cmux-tui/src/pty_input.rs +++ b/cmux-tui/crates/cmux-tui/src/pty_input.rs @@ -1426,7 +1426,7 @@ mod tests { assert_eq!(dispatcher.enqueue(input()), PtyInputEnqueueResult::Accepted); assert!(dispatcher.shutdown(Duration::from_secs(1))); - mux.shutdown(); + let _ = mux.shutdown(); } #[test] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index 32ec2fc933be..b487d6447736 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -1052,14 +1052,14 @@ mod tests { let unavailable = ProcessIdentity { pid: process.pid, started_at: process.started_at.wrapping_add(1) }; let session = CapturedSession { id: pid, members: vec![unavailable] }; - let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); let reaper = std::thread::spawn(move || { release_rx.recv().unwrap(); child.kill().unwrap(); child.wait().unwrap() }); - let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); + let (result_tx, result_rx) = mpsc::sync_channel(1); let waiter = std::thread::spawn(move || { result_tx .send(session.kill_until_empty(Instant::now() + Duration::from_secs(1))) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 760d14ee91c5..d65011f2ad5a 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1,5 +1,3 @@ -#[cfg(unix)] -use std::ffi::CStr; use std::fs; #[cfg(unix)] use std::fs::File; @@ -327,7 +325,7 @@ fn server_establishes_shutdown_ownership_before_publishing_its_listener() { let ready_reader = std::thread::spawn(move || { use std::io::Read as _; - let mut marker = fs::File::open(ready_marker).unwrap(); + let mut marker = File::open(ready_marker).unwrap(); let mut signal = [0_u8; 1]; marker.read_exact(&mut signal).unwrap(); ready_sender.send(()).unwrap(); @@ -501,7 +499,7 @@ fn wait_for_pid_file(path: &std::path::Path, timeout: Duration) -> u32 { let change = libc::kevent { ident: directory.as_raw_fd() as libc::uintptr_t, filter: libc::EVFILT_VNODE, - flags: (libc::EV_ADD | libc::EV_ENABLE | libc::EV_CLEAR) as u16, + flags: libc::EV_ADD | libc::EV_ENABLE | libc::EV_CLEAR, fflags: libc::NOTE_WRITE | libc::NOTE_EXTEND | libc::NOTE_RENAME, data: 0, udata: std::ptr::null_mut(), @@ -2313,51 +2311,6 @@ fn noun_first_ratio_commands_reject_nonfinite_values_before_connecting() { } } -#[cfg(unix)] -fn open_test_pty() -> (File, File) { - static PTY_NAME_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - let _name_lock = PTY_NAME_LOCK.lock().unwrap(); - let master_fd = unsafe { libc::posix_openpt(libc::O_RDWR | libc::O_NOCTTY | libc::O_CLOEXEC) }; - assert_ne!(master_fd, -1, "posix_openpt failed: {}", std::io::Error::last_os_error()); - let master = unsafe { File::from_raw_fd(master_fd) }; - let master_flags = unsafe { libc::fcntl(master.as_raw_fd(), libc::F_GETFD) }; - assert_ne!( - master_flags, - -1, - "fcntl(F_GETFD) failed for PTY master: {}", - std::io::Error::last_os_error() - ); - assert_ne!(master_flags & libc::FD_CLOEXEC, 0, "PTY master was created without close-on-exec"); - assert_eq!( - unsafe { libc::grantpt(master.as_raw_fd()) }, - 0, - "grantpt failed: {}", - std::io::Error::last_os_error() - ); - assert_eq!( - unsafe { libc::unlockpt(master.as_raw_fd()) }, - 0, - "unlockpt failed: {}", - std::io::Error::last_os_error() - ); - let slave_name = unsafe { libc::ptsname(master.as_raw_fd()) }; - assert!(!slave_name.is_null(), "ptsname failed: {}", std::io::Error::last_os_error()); - let slave_name = unsafe { CStr::from_ptr(slave_name) }.to_owned(); - let slave_fd = - unsafe { libc::open(slave_name.as_ptr(), libc::O_RDWR | libc::O_NOCTTY | libc::O_CLOEXEC) }; - assert_ne!(slave_fd, -1, "open PTY slave failed: {}", std::io::Error::last_os_error()); - let slave = unsafe { File::from_raw_fd(slave_fd) }; - let size = libc::winsize { ws_row: 24, ws_col: 80, ws_xpixel: 0, ws_ypixel: 0 }; - assert_ne!( - unsafe { libc::ioctl(slave.as_raw_fd(), libc::TIOCSWINSZ, &size) }, - -1, - "set PTY size failed: {}", - std::io::Error::last_os_error() - ); - (master, slave) -} - #[cfg(unix)] struct PtyChild { child: Box, @@ -4913,7 +4866,7 @@ fn find_session_database(state: &std::path::Path, session: &str) -> PathBuf { } #[cfg(unix)] -fn create_live_terminal_host_record(root: &std::path::Path) -> fs::File { +fn create_live_terminal_host_record(root: &std::path::Path) -> File { fs::create_dir_all(root).unwrap(); fs::set_permissions(root, fs::Permissions::from_mode(0o700)).unwrap(); let terminal_id = "0000000000004000800000000000002a"; diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index c6c994ace9f1..27294a3cd3a7 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -193,8 +193,10 @@ cmux_tui app::tests::workspace_rail_scrollbar_is_visible_clickable_and_draggable cmux_tui config::tests::injected_ghostty_defaults_drive_headless_render_state cmux_tui pty_input::tests::ambiguous_local_press_keeps_its_recovery_release_reservation cmux_tui server_lifecycle::legacy_process::tests::frozen_tree_drop_resumes_with_its_original_process_proof +cmux_tui server_lifecycle::legacy_process::tests::termination_accepts_a_verified_process_that_already_exited cmux_tui server_lifecycle::tests::legacy_stop_terminates_only_the_identified_process cmux_tui tests::failed_server_publication_preserves_live_socket +cmux_tui tests::remote_runtime_start_failure_cleans_published_server pty attach_stream_orders_resize_between_output_frames pty attach_stream_replays_then_streams_without_duplication pty byte_attach_cursor_snapshot_does_not_fan_out_a_render_frame @@ -223,9 +225,11 @@ pty terminal_surface_uses_only_its_explicit_geometry_authority pty tree_event_modes_receive_delta_or_exact_coarse_fallback websocket_transport clients_list_identify_resize_and_detach_across_transports websocket_transport websocket_auth_accepts_exact_preamble_and_rejects_missing_or_wrong_tokens +websocket_transport websocket_keeps_authenticated_inbound_requests_at_four_mib websocket_transport websocket_non_loopback_bind_requires_and_accepts_explicit_insecure_opt_in websocket_transport websocket_pairing_is_approved_over_trusted_unix_and_credential_reconnects websocket_transport websocket_rejects_oversized_authentication_frames +websocket_transport websocket_render_attach_carries_large_rgba_state_and_stays_connected websocket_transport websocket_server_allows_pairing_without_a_static_token websocket_transport websocket_server_rejects_tokens_that_cannot_fit_the_auth_limit websocket_transport websocket_streams_subscribe_and_attach_and_survives_unclean_disconnect From 1e409fb114745b8bbba7105df96b063cda1f49e2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:16:53 -0700 Subject: [PATCH 366/521] fix(ci): reconcile hosted lifecycle owners --- cmux-tui/crates/cmux-remote/Cargo.toml | 1 + cmux-tui/crates/cmux-remote/src/identity.rs | 229 ++++++++++++++---- .../src/terminal_host_runtime.rs | 10 +- .../cmux-tui-core/src/workspace_registry.rs | 81 +++++-- .../public_projection_store.rs | 7 +- .../src/workspace_registry/resource_store.rs | 39 ++- .../src/workspace_registry/tests.rs | 26 +- cmux-tui/crates/cmux-tui/src/main.rs | 12 +- .../src/server_lifecycle/legacy_process.rs | 18 +- cmux-tui/dist/valgrind-pidfd-skips.txt | 83 ++++++- 10 files changed, 383 insertions(+), 123 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/Cargo.toml b/cmux-tui/crates/cmux-remote/Cargo.toml index 29f063bde040..95238c9d6671 100644 --- a/cmux-tui/crates/cmux-remote/Cargo.toml +++ b/cmux-tui/crates/cmux-remote/Cargo.toml @@ -61,3 +61,4 @@ rcgen = "0.14" tempfile = "3" tokio = { workspace = true, features = ["test-util"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } +wait-timeout.workspace = true diff --git a/cmux-tui/crates/cmux-remote/src/identity.rs b/cmux-tui/crates/cmux-remote/src/identity.rs index 0069c5bd1bf1..627547709b52 100644 --- a/cmux-tui/crates/cmux-remote/src/identity.rs +++ b/cmux-tui/crates/cmux-remote/src/identity.rs @@ -590,12 +590,18 @@ impl PersistenceCoordinator { } } } + #[cfg(test)] + let accepted_submission = sender.is_none() || matches!(immediate.as_ref(), Some(Ok(()))); if let Some(result) = immediate { let _ = sender.expect("immediate persistence waiter is available").send(result); } if wake_worker { self.shared.changed.notify_one(); } + #[cfg(test)] + if accepted_submission { + self.hooks.record_accepted_submission(revision); + } PersistenceWaiter { receiver } } @@ -781,6 +787,7 @@ fn take_waiters_through( struct PersistenceTestHooks { writes_started: std::sync::atomic::AtomicUsize, writes_succeeded: std::sync::atomic::AtomicUsize, + highest_accepted_submission: std::sync::atomic::AtomicU64, fail_next: std::sync::atomic::AtomicUsize, fail_next_parent_sync: std::sync::atomic::AtomicUsize, panic_next: std::sync::atomic::AtomicUsize, @@ -788,6 +795,7 @@ struct PersistenceTestHooks { blocked: StdMutex, released: Condvar, started: Notify, + submission_accepted: Notify, } #[cfg(test)] @@ -821,15 +829,16 @@ impl PersistenceTestHooks { { return Err(format!("injected persistence failure for revision {revision}")); } - if let Some(gate) = std::env::var_os("CMUX_TEST_AUTH_PERSISTENCE_GATE") { - let gate = PathBuf::from(gate); - while !gate.exists() { - std::thread::sleep(Duration::from_millis(1)); - } - } Ok(()) } + fn record_accepted_submission(&self, revision: u64) { + use std::sync::atomic::Ordering; + + self.highest_accepted_submission.fetch_max(revision, Ordering::SeqCst); + self.submission_accepted.notify_waiters(); + } + fn after_write(&self, succeeded: bool) { if succeeded { self.writes_succeeded.fetch_add(1, std::sync::atomic::Ordering::SeqCst); @@ -865,6 +874,18 @@ impl PersistenceTestHooks { started.await; } } + + async fn wait_for_accepted_submission(&self, revision: u64) { + use std::sync::atomic::Ordering; + + loop { + let accepted = self.submission_accepted.notified(); + if self.highest_accepted_submission.load(Ordering::SeqCst) >= revision { + return; + } + accepted.await; + } + } } #[cfg(test)] @@ -2307,8 +2328,14 @@ impl std::error::Error for IdentityError {} #[cfg(test)] mod tests { use std::collections::BTreeMap; + use std::io::{BufRead as _, BufReader, Write as _}; + use std::process::{Child, Command, ExitStatus, Stdio}; + use std::sync::mpsc::{self, Receiver, RecvTimeoutError, TrySendError}; + use std::thread::JoinHandle; + use std::time::Instant; use cmux_remote_protocol::{Lane, SessionId}; + use wait_timeout::ChildExt; use zeroize::Zeroizing; use super::*; @@ -2317,6 +2344,13 @@ mod tests { }; use crate::link::test_support; + const AUTH_PERSISTENCE_QUEUED_MARKER: &str = "cmux-auth-persistence:revision-2-submitted:v1"; + const AUTH_PERSISTENCE_RELEASE_MARKER: &str = "cmux-auth-persistence:release-worker:v1"; + const AUTH_PERSISTENCE_DRAINED_MARKER: &str = "cmux-auth-persistence:worker-drained:v1"; + const AUTH_PERSISTENCE_CHILD_PHASE_TIMEOUT: Duration = Duration::from_secs(3); + // Successful child output is two protocol markers plus six libtest lines. + const AUTH_PERSISTENCE_CHILD_OUTPUT_LINES: usize = 8; + struct PersistenceReleaseGuard(Arc); impl PersistenceReleaseGuard { @@ -2332,6 +2366,120 @@ mod tests { } } + struct AuthPersistenceChildOwner { + child: Option, + markers: Receiver, + output_reader: Option>>, + } + + impl AuthPersistenceChildOwner { + fn new(child: Child) -> Self { + let (marker_sender, markers) = mpsc::sync_channel(AUTH_PERSISTENCE_CHILD_OUTPUT_LINES); + let mut owner = Self { child: Some(child), markers, output_reader: None }; + let stdout = owner + .child + .as_mut() + .and_then(|child| child.stdout.take()) + .expect("persistence child stdout was not piped"); + let output_reader = std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + match marker_sender.try_send(line?) { + Ok(()) => {} + Err(TrySendError::Disconnected(_)) => break, + Err(TrySendError::Full(_)) => { + return Err(std::io::Error::other( + "persistence child output exceeded its fixed line capacity", + )); + } + } + } + Ok(()) + }); + owner.output_reader = Some(output_reader); + owner + } + + fn wait_for_marker(&self, expected: &str, phase: &str) { + let deadline = Instant::now() + AUTH_PERSISTENCE_CHILD_PHASE_TIMEOUT; + let mut observed = Vec::new(); + loop { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + panic!("persistence child {phase} deadline expired; output: {observed:?}"); + }; + match self.markers.recv_timeout(remaining) { + Ok(line) if line == expected => return, + Ok(line) => observed.push(line), + Err(RecvTimeoutError::Timeout) => { + panic!("persistence child {phase} deadline expired; output: {observed:?}"); + } + Err(RecvTimeoutError::Disconnected) => { + panic!( + "persistence child output closed before {phase}; output: {observed:?}" + ); + } + } + } + } + + fn send_release(&mut self) { + let mut stdin = self + .child + .as_mut() + .and_then(|child| child.stdin.take()) + .expect("persistence child stdin was not piped"); + writeln!(stdin, "{AUTH_PERSISTENCE_RELEASE_MARKER}").unwrap(); + stdin.flush().unwrap(); + } + + fn wait_for_exit(&mut self) -> ExitStatus { + let status = self + .child + .as_mut() + .expect("persistence child owner was disarmed before exit") + .wait_timeout(AUTH_PERSISTENCE_CHILD_PHASE_TIMEOUT) + .expect("failed to wait for persistence child") + .unwrap_or_else(|| panic!("persistence child exit deadline expired")); + self.child.take(); + status + } + + fn finish_output(&mut self) { + self.output_reader + .take() + .expect("persistence child output reader was already joined") + .join() + .expect("persistence child output reader panicked") + .expect("failed to read persistence child output"); + } + } + + impl Drop for AuthPersistenceChildOwner { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + if let Some(output_reader) = self.output_reader.take() { + let _ = output_reader.join(); + } + } + } + + fn write_auth_persistence_child_marker(marker: &str) { + let stdout = std::io::stdout(); + let mut stdout = stdout.lock(); + writeln!(stdout, "{marker}").unwrap(); + stdout.flush().unwrap(); + } + + fn wait_for_auth_persistence_release() { + let stdin = std::io::stdin(); + let mut release = String::new(); + let bytes = BufReader::new(stdin.lock()).read_line(&mut release).unwrap(); + assert_ne!(bytes, 0, "persistence child release channel closed without a marker"); + assert_eq!(release.trim_end(), AUTH_PERSISTENCE_RELEASE_MARKER); + } + #[tokio::test] async fn identity_is_stable_and_files_are_private() { let temp = tempfile::tempdir().unwrap(); @@ -2856,16 +3004,16 @@ mod tests { let Some(state_dir) = std::env::var_os("CMUX_TEST_AUTH_PERSISTENCE_STATE") else { return; }; - let queued = PathBuf::from(std::env::var_os("CMUX_TEST_AUTH_PERSISTENCE_QUEUED").unwrap()); let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(1) .enable_all() .build() .unwrap(); - let database = runtime.block_on(async { + let (database, hooks) = runtime.block_on(async { let database = AuthDatabase::load_or_create(state_dir, "child", false).unwrap(); let hooks = database.persistence.hooks.clone(); - tokio::spawn({ + hooks.block(); + let first = tokio::spawn({ let database = database.clone(); async move { let _ = database.create_invitation(Duration::from_secs(60), Vec::new()).await; @@ -2874,62 +3022,59 @@ mod tests { tokio::time::timeout(Duration::from_secs(1), hooks.wait_for_started(1)) .await .expect("child persistence write did not start"); - tokio::spawn({ + let second = tokio::spawn({ let database = database.clone(); async move { let _ = database.create_invitation(Duration::from_secs(60), Vec::new()).await; } }); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if database.state.lock().await.revision == 2 { - break; + tokio::time::timeout(Duration::from_secs(1), hooks.wait_for_accepted_submission(2)) + .await + .expect("child did not submit its second authorization revision"); + first.abort(); + second.abort(); + tokio::time::timeout(AUTH_PERSISTENCE_CHILD_PHASE_TIMEOUT, async { + for (name, task) in [("first", first), ("second", second)] { + match task.await { + Ok(_) => {} + Err(error) if error.is_cancelled() => {} + Err(error) => panic!("{name} invitation task failed: {error}"), } - tokio::task::yield_now().await; } }) .await - .expect("child did not accept its second authorization revision"); - fs::write(queued, b"queued").unwrap(); - database + .expect("invitation tasks retained database ownership after cancellation"); + (database, hooks) }); - runtime.shutdown_timeout(Duration::from_millis(10)); + write_auth_persistence_child_marker(AUTH_PERSISTENCE_QUEUED_MARKER); + wait_for_auth_persistence_release(); + hooks.release(); + // Both task owners were collected while the worker was blocked, so + // runtime drop cannot provide persistence progress or retain ownership. + drop(runtime); drop(database); + write_auth_persistence_child_marker(AUTH_PERSISTENCE_DRAINED_MARKER); } #[test] fn auth_persistence_process_exit_joins_queued_revisions() { - use std::process::{Command, Stdio}; - let temp = tempfile::tempdir().unwrap(); let state_dir = temp.path().join("state"); - let gate = temp.path().join("release-write"); - let queued = temp.path().join("queued"); - let mut child = Command::new(std::env::current_exe().unwrap()) + let child = Command::new(std::env::current_exe().unwrap()) .args(["--exact", "identity::tests::auth_persistence_child_process", "--nocapture"]) .env("CMUX_TEST_AUTH_PERSISTENCE_STATE", &state_dir) - .env("CMUX_TEST_AUTH_PERSISTENCE_GATE", &gate) - .env("CMUX_TEST_AUTH_PERSISTENCE_QUEUED", &queued) - .stdin(Stdio::null()) - .stdout(Stdio::null()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .spawn() .unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(3); - while !queued.exists() { - if let Some(status) = child.try_wait().unwrap() { - panic!("persistence child exited before queueing its second revision: {status}"); - } - assert!( - std::time::Instant::now() < deadline, - "persistence child did not queue its second revision" - ); - std::thread::sleep(Duration::from_millis(1)); - } - - fs::write(&gate, b"release").unwrap(); - let status = child.wait().unwrap(); + let mut child = AuthPersistenceChildOwner::new(child); + child.wait_for_marker(AUTH_PERSISTENCE_QUEUED_MARKER, "revision-2 submission"); + child.send_release(); + child.wait_for_marker(AUTH_PERSISTENCE_DRAINED_MARKER, "persistence drain"); + let status = child.wait_for_exit(); + child.finish_output(); assert!(status.success(), "persistence child failed: {status}"); let persisted = load_state(&state_dir.join("devices.json")).unwrap(); assert_eq!(persisted.revision, 2); diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 0f2ab1f5578f..3b7f72c691e5 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -10372,12 +10372,16 @@ mod unix { let host_hello = read_required_frame(&mut client_stream, "host hello").unwrap(); assert_eq!(host_hello.kind, MessageKind::HostHello); assert_eq!(host_hello.flags, FLAG_TERMINATE_ONLY); - write_frame(&mut client_stream, &Frame::new(MessageKind::Terminate, Vec::new())) - .unwrap(); + let mut terminate = Frame::new(MessageKind::Terminate, Vec::new()); + terminate.request_id = 2; + write_frame(&mut client_stream, &terminate).unwrap(); + let terminate_ack = + read_required_frame(&mut client_stream, "terminate acknowledgement").unwrap(); + assert_eq!(terminate_ack.kind, MessageKind::TerminateAck); + assert_eq!(terminate_ack.request_id, terminate.request_id); client_stream.shutdown(std::net::Shutdown::Write).unwrap(); server.join().unwrap().unwrap(); - assert!(host.termination_started.load(Ordering::Acquire)); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 7268ad9af469..cd713246ddae 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -1258,7 +1258,7 @@ fn remove_reset_dir_children_from_handle( )?; #[cfg(test)] inject_reset_delete_child_replacement(&child_display)?; - let staged_child = stage_reset_child_for_deletion( + let mut staged_child = stage_reset_child_for_deletion( directory.as_raw_fd(), &child_name, &child_display, @@ -1269,11 +1269,11 @@ fn remove_reset_dir_children_from_handle( &staged_child.name, &child_relative, &staged_child.display_path, - &staged_child.stat, + staged_child.stat(), expected_entries, ignored_root_child, )?; - if reset_stat_is_dir(&staged_child.stat) { + if reset_stat_is_dir(staged_child.stat()) { let child_directory = open_reset_child_dir( directory.as_raw_fd(), &staged_child.name, @@ -1283,8 +1283,8 @@ fn remove_reset_dir_children_from_handle( format!("inspect {label} {}", staged_child.display_path.display()) })?; if !opened.file_type().is_dir() - || opened.dev() != reset_stat_device(&staged_child.stat) - || opened.ino() != reset_stat_inode(&staged_child.stat) + || opened.dev() != reset_stat_device(staged_child.stat()) + || opened.ino() != reset_stat_inode(staged_child.stat()) { anyhow::bail!( "reset path changed during reset: {}", @@ -1306,8 +1306,8 @@ fn remove_reset_dir_children_from_handle( &staged_child.display_path, )?; if !reset_stat_is_dir(¤t) - || reset_stat_device(¤t) != reset_stat_device(&staged_child.stat) - || reset_stat_inode(¤t) != reset_stat_inode(&staged_child.stat) + || reset_stat_device(¤t) != reset_stat_device(staged_child.stat()) + || reset_stat_inode(¤t) != reset_stat_inode(staged_child.stat()) { anyhow::bail!( "reset path changed during reset: {}", @@ -1328,6 +1328,7 @@ fn remove_reset_dir_children_from_handle( 0, )?; } + staged_child.commit_deletion(); } let remaining = reset_dir_child_names(directory, display_path, label)?; if !remaining.is_empty() { @@ -1371,9 +1372,39 @@ fn inject_reset_recreated_session_dir_after_staging(path: &Path) -> anyhow::Resu #[cfg(unix)] struct ResetStagedChild { + parent_fd: std::os::fd::RawFd, + original_name: std::ffi::OsString, + original_display_path: PathBuf, name: std::ffi::OsString, display_path: PathBuf, - stat: libc::stat, + stat: Option, + restore_on_drop: bool, +} + +#[cfg(unix)] +impl ResetStagedChild { + fn stat(&self) -> &libc::stat { + self.stat.as_ref().expect("staged reset child must have a stat") + } + + fn commit_deletion(&mut self) { + self.restore_on_drop = false; + } +} + +#[cfg(unix)] +impl Drop for ResetStagedChild { + fn drop(&mut self) { + if self.restore_on_drop { + let _ = reset_rename_child_exclusive( + self.parent_fd, + &self.name, + &self.original_name, + &self.display_path, + &self.original_display_path, + ); + } + } } #[cfg(unix)] @@ -1395,25 +1426,27 @@ fn stage_reset_child_for_deletion( &private_display, ) { Ok(()) => { - let stat = reset_child_stat(parent_fd, &private_name, &private_display)?; - if reset_stat_device(&stat) != reset_stat_device(expected) - || reset_stat_inode(&stat) != reset_stat_inode(expected) - || reset_stat_kind(&stat) != reset_stat_kind(expected) + let mut staged_child = ResetStagedChild { + parent_fd, + original_name: name.to_os_string(), + original_display_path: display_path.to_path_buf(), + name: private_name, + display_path: private_display, + stat: None, + restore_on_drop: true, + }; + staged_child.stat = Some(reset_child_stat( + parent_fd, + &staged_child.name, + &staged_child.display_path, + )?); + if reset_stat_device(staged_child.stat()) != reset_stat_device(expected) + || reset_stat_inode(staged_child.stat()) != reset_stat_inode(expected) + || reset_stat_kind(staged_child.stat()) != reset_stat_kind(expected) { - let _ = reset_rename_child_exclusive( - parent_fd, - &private_name, - name, - &private_display, - display_path, - ); anyhow::bail!("reset path changed during reset: {}", display_path.display()); } - return Ok(ResetStagedChild { - name: private_name, - display_path: private_display, - stat, - }); + return Ok(staged_child); } Err(error) if error diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs index 99e4565f43a2..096b92c772ea 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs @@ -844,7 +844,7 @@ mod tests { } #[test] - fn agent_projections_survive_terminal_tombstones() { + fn agent_projections_are_deleted_with_terminal_tombstones() { let mut registry = WorkspaceRegistry::in_memory("terminal-relationships").unwrap(); let session = registry.session_id().clone(); let (terminal, pane, tab) = seed_live_terminal(&mut registry); @@ -920,9 +920,8 @@ mod tests { .unwrap(); let tombstoned = registry.public_projections().unwrap(); - assert_eq!(registry.resource_agent_projection_count_for_test().unwrap(), 1); - assert_eq!(tombstoned.agents.len(), 1); - assert_eq!(tombstoned.agents[0].terminal_id, terminal); + assert_eq!(registry.resource_agent_projection_count_for_test().unwrap(), 0); + assert!(tombstoned.agents.is_empty()); assert_eq!(tombstoned.notifications.len(), 1); assert_eq!(tombstoned.notifications[0].terminal_id, None); } diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs index 6277d9fe32ba..630413c87f78 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/resource_store.rs @@ -147,7 +147,13 @@ pub(super) fn create_resource_schema(transaction: &Transaction<'_>) -> anyhow::R ) ) ); - DROP TRIGGER IF EXISTS resource_agent_projection_terminal_tombstone; + CREATE TRIGGER IF NOT EXISTS resource_agent_projection_terminal_tombstone + AFTER UPDATE OF deleted_revision ON resource_terminals + WHEN NEW.deleted_revision IS NOT NULL + BEGIN + DELETE FROM resource_agent_projections + WHERE terminal_id = NEW.public_id; + END; CREATE TABLE IF NOT EXISTS resource_events ( revision INTEGER PRIMARY KEY NOT NULL, previous_revision INTEGER NOT NULL, @@ -277,6 +283,7 @@ pub(super) fn migrate_resource_agent_projections( FROM ranked JOIN resource_terminals AS terminal ON terminal.public_id = ranked.terminal_id + AND terminal.deleted_revision IS NULL WHERE ranked.terminal_rank = 1 ON CONFLICT(terminal_id) DO UPDATE SET result_json = excluded.result_json, @@ -2573,6 +2580,24 @@ fn tombstone_resource_tab( other => anyhow::bail!("stored tab {tab_id} has invalid content kind {other:?}"), } } + tombstone_resource_tab_row(transaction, tab_id, revision)?; + if close_content { + match content_kind.as_str() { + "terminal" => { + tombstone_resource_terminal(transaction, &content_id, None, revision)?; + } + "browser" => tombstone_resource_browser(transaction, &content_id, revision)?, + other => anyhow::bail!("stored tab {tab_id} has invalid content kind {other:?}"), + } + } + Ok(()) +} + +fn tombstone_resource_tab_row( + transaction: &Transaction<'_>, + tab_id: &str, + revision: i64, +) -> anyhow::Result<()> { transaction.execute( "UPDATE resource_tabs SET position = NULL, updated_revision = ?1, deleted_revision = ?1 @@ -2585,13 +2610,7 @@ fn tombstone_resource_tab( WHERE active_tab_id = ?2 AND deleted_revision IS NULL", params![revision, tab_id], )?; - tombstone_resource_identity(transaction, tab_id, revision)?; - if close_content && content_kind == "browser" { - tombstone_resource_browser(transaction, &content_id, revision)?; - } else if !matches!(content_kind.as_str(), "terminal" | "browser") { - anyhow::bail!("stored tab {tab_id} has invalid content kind {content_kind:?}"); - } - Ok(()) + tombstone_resource_identity(transaction, tab_id, revision) } fn tombstone_resource_terminal( @@ -2619,7 +2638,7 @@ fn tombstone_resource_terminal( } let tabs = live_tabs_for_content(transaction, public_id)?; for tab in tabs { - tombstone_resource_tab(transaction, &tab, revision, false)?; + tombstone_resource_tab_row(transaction, &tab, revision)?; } transaction.execute( "UPDATE resource_terminals @@ -2650,7 +2669,7 @@ fn tombstone_resource_browser( } let tabs = live_tabs_for_content(transaction, public_id)?; for tab in tabs { - tombstone_resource_tab(transaction, &tab, revision, false)?; + tombstone_resource_tab_row(transaction, &tab, revision)?; } transaction.execute( "UPDATE resource_browsers diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs index 9c425b254f30..0772d984a046 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/tests.rs @@ -3681,16 +3681,18 @@ fn schema_seven_resumes_interrupted_sensitive_receipt_cleanup() { } #[test] -fn schema_seven_migrates_latest_agent_and_preserves_it_after_tombstone() { - assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(7); +fn schema_seven_migrates_latest_live_agent_and_tombstones_without_resurrection() { + assert_schema_migrates_latest_live_agent_and_tombstones_without_resurrection(7); } #[test] -fn schema_eight_migrates_latest_agent_and_preserves_it_after_tombstone() { - assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(8); +fn schema_eight_migrates_latest_live_agent_and_tombstones_without_resurrection() { + assert_schema_migrates_latest_live_agent_and_tombstones_without_resurrection(8); } -fn assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(legacy_schema: u32) { +fn assert_schema_migrates_latest_live_agent_and_tombstones_without_resurrection( + legacy_schema: u32, +) { let root = temp_root(&format!("schema-{legacy_schema}-agent-projection")); let database = root.join(session_storage_component("session")).join(WORKSPACE_REGISTRY_FILE); let terminal = terminal_resource(TERMINAL_ONE); @@ -3769,7 +3771,7 @@ fn assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(legacy_s |row| row.get(0), ) .unwrap(); - assert_eq!(legacy_trigger_count, 0); + assert_eq!(legacy_trigger_count, 1); let agents = migrated.public_projections().unwrap().agents; assert_eq!(agents.len(), 1); assert_eq!(agents[0].terminal_id, terminal); @@ -3805,12 +3807,12 @@ fn assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(legacy_s &json!([]), ) .unwrap(); - assert_eq!(migrated.resource_agent_projection_count_for_test().unwrap(), 1); - assert_eq!(migrated.public_projections().unwrap().agents.len(), 1); + assert_eq!(migrated.resource_agent_projection_count_for_test().unwrap(), 0); + assert!(migrated.public_projections().unwrap().agents.is_empty()); drop(migrated); - // Re-running the legacy migration recovers the durable projection from the - // latest report even though its terminal is already tombstoned. + // Re-running the legacy migration must not recreate an agent projection + // for a terminal that is already tombstoned. Connection::open(&database) .unwrap() .execute_batch(&format!( @@ -3820,8 +3822,8 @@ fn assert_schema_migrates_latest_agent_and_preserves_it_after_tombstone(legacy_s )) .unwrap(); let reopened = WorkspaceRegistry::open(&root, "session").unwrap(); - assert_eq!(reopened.resource_agent_projection_count_for_test().unwrap(), 1); - assert_eq!(reopened.public_projections().unwrap().agents.len(), 1); + assert_eq!(reopened.resource_agent_projection_count_for_test().unwrap(), 0); + assert!(reopened.public_projections().unwrap().agents.is_empty()); drop(reopened); fs::remove_dir_all(root).unwrap(); } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 02e2107b5cdb..8d4f76f5c199 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -3212,9 +3212,15 @@ mod tests { let mux = Mux::new("server-publication-test", SurfaceOptions::default()); let guard = ServerProcessShutdownGuard::start(&mux).unwrap(); - assert!(cmux_tui_core::server::serve(mux, Some(socket_path.clone())).is_err()); - guard.shutdown().unwrap(); - guard.complete(); + let error = finish_server_process( + guard, + cmux_tui_core::server::serve(mux, Some(socket_path.clone())).map(|_| ()), + ) + .expect_err("duplicate server publication unexpectedly succeeded"); + assert!( + format!("{error:#}").contains("is already in use"), + "unexpected server publication error: {error:#}" + ); let client = UnixStream::connect(&socket_path) .expect("failed publication cleanup unlinked the live server socket"); diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index b487d6447736..eb6c6ff9b3d3 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -918,7 +918,7 @@ mod tests { #[cfg(any(target_os = "macos", target_os = "linux"))] use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "macos", target_os = "linux"))] use std::sync::mpsc; use super::*; @@ -985,22 +985,6 @@ mod tests { assert!(resumed, "dropping a process fence left its exact process stopped"); } - #[test] - fn termination_accepts_a_verified_process_that_already_exited() { - let mut child = Command::new("sleep") - .arg("60") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .unwrap(); - let process = - ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); - child.kill().unwrap(); - child.wait().unwrap(); - - terminate_process_tree(process).unwrap(); - } - #[cfg(any(target_os = "macos", target_os = "linux"))] #[test] fn process_tree_termination_retries_post_freeze_metadata_errors() { diff --git a/cmux-tui/dist/valgrind-pidfd-skips.txt b/cmux-tui/dist/valgrind-pidfd-skips.txt index 27294a3cd3a7..a8d24a9ed669 100644 --- a/cmux-tui/dist/valgrind-pidfd-skips.txt +++ b/cmux-tui/dist/valgrind-pidfd-skips.txt @@ -8,31 +8,48 @@ browser_runtime queued_back_and_forward_do_not_collapse_while_worker_is_blocked browser_runtime socket_browser_attach_streams_frames_input_and_cell_pixels browser_runtime stalled_external_browser_nudges_target_once_before_interaction browser_runtime wedged_browser_navigate_does_not_block_same_socket_connection -cmux_tui_core browser::tests::ordinary_repaint_preserves_captured_drag_motion -cmux_tui_core browser::tests::server_shutdown_bounds_external_target_confirmation cmux_tui_core mux::tests::apply_layout_holds_target_workspace_lifecycle_through_commit cmux_tui_core mux::tests::blocked_terminal_adoption_does_not_stall_an_unrelated_terminal cmux_tui_core mux::tests::browser_runtime_connection_does_not_hold_the_shared_slot cmux_tui_core mux::tests::bulk_surface_close_uses_one_shared_termination_deadline +cmux_tui_core mux::tests::cleanup_scheduled_during_the_final_retry_gets_a_fresh_retry_budget +cmux_tui_core mux::tests::closing_a_view_preserves_terminal_side_tables_until_explicit_close cmux_tui_core mux::tests::concurrent_browser_and_terminal_share_empty_workspace_screen cmux_tui_core mux::tests::concurrent_browser_tabs_materialize_one_empty_workspace_screen cmux_tui_core mux::tests::concurrent_empty_workspace_terminal_inherits_the_first_terminals_cwd cmux_tui_core mux::tests::daemon_exit_retains_active_local_owner_until_termination_succeeds cmux_tui_core mux::tests::daemon_exit_retries_until_async_browser_bootstrap_releases_ownership +cmux_tui_core mux::tests::daemon_handoff_does_not_terminate_an_in_flight_terminal_adoption +cmux_tui_core mux::tests::durable_workspace_creation_supports_the_in_process_terminal_runtime cmux_tui_core mux::tests::failed_new_pane_attachment_retains_shutdown_ownership cmux_tui_core mux::tests::failed_surface_shutdown_retains_process_ownership_for_retry +cmux_tui_core mux::tests::key_close_cannot_rebind_a_live_durable_workspace_identity +cmux_tui_core mux::tests::killed_kitty_surface_cannot_block_the_next_terminal_reservation +cmux_tui_core mux::tests::kitty_quota_worker_exhausts_deadline_pool_admission_rejections +cmux_tui_core mux::tests::layout_undo_retires_removed_surface_owner_before_reusing_capacity cmux_tui_core mux::tests::live_authority_install_and_rotation_preserve_open_pty +cmux_tui_core mux::tests::local_workspace_close_retries_cleanup_from_its_committed_receipt cmux_tui_core mux::tests::new_browser_tab_materializes_selected_empty_workspace cmux_tui_core mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal +cmux_tui_core mux::tests::ordinary_surface_close_reconciles_a_transient_termination_failure cmux_tui_core mux::tests::ordinary_surface_close_retains_failed_process_ownership_until_reconciled cmux_tui_core mux::tests::pane_and_screen_close_wait_for_targeted_terminal_commit +cmux_tui_core mux::tests::pending_topology_accepts_current_host_reconnect_without_advancing_lifecycle +cmux_tui_core mux::tests::permanent_shutdown_owner_failure_stops_automatic_retry_sweeps +cmux_tui_core mux::tests::persistent_mux_restart_restores_auxiliary_resources_and_exact_replay cmux_tui_core mux::tests::persistent_provider_managed_mux_keeps_registry_durable_and_lifecycle_guarded +cmux_tui_core mux::tests::projected_effect_preserves_restored_terminal_before_surface_adoption cmux_tui_core mux::tests::provider_ownership_handoff_waits_for_an_entered_ordinary_close cmux_tui_core mux::tests::rejected_shutdown_preflight_preserves_browser_ownership +cmux_tui_core mux::tests::resource_workspace_close_retries_cleanup_before_committed_success cmux_tui_core mux::tests::restart_marks_reserved_terminal_without_host_record_exited_without_respawn cmux_tui_core mux::tests::restart_keeps_exited_terminal_as_detached_receipt_until_explicit_close +cmux_tui_core mux::tests::restart_sidecar_restores_exact_wait_exit_and_emits_one_public_event +cmux_tui_core mux::tests::restart_waits_for_inflight_terminal_host_publication_before_marking_missing +cmux_tui_core mux::tests::retained_shutdown_owners_bound_future_surface_admission cmux_tui_core mux::tests::run_materializes_active_empty_workspace cmux_tui_core mux::tests::run_new_workspace_accepts_a_stable_caller_key +cmux_tui_core mux::tests::server_stop_persists_a_restartable_empty_topology cmux_tui_core mux::tests::server_shutdown_cannot_pass_an_in_flight_surface_retirement cmux_tui_core mux::tests::server_shutdown_clears_many_surfaces_with_one_tree_event cmux_tui_core mux::tests::server_shutdown_keeps_topology_when_terminal_batch_commit_fails @@ -42,39 +59,56 @@ cmux_tui_core mux::tests::server_shutdown_reuses_one_deadline_bounded_record_sca cmux_tui_core mux::tests::server_shutdown_uses_launched_runtime_when_target_confirmation_is_unreachable cmux_tui_core mux::tests::server_shutdown_waits_for_async_browser_bootstrap cmux_tui_core mux::tests::server_shutdown_waits_for_in_flight_surface_creation -cmux_tui_core mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline -cmux_tui_core mux::tests::surface_creation_fence_has_a_bounded_wait cmux_tui_core mux::tests::surface_session_subscription_tracks_real_tab_moves_without_layout_churn -cmux_tui_core mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot +cmux_tui_core mux::tests::terminal_close_reports_exact_host_record_read_failure +cmux_tui_core mux::tests::terminal_move_and_destination_close_detach_view_in_one_registry_order cmux_tui_core mux::tests::workspace_mutation_close_waits_for_targeted_terminal_commit_and_replays cmux_tui_core process_session::tests::natural_reap_batch_refreshes_all_sessions_with_one_scan cmux_tui_core process_session::tests::session_members_are_never_signaled_through_a_reusable_pid cmux_tui_core process_session::tests::shutdown_treats_zombie_members_as_drained_before_reserved_parent +cmux_tui_core resource_router::auxiliary::tests::sidebar_resource_lifecycle_uses_the_real_plugin_pty_and_exact_receipts +cmux_tui_core resource_router::content::tests::one_terminal_can_be_detached_reprojected_and_closed_explicitly +cmux_tui_core resource_router::content::tests::terminal_close_terminates_a_restored_host_before_acknowledgement +cmux_tui_core resource_router::content::tests::terminal_external_effect_replays_before_selector_resolution +cmux_tui_core resource_router::content::tests::terminal_viewport_scroll_uses_one_bounded_receipt_without_session_journal_churn +cmux_tui_core resource_router::content::tests::terminal_wait_exit_resolves_detached_id_after_exit_upsert_then_delete +cmux_tui_core resource_router::content::tests::terminal_wait_exit_wakes_when_a_concurrent_terminal_close_tombstones_it +cmux_tui_core resource_router::topology::tests::pane_close_detaches_views_without_settling_terminal_exit_waits +cmux_tui_core server::tests::concurrent_terminal_close_settles_unbounded_connection_wait_exit +cmux_tui_core server::tests::connection_wait_exit_resolves_durable_detached_terminal_after_restart cmux_tui_core server::tests::daemon_shutdown_atomically_fences_native_browser_ownership +cmux_tui_core server::tests::daemon_shutdown_force_preserves_the_identity_fence cmux_tui_core server::tests::daemon_shutdown_is_local_fenced_and_queues_ack_first cmux_tui_core server::tests::clear_history_does_not_block_unrelated_surface_input_on_one_connection cmux_tui_core server::tests::guarded_browser_attach_rejects_a_late_capability_upgrade -cmux_tui_core server::tests::identify_advertises_additive_capabilities cmux_tui_core server::tests::local_shutdown_exits_after_cleanup_when_the_peer_drops_before_ack cmux_tui_core server::tests::local_shutdown_responds_before_requesting_mux_exit cmux_tui_core server::tests::local_shutdown_waits_for_the_acknowledgement_write -cmux_tui_core server::tests::repeated_identify_never_rescans_the_global_process_table +cmux_tui_core server::tests::resource_shutdown_replay_reserves_handoff_and_retries_the_post_ack_exit +cmux_tui_core server::tests::resource_shutdown_requires_local_authority_and_force_for_a_live_browser_owner +cmux_tui_core server::tests::sidebar_resource_attach_acknowledges_before_styled_snapshot_and_cancels cmux_tui_core server::tests::title_changed_event_includes_authoritative_surface_title +cmux_tui_core sidebar_resource::tests::fake_plugin_input_reaches_the_styled_render_attachment cmux_tui_core surface::tests::explicit_shutdown_deadline_does_not_wait_for_natural_cleanup_sweep cmux_tui_core surface::tests::failed_local_pty_initialization_reaps_spawned_child cmux_tui_core surface::tests::local_child_observation_uses_the_reserved_shared_reaper cmux_tui_core surface::tests::local_same_pair_alt_screen_roundtrip_forces_resolved_cursor_colors cmux_tui_core surface::tests::local_session_kill_observes_the_waitable_leader_under_its_owner_lock +cmux_tui_core surface::tests::local_surface_retains_real_status_after_final_pty_bytes cmux_tui_core surface::tests::normal_child_exit_cleans_same_session_descendants_before_reaping cmux_tui_core surface::tests::normal_child_exit_retries_a_failed_session_cleanup +cmux_tui_core surface::tests::pty_eof_publishes_final_render_frame_before_surface_removal cmux_tui_core terminal_host_runtime::unix::tests::failed_host_pty_initialization_reaps_spawned_child cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_exit_cleans_descendants_that_keep_the_slave_open cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_exit_retries_a_failed_session_cleanup cmux_tui_core terminal_host_runtime::unix::tests::hosted_child_observation_uses_the_reserved_shared_reaper cmux_tui_core terminal_host_runtime::unix::tests::hosted_session_kill_observes_the_waitable_leader_under_its_owner_lock +cmux_tui_core terminal_host_runtime::unix::tests::launch_failure_is_reported_before_bootstrap_pipe_closes cmux_tui_core terminal_host_runtime::unix::tests::spawned_host_process_drop_transfers_waiting_to_the_shared_reaper cmux_tui_core terminal_host_runtime::unix::tests::unauthenticated_host_client_handshake_is_time_bounded cmux_tui app::tests::active_scrollbar_drag_rebases_after_terminal_output_changes_geometry +cmux_tui app::tests::admitted_tab_drag_survives_a_pending_pointer_map_refresh +cmux_tui app::tests::alt_directional_focus_traverses_sidebar_at_the_pane_boundary cmux_tui app::tests::alt_n_rejects_a_new_pane_with_no_visible_content cmux_tui app::tests::alt_n_uses_zellij_default_vertical_distribution cmux_tui app::tests::attached_tab_mouse_click_uses_rendered_hit_and_survives_routing_refresh @@ -89,6 +123,7 @@ cmux_tui app::tests::catalog_refresh_preserves_machine_and_workspace_selection_i cmux_tui app::tests::clear_history_fallback_operations_remain_ordered cmux_tui app::tests::clipped_terminal_input_uses_the_logical_source_column cmux_tui app::tests::close_tab_action_honors_its_explicit_pane_target +cmux_tui app::tests::closing_connection_dialog_prevents_an_active_connect_from_replacing_the_session cmux_tui app::tests::command_k_clears_prior_output_without_a_session_mutation cmux_tui app::tests::command_k_matches_the_reported_physical_key_across_layouts cmux_tui app::tests::command_k_reaches_a_kitty_app_on_the_alternate_screen @@ -108,22 +143,29 @@ cmux_tui app::tests::disabled_mouse_snapshot_does_not_consume_press cmux_tui app::tests::dispatcher_timeout_preserves_ambiguous_press_for_recovery_release cmux_tui app::tests::doubled_prefix_keeps_the_focused_sidebar_plugin_target cmux_tui app::tests::doubled_prefix_runs_the_shared_send_prefix_action +cmux_tui app::tests::enhanced_prefixed_split_survives_pointer_mutation_and_focus_refresh cmux_tui app::tests::event_loop_renders_paint_before_following_pointer_input +cmux_tui app::tests::every_context_menu_exposes_sidebar_visibility cmux_tui app::tests::evicted_known_undelivered_motion_allows_same_cell_retry cmux_tui app::tests::failed_command_k_preserves_viewport_and_selection cmux_tui app::tests::failed_machine_switch_preserves_the_current_session cmux_tui app::tests::failed_mux_recovery_releases_barrier_and_discards_ambiguous_input cmux_tui app::tests::failed_pty_owned_press_does_not_fall_through_to_cmux_mouse_actions +cmux_tui app::tests::failed_semantic_destination_discards_only_its_dependent_input cmux_tui app::tests::files_filter_exposes_ratatui_cursor_and_accepts_mouse_cursor_placement cmux_tui app::tests::files_sidebar_renders_temp_directory_and_unread_header_badge cmux_tui app::tests::focus_loss_purges_pointer_press_waiting_for_a_paint cmux_tui app::tests::focus_sidebar_focuses_builtin_sidebar cmux_tui app::tests::focused_sidebar_tab_toggles_builtin_views_and_back cmux_tui app::tests::focused_sidebar_uses_an_accent_header_and_divider +cmux_tui app::tests::graphics_scene_cache_rebuilds_only_the_dirty_surface_projection +cmux_tui app::tests::graphics_scene_cache_skips_text_only_snapshot_rebuilds cmux_tui app::tests::guarded_mouse_encoding_rejects_a_changed_terminal_snapshot cmux_tui app::tests::guarded_mouse_encoding_rejects_changed_terminal_geometry cmux_tui app::tests::hiding_then_showing_sidebar_rehydrates_same_size_plugin_status cmux_tui app::tests::horizontal_status_bar_reserves_terminal_cells_for_wide_messages +cmux_tui app::tests::horizontal_viewport_keeps_visible_kitty_placement_aligned +cmux_tui app::tests::immediate_split_receipt_routes_following_input_past_later_creation cmux_tui app::tests::immediate_terminal_press_fails_closed_before_surface_output_marks_route_stale cmux_tui app::tests::immediate_terminal_press_rejects_content_changed_before_surface_output cmux_tui app::tests::immediate_untracked_wheel_fails_closed_before_surface_output_marks_route_stale @@ -133,6 +175,8 @@ cmux_tui app::tests::key_typed_during_pending_sidebar_focus_follows_successful_f cmux_tui app::tests::key_typed_during_pending_sidebar_focus_keeps_pane_target_on_failure cmux_tui app::tests::layout_undo_action_confirms_before_closing_a_created_pane cmux_tui app::tests::machine_session_replacement_settles_pointer_capture_on_the_old_session +cmux_tui app::tests::modifier_only_press_preserves_selection_and_durable_notice +cmux_tui app::tests::modifier_only_presses_do_not_consume_prefix_before_shifted_binding cmux_tui app::tests::mouse_wheel_scrolls_machine_and_workspace_rail_viewports_independently cmux_tui app::tests::mux_recovery_barrier_defers_input_until_authoritative_tree_is_applied cmux_tui app::tests::non_switch_machine_action_keeps_the_current_session_and_rails @@ -144,13 +188,19 @@ cmux_tui app::tests::pane_context_maximize_focuses_the_explicit_inactive_pane cmux_tui app::tests::pane_context_maximize_preserves_its_explicit_intent_after_remote_state_changes cmux_tui app::tests::pane_context_new_pane_runs_the_same_smart_layout_action_as_alt_n cmux_tui app::tests::pane_cursor_and_active_border_yield_to_builtin_and_plugin_sidebars +cmux_tui app::tests::pane_tab_context_menu_renames_the_exact_inactive_tab cmux_tui app::tests::physical_release_queues_behind_a_contended_terminal_press cmux_tui app::tests::plugin_sidebar_registers_resize_drag_hit cmux_tui app::tests::pointer_input_waits_for_a_cached_surface_attach_claim cmux_tui app::tests::pointer_motion_does_not_wait_for_terminal_parsing cmux_tui app::tests::prefix_bar_and_shortcut_modal_use_the_resolved_action_catalog +cmux_tui app::tests::prefix_chord_is_resolved_before_deferred_session_barrier cmux_tui app::tests::prefixed_clear_history_never_forwards_only_the_suffix_key cmux_tui app::tests::prepared_machine_session_events_stay_paused_until_commit_activation +cmux_tui app::tests::projection_agent_rows_hide_finished_reports +cmux_tui app::tests::projection_enter_on_active_surface_returns_focus_to_pane +cmux_tui app::tests::projection_stale_action_selection_does_not_retarget_resource_row +cmux_tui app::tests::projection_workspace_row_activates_on_mouse_down cmux_tui app::tests::pty_mouse_press_does_not_wait_for_terminal_parsing cmux_tui app::tests::pty_mouse_tracking_forwards_click_release_and_wheel_with_shift_override cmux_tui app::tests::pty_mouse_uses_the_canonical_rendered_grid_during_resize_margins @@ -159,6 +209,7 @@ cmux_tui app::tests::rejected_release_clears_the_active_pty_drag cmux_tui app::tests::remote_command_k_uses_authoritative_screen_and_keyboard_modes cmux_tui app::tests::remote_screen_switch_records_the_new_active_pane cmux_tui app::tests::remote_workspace_switch_records_the_new_active_pane +cmux_tui app::tests::repeated_split_then_ctrl_d_is_receipt_ordered_without_prefix_leaks cmux_tui app::tests::replaced_session_ignores_old_surface_lane_completion cmux_tui app::tests::replacement_provider_notice_cannot_mask_missing_workspace_mirror_error cmux_tui app::tests::retained_right_button_capture_crosses_the_menu_frame_it_opens @@ -168,6 +219,7 @@ cmux_tui app::tests::right_button_capture_cannot_cross_a_new_pairing_dialog cmux_tui app::tests::selection_drag_and_release_bypass_a_pending_focus_mutation cmux_tui app::tests::selection_drag_retargets_to_the_animated_pane_geometry cmux_tui app::tests::sidebar_context_menu_focus_is_idempotent +cmux_tui app::tests::sidebar_headers_are_the_only_pointer_entrypoint_for_rail_focus cmux_tui app::tests::sidebar_plugin_sync_is_deduped_after_it_is_applied cmux_tui app::tests::single_surface_client_rejects_hidden_browser_creation cmux_tui app::tests::single_surface_client_rejects_hidden_pane_closure @@ -177,9 +229,14 @@ cmux_tui app::tests::single_surface_status_overlays_the_terminal_without_a_statu cmux_tui app::tests::split_ratio_samples_coalesce_without_snapshots_before_final_settlement cmux_tui app::tests::stale_layout_undo_confirmation_keeps_the_created_pane cmux_tui app::tests::stale_session_events_are_ignored_after_an_in_place_switch +cmux_tui app::tests::status_message_context_menu_offers_full_message_copy +cmux_tui app::tests::status_message_drag_selection_highlights_the_visible_text +cmux_tui app::tests::surface_exit_retires_stale_view_until_authoritative_topology_refresh cmux_tui app::tests::surface_only_context_menu_omits_client_management cmux_tui app::tests::tab_selection_is_client_local cmux_tui app::tests::tab_switch_moves_size_lease_without_dropping_hidden_surface +cmux_tui app::tests::tabs_column_context_menu_renames_the_exact_clicked_tab +cmux_tui app::tests::tabs_column_renders_selected_workspace_tabs_and_activates_through_native_focus cmux_tui app::tests::terminal_geometry_mutations_enter_the_pointer_routing_lane cmux_tui app::tests::terminal_paint_reuses_unchanged_pointer_owner_snapshots cmux_tui app::tests::terminal_pointer_motion_encoding_stays_inline @@ -189,18 +246,25 @@ cmux_tui app::tests::viewport_columns_resize_with_shortcuts_and_mouse_drag cmux_tui app::tests::viewport_geometry_changes_reveal_the_active_pane_without_canceling_manual_scroll cmux_tui app::tests::viewport_pane_overflows_the_existing_tiled_layout cmux_tui app::tests::viewport_resize_drag_keeps_its_mouse_down_coordinate_origin +cmux_tui app::tests::workspace_agent_tree_renders_collapses_and_renames_the_exact_surface +cmux_tui app::tests::workspace_keyboard_enter_returns_focus_to_pane cmux_tui app::tests::workspace_rail_scrollbar_is_visible_clickable_and_draggable cmux_tui config::tests::injected_ghostty_defaults_drive_headless_render_state cmux_tui pty_input::tests::ambiguous_local_press_keeps_its_recovery_release_reservation +cmux_tui pty_input::tests::clean_terminal_exit_is_known_not_delivered_and_does_not_fail_its_lane +cmux_tui pty_input::tests::exit_observed_after_a_failed_write_keeps_delivery_ambiguous cmux_tui server_lifecycle::legacy_process::tests::frozen_tree_drop_resumes_with_its_original_process_proof -cmux_tui server_lifecycle::legacy_process::tests::termination_accepts_a_verified_process_that_already_exited cmux_tui server_lifecycle::tests::legacy_stop_terminates_only_the_identified_process +cmux_tui session::remote::tests::real_server_attach_and_resize_preserve_kitty_number_aliases cmux_tui tests::failed_server_publication_preserves_live_socket cmux_tui tests::remote_runtime_start_failure_cleans_published_server +pty attach_resize_replay_preserves_an_inflight_kitty_transmission pty attach_stream_orders_resize_between_output_frames pty attach_stream_replays_then_streams_without_duplication +pty byte_attach_between_transmit_and_place_keeps_the_unplaced_image pty byte_attach_cursor_snapshot_does_not_fan_out_a_render_frame pty control_socket_attach_palette_is_full_sparse_state_and_reset_clears_all_256 +pty control_socket_attach_serializes_kitty_aliases_on_initial_and_resize_replay pty control_socket_attach_stream_receives_merged_colors_changed pty control_socket_attach_vt_state_includes_effective_colors pty control_socket_attach_vt_state_reports_authoritative_cursor_before_replay @@ -210,13 +274,16 @@ pty control_socket_read_screen_reports_rendered_viewport_after_scrollback_clear pty control_socket_round_trip pty control_socket_wait_for_matches_one_shot_output_already_on_screen pty create_empty_workspace_is_visible_and_materialized_in_place +pty default_byte_attach_replays_preexisting_kitty_image pty default_colors_apply_to_existing_and_future_surfaces pty headless_creation_uses_explicit_or_authoritative_client_size pty new_tab_on_empty_headless_session_creates_workspace pty read_scrollback_pages_oldest_rows_and_clamps_bounds pty render_attach_headless_fans_one_frame_to_render_and_byte_consumers +pty render_attach_initial_snapshot_includes_unplaced_image_for_later_placement pty render_attach_resize_is_a_full_replacement_at_the_new_size pty render_attach_snapshot_and_raced_write_have_no_gap_or_duplicate_frame +pty render_attach_snapshot_contains_preexisting_kitty_image pty send_paste_wraps_only_while_dec_mode_2004_is_enabled pty surface_exit_detaches_terminal_view_and_emits_event pty surface_resize_reports_whether_the_size_changed From fafb0bc1162865db4b3fd0d7bdd2ff87677adebb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 01:23:42 -0700 Subject: [PATCH 367/521] fix(tui): reconcile hosted lock and formatting --- cmux-tui/Cargo.lock | 1 + .../cmux-tui-core/src/workspace_registry.rs | 22 +++++-------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index 2f3cd5a74782..034f37f62d44 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -609,6 +609,7 @@ dependencies = [ "tower-http", "url", "uuid", + "wait-timeout", "x25519-dalek", "zeroize", ] diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index c77259335513..9dfddfae5564 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -2157,14 +2157,12 @@ fn metadata_identity(metadata: &fs::Metadata) -> String { fn sqlite_filesystem_path(path: &Path) -> anyhow::Result> { #[cfg(windows)] { - let parent = path - .parent() - .ok_or_else(|| anyhow::anyhow!("workspace state path has no parent"))?; + let parent = + path.parent().ok_or_else(|| anyhow::anyhow!("workspace state path has no parent"))?; let file_name = path .file_name() .ok_or_else(|| anyhow::anyhow!("workspace state path has no file name"))?; - let parent = fs::canonicalize(parent) - .context("resolve workspace state directory")?; + let parent = fs::canonicalize(parent).context("resolve workspace state directory")?; Ok(Cow::Owned(parent.join(file_name))) } #[cfg(not(windows))] @@ -2173,18 +2171,11 @@ fn sqlite_filesystem_path(path: &Path) -> anyhow::Result> { } } -fn open_registry_database_with_flags( - path: &Path, - flags: OpenFlags, -) -> anyhow::Result { +fn open_registry_database_with_flags(path: &Path, flags: OpenFlags) -> anyhow::Result { let path = sqlite_filesystem_path(path)?; #[cfg(windows)] { - Ok(Connection::open_with_flags_and_vfs( - path.as_ref(), - flags, - "win32-longpath", - )?) + Ok(Connection::open_with_flags_and_vfs(path.as_ref(), flags, "win32-longpath")?) } #[cfg(not(windows))] { @@ -4973,8 +4964,7 @@ fn ensure_missing_pepper_can_migrate(root: &Path, pepper_path: &Path) -> anyhow: if !database.try_exists()? { continue; } - let connection = open_registry_database_read_only(&database) - .with_context(|| { + let connection = open_registry_database_read_only(&database).with_context(|| { format!("inspect registry before recreating missing pepper {}", database.display()) })?; let schema = meta_value(&connection, "schema_version")? From 71cdbef53597132aab4de1bd80b514c024d788bd Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 02:14:57 -0700 Subject: [PATCH 368/521] fix(tui): align hosted platform ownership --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 4 +--- .../src/workspace_registry/public_projection_store.rs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 6c0d3d6ccbc9..b7df2742f91b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1820,6 +1820,7 @@ impl LocalPtyProcess { if let Some(pty) = surface.as_pty() { *pty.exit.lock().unwrap() = Some(exit); } + #[cfg(windows)] close_local_terminal_master_after_exit(&surface); *process.exited.0.lock().unwrap() = true; process.exited.1.notify_all(); @@ -2632,9 +2633,6 @@ fn close_local_terminal_master_after_exit(surface: &Arc) { drop(master); } -#[cfg(not(windows))] -fn close_local_terminal_master_after_exit(_surface: &Arc) {} - fn terminal_public_id_from_resource_identity( identity: &TabResourceIdentity, invalid_context: &str, diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs index a4edd8851c91..b352791b7326 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry/public_projection_store.rs @@ -905,7 +905,7 @@ mod tests { }), ResourceChange::TombstoneTab { tab_id: tab, close_content: true }, ResourceChange::TombstoneTerminal { - public_id: terminal.clone(), + public_id: terminal, expected_incarnation: None, }, ResourceChange::SetTabOrder { pane_id: pane, tab_ids: Vec::new() }, From 976871afaced70b86c367bfe74cd2525b9f7ad35 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 02:34:46 -0700 Subject: [PATCH 369/521] fix(tui): satisfy hosted test lints Use the existing test imports and explicitly discard final best-effort cleanup results. --- cmux-tui/crates/cmux-remote/src/identity.rs | 21 +++++++++------------ cmux-tui/crates/cmux-tui/src/pty_input.rs | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-remote/src/identity.rs b/cmux-tui/crates/cmux-remote/src/identity.rs index 627547709b52..1abe6b84f1a4 100644 --- a/cmux-tui/crates/cmux-remote/src/identity.rs +++ b/cmux-tui/crates/cmux-remote/src/identity.rs @@ -2716,13 +2716,13 @@ mod tests { ); drop(blocked); - let deadline = std::time::Instant::now() + Duration::from_secs(1); + let deadline = Instant::now() + Duration::from_secs(1); loop { match AuthDatabase::load_or_create(temp.path(), "successor", false) { Ok(_) => break, Err(IdentityError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock => { assert!( - std::time::Instant::now() < deadline, + Instant::now() < deadline, "the state lease remained held after the blocking write completed" ); std::thread::yield_now(); @@ -2777,7 +2777,7 @@ mod tests { let drop_thread = std::thread::spawn(move || drop(database)); drop(blocked); drop_thread.join().unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(1); + let deadline = Instant::now() + Duration::from_secs(1); loop { match AuthDatabase::load_or_create(temp.path(), "successor", false) { Ok(successor) => { @@ -2786,7 +2786,7 @@ mod tests { } Err(IdentityError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock => { assert!( - std::time::Instant::now() < deadline, + Instant::now() < deadline, "the state lease remained held after queued persistence completed" ); std::thread::yield_now(); @@ -3100,7 +3100,7 @@ mod tests { .unwrap(); assert_eq!(unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX) }, 0); - let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let (sender, receiver) = mpsc::sync_channel(1); let creator = std::thread::spawn(move || { sender.send(load_or_create_identity(&identity_path)).unwrap(); }); @@ -4061,8 +4061,8 @@ mod tests { ); let state = Arc::new(Mutex::new(auth)); let cleanup = PendingDecisionCleanup::new(&state, invitation_id.clone(), token); - let (locked_tx, locked_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (locked_tx, locked_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); let holder = std::thread::spawn({ let state = Arc::clone(&state); move || { @@ -4077,17 +4077,14 @@ mod tests { release_tx.send(()).unwrap(); holder.join().unwrap(); - let deadline = std::time::Instant::now() + Duration::from_secs(1); + let deadline = Instant::now() + Duration::from_secs(1); loop { if let Ok(locked) = state.try_lock() && !locked.pending.contains_key(&invitation_id) { break; } - assert!( - std::time::Instant::now() < deadline, - "pending enrollment cleanup never completed" - ); + assert!(Instant::now() < deadline, "pending enrollment cleanup never completed"); std::thread::sleep(Duration::from_millis(5)); } } diff --git a/cmux-tui/crates/cmux-tui/src/pty_input.rs b/cmux-tui/crates/cmux-tui/src/pty_input.rs index 407c47ec1402..bf8fdd58bda2 100644 --- a/cmux-tui/crates/cmux-tui/src/pty_input.rs +++ b/cmux-tui/crates/cmux-tui/src/pty_input.rs @@ -1457,7 +1457,7 @@ mod tests { assert!(failure.lane_failed); assert_eq!(failure.label, "PTY input"); - mux.shutdown(); + let _ = mux.shutdown(); } fn mutation_with_retained_bytes(retained_bytes: usize) -> PtyInputEvent { From 02e921c5a0c43d32f0d6accb890538d9602b043a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 03:11:01 -0700 Subject: [PATCH 370/521] test(tui): expose unconfirmed shutdown owners Add deterministic owner-signal coverage for Chrome reaper retry confirmation and provider peer shutdown bounds. --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 57 +++++++++++++ .../cmux-tui-core/src/provider_management.rs | 80 +++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index b530f305c231..abfcd3fecf8f 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -482,6 +482,24 @@ fn parse_devtools_url(line: &str) -> Option { mod tests { use super::*; + #[cfg(unix)] + struct ForcedReaperPending; + + #[cfg(unix)] + impl ForcedReaperPending { + fn new() -> Self { + FORCE_REAPER_PENDING.store(true, Ordering::Release); + Self + } + } + + #[cfg(unix)] + impl Drop for ForcedReaperPending { + fn drop(&mut self) { + FORCE_REAPER_PENDING.store(false, Ordering::Release); + } + } + #[test] fn parses_devtools_endpoint() { assert_eq!( @@ -659,6 +677,45 @@ mod tests { assert!(reaped, "dropping Chrome discarded an unconfirmed child without reaping it"); } + #[cfg(unix)] + #[test] + fn kill_retry_waits_for_reaper_completion_before_reporting_success() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); + let child = Command::new("cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + let chrome = Chrome { + child: Mutex::new(Some(child)), + profile_dir: make_profile_dir().unwrap(), + profile_ephemeral: true, + web_socket_url: "ws://127.0.0.1/unused".to_string(), + reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), + }; + let pending = ForcedReaperPending::new(); + + FORCE_KILL_TIMEOUT.set(true); + let first = chrome.kill_until(Instant::now() + Duration::from_secs(1)); + FORCE_KILL_TIMEOUT.set(false); + wait_for_reaper_test_event(&events, ReaperTestEvent::Attempt(pid)); + + FORCE_KILL_TIMEOUT.set(true); + let second = chrome.kill_until(Instant::now() + Duration::from_secs(1)); + FORCE_KILL_TIMEOUT.set(false); + + drop(pending); + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid)); + let third = chrome.kill_until(Instant::now() + Duration::from_secs(1)); + + assert!(!first, "the first forced timeout reported completion"); + assert!(!second, "a retry reported success before the reaper confirmed process exit"); + assert!(third, "a retry did not report success after confirmed process exit"); + } + #[cfg(unix)] #[test] fn reaper_spawn_failure_never_waits_on_the_caller_thread() { diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index 6b1536bcd158..be4a2bb4922c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -225,6 +225,27 @@ pub struct ProviderManagementServer { thread: Option>, } +#[cfg(all(test, target_os = "linux"))] +struct ProviderPeerTestHook { + entered: std::sync::mpsc::SyncSender<()>, + release: Mutex>, +} + +#[cfg(all(test, target_os = "linux"))] +static PROVIDER_PEER_TEST_HOOK: Mutex>> = Mutex::new(None); + +#[cfg(all(test, target_os = "linux"))] +static PROVIDER_PEER_TEST_LOCK: Mutex<()> = Mutex::new(()); + +#[cfg(all(test, target_os = "linux"))] +fn wait_for_provider_peer_test_release() { + let hook = PROVIDER_PEER_TEST_HOOK.lock().unwrap().clone(); + if let Some(hook) = hook { + let _ = hook.entered.send(()); + let _ = hook.release.lock().unwrap().recv(); + } +} + #[cfg(any(target_os = "linux", test))] fn reap_finished_peer_threads(peers: &mut Vec>) { let mut index = 0; @@ -294,6 +315,8 @@ pub fn serve( let peer_active = active.clone(); match std::thread::Builder::new().name("provider-management-peer".into()).spawn( move || { + #[cfg(test)] + wait_for_provider_peer_test_release(); let result = (|| { let uid = peer_uid(&stream)?; if uid != 0 { @@ -443,6 +466,31 @@ mod tests { use std::sync::Arc; use std::time::{Duration, Instant}; + #[cfg(target_os = "linux")] + struct ProviderPeerTestHookGuard; + + #[cfg(target_os = "linux")] + impl ProviderPeerTestHookGuard { + fn install() -> (Self, std::sync::mpsc::Receiver<()>, std::sync::mpsc::SyncSender<()>) { + let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release, release_rx) = std::sync::mpsc::sync_channel(1); + let previous = + PROVIDER_PEER_TEST_HOOK.lock().unwrap().replace(Arc::new(ProviderPeerTestHook { + entered, + release: Mutex::new(release_rx), + })); + assert!(previous.is_none(), "provider peer test hook was already installed"); + (Self, entered_rx, release) + } + } + + #[cfg(target_os = "linux")] + impl Drop for ProviderPeerTestHookGuard { + fn drop(&mut self) { + PROVIDER_PEER_TEST_HOOK.lock().unwrap().take(); + } + } + const MUX_GENERATION: &str = "0123456789abcdef0123456789abcdef"; const AUTHORITY_ONE: &str = "provider-authority-one-0000000000000001"; const AUTHORITY_TWO: &str = "provider-authority-two-0000000000000002"; @@ -487,6 +535,38 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn dropping_server_does_not_wait_without_limit_for_a_tracked_peer() { + use std::os::unix::net::{UnixListener, UnixStream}; + + let _test_guard = PROVIDER_PEER_TEST_LOCK.lock().unwrap(); + let (hook_guard, entered, release) = ProviderPeerTestHookGuard::install(); + let socket = std::env::temp_dir() + .join(format!("cmux-provider-management-drop-deadline-{}.sock", std::process::id())); + let _ = std::fs::remove_file(&socket); + let listener = UnixListener::bind(&socket).unwrap(); + let server = serve(listener, mux()).unwrap(); + let client = UnixStream::connect(&socket).unwrap(); + let peer_entered = entered.recv_timeout(Duration::from_secs(1)).is_ok(); + let (dropped, drop_result) = std::sync::mpsc::sync_channel(1); + let dropper = std::thread::spawn(move || { + drop(server); + let _ = dropped.send(()); + }); + + let prompt = drop_result.recv_timeout(Duration::from_secs(1)).is_ok(); + let _ = release.send(()); + drop(client); + let dropper_result = dropper.join(); + drop(hook_guard); + let _ = std::fs::remove_file(socket); + + assert!(peer_entered, "the provider peer did not reach its tracked owner signal"); + assert!(dropper_result.is_ok(), "the provider server drop helper panicked"); + assert!(prompt, "provider server drop waited without limit for a tracked peer"); + } + #[cfg(target_os = "linux")] #[test] fn explicit_unsupported_response_is_the_only_upgrade_signal() { From dc5789d84f47cd312ad313423c8baf428e5f69f1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 03:21:28 -0700 Subject: [PATCH 371/521] fix(tui): retain shutdown lifecycle owners Keep process cleanup receipts across retries, restore terminal-close ownership, align deterministic shutdown fixtures, and remove stale review-policy shapes. --- ...lSurfaceRuntimeTeardownExecutionLane.swift | 49 ---------- ...minalSurfaceRuntimeTeardownStartGate.swift | 51 ++++++++++ cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 97 +++++++++++++------ .../cmux-tui-core/src/provider_management.rs | 65 +++++++++++-- .../src/resource_router/content.rs | 10 -- cmux-tui/crates/cmux-tui-core/src/server.rs | 6 +- cmux-tui/crates/cmux-tui/src/main.rs | 10 +- .../crates/cmux-tui/src/remote_runtime.rs | 79 +++++++++++---- .../src/server_lifecycle/legacy_process.rs | 15 ++- cmux-tui/crates/cmux-tui/tests/cli.rs | 14 ++- tests/test_tui_workflow_toolchain.py | 97 ------------------- 11 files changed, 270 insertions(+), 223 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift delete mode 100644 tests/test_tui_workflow_toolchain.py diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift index 540fade6667b..92f378d54ce3 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -1,5 +1,4 @@ internal import Foundation -internal import os /// Selects the ownership boundary for a native surface free. nonisolated enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { @@ -59,51 +58,3 @@ nonisolated struct TerminalSurfaceRuntimePreparedTeardownExecution: Sendable { startGate.start() } } - -/// Safety: the lock protects every state transition and the stored continuation. -/// The single waiter and idempotent start move that continuation out at most -/// once, and resume it only after releasing the lock. -private final class TerminalSurfaceRuntimeTeardownStartGate: @unchecked Sendable { - private enum State { - case pending - case waiting(CheckedContinuation) - case started - } - - private let state = OSAllocatedUnfairLock(initialState: State.pending) - - func wait() async { - await withCheckedContinuation { continuation in - let startImmediately = state.withLock { state in - switch state { - case .pending: - state = .waiting(continuation) - return false - case .started: - return true - case .waiting: - preconditionFailure("teardown start gate waited more than once") - } - } - if startImmediately { - continuation.resume() - } - } - } - - func start() { - let continuation = state.withLock { state in - switch state { - case .pending: - state = .started - return nil - case .waiting(let continuation): - state = .started - return continuation - case .started: - return nil - } - } - continuation?.resume() - } -} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift new file mode 100644 index 000000000000..41df806dafb7 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift @@ -0,0 +1,51 @@ +internal import os + +/// Safety: the lock protects every state transition and the stored continuation. +/// The single waiter and idempotent start move that continuation out at most +/// once, and resume it only after releasing the lock. +final class TerminalSurfaceRuntimeTeardownStartGate: @unchecked Sendable { + private enum State { + case pending + case waiting(CheckedContinuation) + case started + } + + private let state = OSAllocatedUnfairLock(initialState: State.pending) + + func wait() async { + await withCheckedContinuation { continuation in + let startImmediately = state.withLock { state in + switch state { + case .pending: + state = .waiting(continuation) + return false + case .started: + return true + case .waiting: + preconditionFailure( + "teardown start gate waited more than once" + ) + } + } + if startImmediately { + continuation.resume() + } + } + } + + func start() { + let continuation = state.withLock { state in + switch state { + case .pending: + state = .started + return nil + case .waiting(let continuation): + state = .started + return continuation + case .started: + return nil + } + } + continuation?.resume() + } +} diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index abfcd3fecf8f..b13d8cbf5da8 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -55,11 +55,16 @@ pub enum BrowserMode { /// A launched Chrome/Chromium process plus its profile dir. pub struct Chrome { - child: Mutex>, + process: Mutex, profile_dir: PathBuf, profile_ephemeral: bool, web_socket_url: String, - reaper: Mutex>, +} + +enum ChromeProcessState { + Owned { child: Child, reaper: ChromeReaperLease }, + Reaping { completion: mpsc::Receiver<()> }, + Complete, } impl Chrome { @@ -145,11 +150,10 @@ impl Chrome { }; Ok(Chrome { - child: Mutex::new(Some(child)), + process: Mutex::new(ChromeProcessState::Owned { child, reaper }), profile_dir, profile_ephemeral, web_socket_url, - reaper: Mutex::new(Some(reaper)), }) } @@ -162,30 +166,41 @@ impl Chrome { } pub fn kill_until(&self, deadline: Instant) -> bool { - let mut slot = self.child.lock().unwrap(); - let Some(child) = slot.take() else { return true }; - drop(slot); - let reaper = - self.reaper.lock().unwrap().take().expect("live Chrome retains its reaper lease"); - reap_child_until( - reaper, - child, - self.profile_ephemeral.then(|| self.profile_dir.clone()), - deadline, - ) + let mut state = self.process.lock().unwrap(); + if matches!(&*state, ChromeProcessState::Owned { .. }) { + let ChromeProcessState::Owned { child, reaper } = + std::mem::replace(&mut *state, ChromeProcessState::Complete) + else { + unreachable!() + }; + let completion = reap_child_with_completion( + reaper, + child, + self.profile_ephemeral.then(|| self.profile_dir.clone()), + ); + *state = ChromeProcessState::Reaping { completion }; + } + let completed = match &*state { + ChromeProcessState::Owned { .. } => unreachable!(), + ChromeProcessState::Reaping { completion } => { + wait_for_reap_completion(completion, deadline) + } + ChromeProcessState::Complete => true, + }; + if completed { + *state = ChromeProcessState::Complete; + } + completed } } impl Drop for Chrome { fn drop(&mut self) { - let child = self.child.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()).take(); - if let Some(child) = child { + let state = self.process.get_mut().unwrap_or_else(|poisoned| poisoned.into_inner()); + let state = std::mem::replace(state, ChromeProcessState::Complete); + if let ChromeProcessState::Owned { child, reaper } = state { reap_child_detached( - self.reaper - .get_mut() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .take() - .expect("live Chrome retains its reaper lease"), + reaper, child, self.profile_ephemeral.then(|| self.profile_dir.clone()), ); @@ -277,14 +292,27 @@ fn reap_child_until( profile_dir: Option, deadline: Instant, ) -> bool { + let completion = reap_child_with_completion(reaper, child, profile_dir); + wait_for_reap_completion(&completion, deadline) +} + +fn reap_child_with_completion( + reaper: ChromeReaperLease, + child: Child, + profile_dir: Option, +) -> mpsc::Receiver<()> { let (completion_tx, completion_rx) = mpsc::sync_channel(1); reap_child(reaper, child, profile_dir, Some(completion_tx)); + completion_rx +} + +fn wait_for_reap_completion(completion: &mpsc::Receiver<()>, deadline: Instant) -> bool { #[cfg(test)] if FORCE_KILL_TIMEOUT.get() { return false; } let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false }; - completion_rx.recv_timeout(remaining).is_ok() + completion.recv_timeout(remaining).is_ok() } fn reap_child( @@ -591,15 +619,17 @@ mod tests { .unwrap(); let profile_dir = make_profile_dir().unwrap(); let chrome = Chrome { - child: Mutex::new(Some(child)), + process: Mutex::new(ChromeProcessState::Owned { + child, + reaper: chrome_reaper_lease().unwrap(), + }), profile_dir, profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); - assert!(chrome.child.lock().unwrap().is_none()); + assert!(matches!(&*chrome.process.lock().unwrap(), ChromeProcessState::Complete)); } #[cfg(unix)] @@ -617,11 +647,10 @@ mod tests { let active = Arc::new(AtomicUsize::new(0)); let reaper = reserve_reaper_lease(sender.clone(), active.clone(), 1).unwrap(); let chrome = Chrome { - child: Mutex::new(Some(child)), + process: Mutex::new(ChromeProcessState::Owned { child, reaper }), profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Mutex::new(Some(reaper)), }; assert!(chrome.kill_until(Instant::now() + Duration::from_secs(1))); @@ -654,11 +683,13 @@ mod tests { .unwrap(); let pid = i32::try_from(child.id()).unwrap(); let chrome = Chrome { - child: Mutex::new(Some(child)), + process: Mutex::new(ChromeProcessState::Owned { + child, + reaper: chrome_reaper_lease().unwrap(), + }), profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), }; FORCE_KILL_TIMEOUT.set(true); @@ -690,11 +721,13 @@ mod tests { .unwrap(); let pid = child.id(); let chrome = Chrome { - child: Mutex::new(Some(child)), + process: Mutex::new(ChromeProcessState::Owned { + child, + reaper: chrome_reaper_lease().unwrap(), + }), profile_dir: make_profile_dir().unwrap(), profile_ephemeral: true, web_socket_url: "ws://127.0.0.1/unused".to_string(), - reaper: Mutex::new(Some(chrome_reaper_lease().unwrap())), }; let pending = ForcedReaperPending::new(); diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index be4a2bb4922c..fa3d2220135f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -10,9 +10,9 @@ use std::path::Path; #[cfg(target_os = "linux")] use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; #[cfg(target_os = "linux")] -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, mpsc}; #[cfg(target_os = "linux")] -use std::time::Duration; +use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; use zeroize::Zeroize; @@ -223,6 +223,7 @@ fn peer_uid(stream: &std::os::unix::net::UnixStream) -> io::Result { pub struct ProviderManagementServer { shutdown: Arc, thread: Option>, + completion: Option>, } #[cfg(all(test, target_os = "linux"))] @@ -260,18 +261,34 @@ fn reap_finished_peer_threads(peers: &mut Vec>) { } #[cfg(target_os = "linux")] -impl Drop for ProviderManagementServer { - fn drop(&mut self) { +impl ProviderManagementServer { + pub fn shutdown_until(&mut self, deadline: Instant) -> bool { self.shutdown.store(true, Ordering::Release); + let Some(completion) = self.completion.as_ref() else { return true }; + let remaining = deadline.saturating_duration_since(Instant::now()); + if completion.recv_timeout(remaining).is_err() { + return false; + } + self.completion.take(); if let Some(thread) = self.thread.take() { let _ = thread.join(); } + true + } +} + +#[cfg(target_os = "linux")] +impl Drop for ProviderManagementServer { + fn drop(&mut self) { + // The listener thread owns its listener and peer handles. A plain drop + // requests shutdown and leaves that thread to finish its owned cleanup. + self.shutdown.store(true, Ordering::Release); } } /// Serve the systemd-provided listener until the returned owner is dropped. -/// Dropping the owner closes every active peer and joins all management -/// threads before returning. +/// `shutdown_until` confirms that every active peer and management thread has +/// finished. Dropping without that confirmation requests detached cleanup. #[cfg(target_os = "linux")] pub fn serve( listener: std::os::unix::net::UnixListener, @@ -284,6 +301,7 @@ pub fn serve( listener.set_nonblocking(true)?; let shutdown = Arc::new(AtomicBool::new(false)); let thread_shutdown = shutdown.clone(); + let (completion_tx, completion_rx) = mpsc::sync_channel(1); let thread = std::thread::Builder::new().name("provider-management".into()).spawn(move || { let active = Arc::new(Mutex::new(std::collections::HashMap::new())); @@ -350,8 +368,9 @@ pub fn serve( for peer in peers { let _ = peer.join(); } + let _ = completion_tx.send(()); })?; - Ok(ProviderManagementServer { shutdown, thread: Some(thread) }) + Ok(ProviderManagementServer { shutdown, thread: Some(thread), completion: Some(completion_rx) }) } #[derive(Debug)] @@ -567,6 +586,38 @@ mod tests { assert!(prompt, "provider server drop waited without limit for a tracked peer"); } + #[cfg(target_os = "linux")] + #[test] + fn shutdown_until_retains_owner_and_receipt_across_a_missed_deadline() { + use std::os::unix::net::{UnixListener, UnixStream}; + + let _test_guard = PROVIDER_PEER_TEST_LOCK.lock().unwrap(); + let (hook_guard, entered, release) = ProviderPeerTestHookGuard::install(); + let socket = std::env::temp_dir() + .join(format!("cmux-provider-management-shutdown-retry-{}.sock", std::process::id())); + let _ = std::fs::remove_file(&socket); + let listener = UnixListener::bind(&socket).unwrap(); + let mut server = serve(listener, mux()).unwrap(); + let client = UnixStream::connect(&socket).unwrap(); + let peer_entered = entered.recv_timeout(Duration::from_secs(1)).is_ok(); + + let first = server.shutdown_until(Instant::now()); + let retained = server.thread.is_some() && server.completion.is_some(); + let _ = release.send(()); + drop(client); + let second = server.shutdown_until(Instant::now() + Duration::from_secs(5)); + let cleared = server.thread.is_none() && server.completion.is_none(); + drop(server); + drop(hook_guard); + let _ = std::fs::remove_file(socket); + + assert!(peer_entered, "the provider peer did not reach its tracked owner signal"); + assert!(!first, "an expired deadline reported provider shutdown completion"); + assert!(retained, "a missed deadline discarded the provider owner or completion receipt"); + assert!(second, "the released provider peer did not complete within the final deadline"); + assert!(cleared, "confirmed provider shutdown retained its owner or completion receipt"); + } + #[cfg(target_os = "linux")] #[test] fn explicit_unsupported_response_is_the_only_upgrade_signal() { diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs index b029b919fdee..793b405cc48d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs @@ -365,16 +365,6 @@ fn execute_terminal_effect( ResourceError::not_found("terminal", terminal_id.as_str()), ); }; - if prepared.operation == "terminal.close" { - let commit = mux.commit_resource_content_close_effect( - surface_id, - &ContentPublicId::Terminal(terminal_id), - &prepared.idempotency_key, - &prepared.operation, - &prepared.fingerprint, - ); - return finish_projection_commit(mux, prepared, commit); - } let Some(surface) = mux.surface(surface_id) else { return effects::commit_known_failure( mux, diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 8facbfbd13f2..67f88fb93265 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -9465,7 +9465,7 @@ fn handle_command_with_cancellation( anyhow::bail!("shutdown is only available over the local session socket"); } mux.close_all_surfaces_for_shutdown() - .map_err(|_| anyhow::anyhow!(SERVER_SHUTDOWN_INCOMPLETE_ERROR))?; + .map_err(|error| error.context(SERVER_SHUTDOWN_INCOMPLETE_ERROR))?; Ok(json!({})) } Command::ShutdownDaemon { pid, generation, force } => { @@ -16546,6 +16546,10 @@ mod tests { let error = handle_command(&mux, client, Command::Shutdown, &writer).unwrap_err(); assert_eq!(error.to_string(), SERVER_SHUTDOWN_INCOMPLETE_ERROR); + assert!( + format!("{error:#}").contains("could not terminate 1 surface process"), + "shutdown diagnostic discarded its retained owner: {error:#}" + ); } #[test] diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 8d4f76f5c199..4a8776827704 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -1419,7 +1419,15 @@ struct ServerProcessOwners { impl ServerProcessOwners { fn shutdown_until(&mut self, deadline: std::time::Instant) -> ServerProcessOwnersShutdown { #[cfg(target_os = "linux")] - drop(self.provider_management.take()); + if let Some(provider_management) = self.provider_management.as_mut() { + if provider_management.shutdown_until(deadline) { + drop(self.provider_management.take()); + } else { + return ServerProcessOwnersShutdown::Pending { + message: "provider management cleanup exceeded the shutdown deadline", + }; + } + } if let Some(websocket) = self.websocket.as_mut() { match websocket.shutdown_until(deadline) { Ok(true) => drop(self.websocket.take()), diff --git a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs index 63d837e0e9e6..e338cf3fa7d6 100644 --- a/cmux-tui/crates/cmux-tui/src/remote_runtime.rs +++ b/cmux-tui/crates/cmux-tui/src/remote_runtime.rs @@ -339,6 +339,10 @@ pub struct DaemonRuntimeHandle { shutdown: watch::Sender, thread: Option>>, completion: Arc, + #[cfg(debug_assertions)] + shutdown_test_started: bool, + #[cfg(debug_assertions)] + shutdown_test_completed: bool, } #[derive(Default)] @@ -400,7 +404,42 @@ impl DaemonRuntimeHandle { let _ = self.shutdown.send(true); } + #[cfg(debug_assertions)] + fn begin_shutdown_test_hooks(&mut self) -> anyhow::Result<()> { + if self.shutdown_test_started { + return Ok(()); + } + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_STARTED_SIGNAL") { + fs::write(signal, b"1")?; + } + if let Some(gate) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_GATE") { + use std::io::Read as _; + + let mut gate = fs::File::open(gate)?; + let mut signal = [0_u8; 1]; + gate.read_exact(&mut signal)?; + } + self.shutdown_test_started = true; + Ok(()) + } + + #[cfg(debug_assertions)] + fn complete_shutdown_test_hooks(&mut self) -> anyhow::Result<()> { + if self.shutdown_test_completed { + return Ok(()); + } + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_COMPLETE_SIGNAL") { + fs::write(signal, b"1")?; + } + self.shutdown_test_completed = true; + Ok(()) + } + pub(crate) fn shutdown_until(&mut self, deadline: std::time::Instant) -> DaemonRuntimeShutdown { + #[cfg(debug_assertions)] + if let Err(error) = self.begin_shutdown_test_hooks() { + return DaemonRuntimeShutdown::Complete(Err(error)); + } self.request_shutdown(); if self.thread.is_none() { return DaemonRuntimeShutdown::Complete(Err(anyhow!( @@ -408,7 +447,12 @@ impl DaemonRuntimeHandle { ))); } if self.completion.wait_until(deadline) { - DaemonRuntimeShutdown::Complete(self.join_runtime_thread()) + let result = self.join_runtime_thread(); + #[cfg(debug_assertions)] + if let Err(error) = self.complete_shutdown_test_hooks() { + return DaemonRuntimeShutdown::Complete(Err(error)); + } + DaemonRuntimeShutdown::Complete(result) } else { DaemonRuntimeShutdown::Pending } @@ -454,30 +498,20 @@ impl DaemonRuntimeHandle { shutdown, thread: Some(thread), completion, + #[cfg(debug_assertions)] + shutdown_test_started: false, + #[cfg(debug_assertions)] + shutdown_test_completed: false, } } pub fn shutdown(mut self) -> anyhow::Result<()> { #[cfg(debug_assertions)] - let shutdown_started = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_STARTED_SIGNAL"); - #[cfg(debug_assertions)] - if let Some(signal) = shutdown_started.as_ref() { - fs::write(signal, b"1")?; - } - #[cfg(debug_assertions)] - if let Some(gate) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_GATE") { - use std::io::Read as _; - - let mut gate = fs::File::open(gate)?; - let mut signal = [0_u8; 1]; - gate.read_exact(&mut signal)?; - } + self.begin_shutdown_test_hooks()?; self.request_shutdown(); let result = self.join_runtime_thread(); #[cfg(debug_assertions)] - if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_REMOTE_SHUTDOWN_COMPLETE_SIGNAL") { - fs::write(signal, b"1")?; - } + self.complete_shutdown_test_hooks()?; result } } @@ -1898,7 +1932,16 @@ fn start_daemon_runtime_with_timeout( return Err(anyhow!("remote daemon did not become ready: {error}")); } }; - Ok(DaemonRuntimeHandle { info, shutdown: shutdown_tx, thread: Some(thread), completion }) + Ok(DaemonRuntimeHandle { + info, + shutdown: shutdown_tx, + thread: Some(thread), + completion, + #[cfg(debug_assertions)] + shutdown_test_started: false, + #[cfg(debug_assertions)] + shutdown_test_completed: false, + }) } #[allow(clippy::too_many_arguments)] diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index eb6c6ff9b3d3..a8b1cb5b80c6 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -967,12 +967,13 @@ mod tests { Err(error) => panic!("failed to drain stopped child output: {error}"), } } + assert_eq!(unsafe { libc::fcntl(descriptor, libc::F_SETFL, flags) }, 0); + let (resumed_sender, resumed_receiver) = mpsc::sync_channel(1); + let resumed_reader = std::thread::spawn(move || { + resumed_sender.send(stdout.read(&mut output).map(|read| read != 0)).unwrap(); + }); drop(tree); - let mut poll_descriptor = libc::pollfd { fd: descriptor, events: libc::POLLIN, revents: 0 }; - // SAFETY: poll_descriptor points to one initialized pollfd for this call. - assert_eq!(unsafe { libc::poll(&raw mut poll_descriptor, 1, 250) }, 1); - assert_ne!(poll_descriptor.revents & libc::POLLIN, 0); - let resumed = stdout.read(&mut output).unwrap() != 0; + let resumed = resumed_receiver.recv_timeout(Duration::from_millis(250)); STABLE_HANDLE_CAPTURE_BUDGET.set(None); // SAFETY: only the exact test-owned child can still use this unreaped PID. @@ -981,7 +982,11 @@ mod tests { } let _ = child.kill(); let _ = child.wait(); + resumed_reader.join().unwrap(); + let resumed = resumed + .expect("resumed child did not publish output before the final failure deadline") + .expect("failed to read resumed child output"); assert!(resumed, "dropping a process fence left its exact process stopped"); } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 9687d9883ac2..2ed313d7ede2 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -873,7 +873,7 @@ fn legacy_server_process_helper() { "self": false, })); } - writeln!( + if writeln!( stream, "{}", serde_json::json!({ @@ -882,7 +882,15 @@ fn legacy_server_process_helper() { "data": clients, }) ) - .unwrap(); + .is_err() + { + // The bounded helper can reject exclusivity and close this + // client before its last response is written. A real legacy + // server keeps serving after that client disconnects. + loop { + std::thread::park(); + } + } continue; } assert_eq!(list_request["cmd"].as_str(), Some("list-workspaces")); @@ -2633,7 +2641,7 @@ fn incompatible_server_status_and_public_api_use_separate_compatibility_domains( } else { commands.push(request["operation"].as_str().unwrap().to_string()); serde_json::json!({ - "protocol": "cmux.protocol/1", + "protocol": cmux_tui_core::resource::PROTOCOL, "type": "response", "id": request["id"], "ok": true, diff --git a/tests/test_tui_workflow_toolchain.py b/tests/test_tui_workflow_toolchain.py deleted file mode 100644 index c7f13c2088f4..000000000000 --- a/tests/test_tui_workflow_toolchain.py +++ /dev/null @@ -1,97 +0,0 @@ -from pathlib import Path -import re - - -ROOT = Path(__file__).resolve().parents[1] -WORKFLOW = ROOT / ".github" / "workflows" / "cmux-tui.yml" -VALGRIND_PIDFD_SKIPS = ROOT / "cmux-tui" / "dist" / "valgrind-pidfd-skips.txt" -VALGRIND_EXECUTION_JOB = "valgrind-leak-check-shard" -RUST_JOBS = ( - VALGRIND_EXECUTION_JOB, - "test", - "bindings-e2e", - "windows-experimental", -) -BROWSER_RUNTIME_PIDFD_TESTS = { - "browser_capture_scale_applies_to_metrics_screencast_and_input", - "browser_tab_creation_is_async_and_surfaces_bootstrap_failure", - "control_command_reports_backpressure_when_worker_queue_is_full", - "queued_back_and_forward_do_not_collapse_while_worker_is_blocked", - "socket_browser_attach_streams_frames_input_and_cell_pixels", - "stalled_external_browser_nudges_target_once_before_interaction", - "wedged_browser_navigate_does_not_block_same_socket_connection", -} -CMUX_TUI_CORE_VALGRIND_INCOMPATIBLE_TESTS = { - "browser::tests::ordinary_repaint_preserves_captured_drag_motion", - "browser::tests::server_shutdown_bounds_external_target_confirmation", - "mux::tests::bulk_surface_close_uses_one_shared_termination_deadline", - "mux::tests::ordinary_browser_close_terminates_the_owner_staged_during_removal", - "mux::tests::shutdown_fanout_does_not_claim_another_batch_after_the_deadline", - "mux::tests::surface_creation_fence_has_a_bounded_wait", - "mux::tests::terminal_adoption_rescan_uses_one_registry_snapshot", - "server::tests::clear_history_does_not_block_unrelated_surface_input_on_one_connection", -} - - -def job_block(workflow: str, job: str) -> str: - marker = f"\n {job}:\n" - remainder = workflow.split(marker, 1)[1] - next_job = re.search(r"\n [a-zA-Z0-9_-]+:\n", remainder) - return remainder[: next_job.start()] if next_job else remainder - - -def test_tui_rust_jobs_install_the_release_toolchain() -> None: - workflow = WORKFLOW.read_text() - assert '\nenv:\n RUST_TOOLCHAIN: "1.95.0"\n' in workflow - - for job in RUST_JOBS: - block = job_block(workflow, job) - assert "rustup toolchain install \"$RUST_TOOLCHAIN\" --profile minimal" in block - assert "rustup default \"$RUST_TOOLCHAIN\"" in block - assert "if ! command -v cargo" not in block - - test_job = job_block(workflow, "test") - assert "--component clippy,rustfmt" in test_job - - -def test_valgrind_fallback_keeps_non_pidfd_tests() -> None: - entries = [ - tuple(line.split(maxsplit=1)) - for line in VALGRIND_PIDFD_SKIPS.read_text().splitlines() - if line and not line.startswith("#") - ] - assert all(len(entry) == 2 for entry in entries) - assert len(entries) == len(set(entries)) - assert {scope for scope, _ in entries} == { - "browser_runtime", - "cmux_tui", - "cmux_tui_core", - "pty", - "websocket_transport", - } - assert { - test_name for scope, test_name in entries if scope == "browser_runtime" - } == BROWSER_RUNTIME_PIDFD_TESTS - assert CMUX_TUI_CORE_VALGRIND_INCOMPATIBLE_TESTS <= { - test_name for scope, test_name in entries if scope == "cmux_tui_core" - } - - valgrind_job = job_block(WORKFLOW.read_text(), VALGRIND_EXECUTION_JOB) - assert "Skipping pidfd-dependent test binary" not in valgrind_job - assert "dist/valgrind-pidfd-skips.txt" in valgrind_job - assert "test_scope=browser_runtime" in valgrind_job - assert 'test_args+=(--skip "$test_name")' in valgrind_job - - -def test_macos_serializes_process_barrier_tests() -> None: - test_job = job_block(WORKFLOW.read_text(), "test") - assert 'if [[ "$RUNNER_OS" == "macOS" ]]; then' in test_job - assert 'export CMUX_TEST_TIMEOUT_SCALE="4"' in test_job - assert "args+=(-- --test-threads=1)" in test_job - assert 'cargo test --workspace --locked "${args[@]}"' in test_job - - -if __name__ == "__main__": - test_tui_rust_jobs_install_the_release_toolchain() - test_valgrind_fallback_keeps_non_pidfd_tests() - test_macos_serializes_process_barrier_tests() From cdb2aae6c9ced8b6dd64891b89d933ad8ac66458 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 03:46:45 -0700 Subject: [PATCH 372/521] fix(tui): publish machine identity atomically Use no-replace rename instead of a timed hard-link window, and align provider test channel imports with hosted Clippy. --- .../cmux-tui-core/src/provider_management.rs | 18 ++-- .../cmux-tui/src/machine_agent/identity.rs | 93 ++++++++++++------- 2 files changed, 68 insertions(+), 43 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs index fa3d2220135f..1cf2033fce9d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/provider_management.rs +++ b/cmux-tui/crates/cmux-tui-core/src/provider_management.rs @@ -9,8 +9,10 @@ use std::io::{self, Read, Write}; use std::path::Path; #[cfg(target_os = "linux")] use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +#[cfg(any(target_os = "linux", test))] +use std::sync::mpsc; #[cfg(target_os = "linux")] -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::{Arc, Mutex}; #[cfg(target_os = "linux")] use std::time::{Duration, Instant}; @@ -228,8 +230,8 @@ pub struct ProviderManagementServer { #[cfg(all(test, target_os = "linux"))] struct ProviderPeerTestHook { - entered: std::sync::mpsc::SyncSender<()>, - release: Mutex>, + entered: mpsc::SyncSender<()>, + release: Mutex>, } #[cfg(all(test, target_os = "linux"))] @@ -490,9 +492,9 @@ mod tests { #[cfg(target_os = "linux")] impl ProviderPeerTestHookGuard { - fn install() -> (Self, std::sync::mpsc::Receiver<()>, std::sync::mpsc::SyncSender<()>) { - let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); - let (release, release_rx) = std::sync::mpsc::sync_channel(1); + fn install() -> (Self, mpsc::Receiver<()>, mpsc::SyncSender<()>) { + let (entered, entered_rx) = mpsc::sync_channel(1); + let (release, release_rx) = mpsc::sync_channel(1); let previous = PROVIDER_PEER_TEST_HOOK.lock().unwrap().replace(Arc::new(ProviderPeerTestHook { entered, @@ -541,7 +543,7 @@ mod tests { assert!(Instant::now() < deadline, "peer thread did not finish"); std::thread::yield_now(); } - let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); let active = std::thread::spawn(move || release_rx.recv().unwrap()); let mut peers = vec![completed, active]; @@ -568,7 +570,7 @@ mod tests { let server = serve(listener, mux()).unwrap(); let client = UnixStream::connect(&socket).unwrap(); let peer_entered = entered.recv_timeout(Duration::from_secs(1)).is_ok(); - let (dropped, drop_result) = std::sync::mpsc::sync_channel(1); + let (dropped, drop_result) = mpsc::sync_channel(1); let dropper = std::thread::spawn(move || { drop(server); let _ = dropped.send(()); diff --git a/cmux-tui/crates/cmux-tui/src/machine_agent/identity.rs b/cmux-tui/crates/cmux-tui/src/machine_agent/identity.rs index fb4323cd6576..32a934fbfee7 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_agent/identity.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_agent/identity.rs @@ -1,11 +1,11 @@ +use std::ffi::CString; use std::fmt; use std::fs::{self, DirBuilder, File, OpenOptions}; use std::io::{self, Read, Write}; use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; use std::path::{Path, PathBuf}; -use std::thread; -use std::time::Duration; use cmux_tui_machine_agent_protocol::{MachineSecret, OpaqueId, SessionName}; use serde::{Deserialize, Serialize}; @@ -13,7 +13,6 @@ use zeroize::Zeroize; const STATE_VERSION: u16 = 1; const MAX_STATE_BYTES: u64 = 4096; -const CONCURRENT_CREATE_RETRY_DELAY: Duration = Duration::from_millis(10); #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct MachineIdentity { @@ -56,7 +55,7 @@ pub(super) fn load_or_create(path: &Path) -> anyhow::Result { match fs::symlink_metadata(path) { Ok(_) => { remove_orphaned_temporary_links(path)?; - load_with_link_retry(path) + load(path) } Err(error) if error.kind() == io::ErrorKind::NotFound => create(path), Err(error) => Err(error.into()), @@ -127,18 +126,6 @@ fn load(path: &Path) -> anyhow::Result { Ok(MachineIdentity { machine_id: stored.machine_id, secret: stored.secret }) } -fn load_with_link_retry(path: &Path) -> anyhow::Result { - match load(path) { - Err(error) - if error.downcast_ref::().is_some_and(|error| error.0 == 2) => - { - thread::sleep(CONCURRENT_CREATE_RETRY_DELAY); - load(path) - } - result => result, - } -} - fn create(path: &Path) -> anyhow::Result { let parent = state_parent(path)?; let identity = MachineIdentity { @@ -163,16 +150,11 @@ fn create(path: &Path) -> anyhow::Result { verify_private_file(&file, "temporary machine-agent state")?; file.write_all(&encoded)?; file.sync_all()?; - match fs::hard_link(&temporary, path) { + match atomic_rename_noreplace(&temporary, path) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::AlreadyExists => return Ok(false), Err(error) => return Err(error.into()), } - match fs::remove_file(&temporary) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } sync_directory(parent)?; Ok(true) })(); @@ -187,10 +169,48 @@ fn create(path: &Path) -> anyhow::Result { } Ok(loaded) } else { - load_with_link_retry(path) + load(path) } } +fn atomic_rename_noreplace(from: &Path, to: &Path) -> io::Result<()> { + let from = CString::new(from.as_os_str().as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "source path contains a NUL byte") + })?; + let to = CString::new(to.as_os_str().as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "target path contains a NUL byte") + })?; + #[cfg(target_vendor = "apple")] + let result = unsafe { + libc::renameatx_np( + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_EXCL, + ) + }; + #[cfg(target_os = "linux")] + let result = unsafe { + libc::syscall( + libc::SYS_renameat2, + libc::AT_FDCWD, + from.as_ptr(), + libc::AT_FDCWD, + to.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + #[cfg(not(any(target_vendor = "apple", target_os = "linux")))] + let result = { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "atomic no-replace rename is unavailable on this platform", + )); + }; + if result == 0 { Ok(()) } else { Err(io::Error::last_os_error()) } +} + fn open_no_follow(path: &Path, write: bool) -> io::Result { OpenOptions::new() .read(!write) @@ -407,18 +427,21 @@ mod tests { } #[test] - fn identity_retries_the_live_two_link_creation_window_once() { - let state = test_state("live-creation-link"); - let identity = load_or_create(&state.path).unwrap(); - let temporary = state.directory.join(".identity.json.live.tmp"); - fs::hard_link(&state.path, &temporary).unwrap(); - let remover = thread::spawn(move || { - thread::sleep(Duration::from_millis(1)); - fs::remove_file(temporary).unwrap(); - }); - - assert_eq!(load_with_link_retry(&state.path).unwrap(), identity); - remover.join().unwrap(); + fn identity_creation_loser_preserves_atomic_winner_with_one_link() { + let state = test_state("concurrent-creation-loser"); + let winner = create(&state.path).unwrap(); + let loser = create(&state.path).unwrap(); + + assert_eq!(loser, winner); + assert_eq!(load(&state.path).unwrap(), winner); + assert_eq!(fs::metadata(&state.path).unwrap().nlink(), 1); + let temporary_prefix = ".identity.json."; + let temporary_suffix = ".tmp"; + assert!(!fs::read_dir(&state.directory).unwrap().any(|entry| { + let name = entry.unwrap().file_name(); + let name = name.to_string_lossy(); + name.starts_with(temporary_prefix) && name.ends_with(temporary_suffix) + })); } #[test] From 0959171abdbfb054e8fed5b947a5f37d997edc0a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:14:47 -0700 Subject: [PATCH 373/521] test(tui): cover process scan spawn barrier --- cmux-tui/crates/cmux-tui/src/config.rs | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index cf3d55d44dc4..4c50db2bfd97 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -5586,6 +5586,37 @@ mod tests { assert!(matches!(result, GhosttyHelperDefaults::Resolved(_))); } + #[cfg(target_os = "macos")] + #[test] + fn ghostty_process_scan_waits_for_the_process_creation_barrier() { + let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = mpsc::channel(); + let (result_sender, result_receiver) = mpsc::channel(); + let helper = std::thread::spawn(move || { + started_sender.send(()).unwrap(); + let result = ghostty_helper_process_table_snapshot(); + result_sender.send(result).unwrap(); + }); + let started = started_receiver.recv_timeout(Duration::from_secs(1)).is_ok(); + let early = result_receiver.recv_timeout(Duration::from_millis(250)); + let waited = matches!(&early, Err(mpsc::RecvTimeoutError::Timeout)); + + drop(barrier); + let result = match early { + Ok(result) => result, + Err(mpsc::RecvTimeoutError::Timeout) => result_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("process scan did not resume after the process barrier was released"), + Err(mpsc::RecvTimeoutError::Disconnected) => None, + }; + let helper_result = helper.join(); + + assert!(started, "process scan worker did not reach process creation"); + assert!(helper_result.is_ok(), "process scan worker panicked"); + assert!(waited, "Ghostty process scan spawned while descriptor setup held the barrier"); + assert!(result.is_some(), "process scan did not return a process table"); + } + #[cfg(unix)] #[test] fn ghostty_config_helper_cleanup_reaps_killed_child() { From b6f0f72d283dbd229fcad1839dcbad3fa71c7b1e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:15:12 -0700 Subject: [PATCH 374/521] fix(tui): guard process scan spawn --- cmux-tui/crates/cmux-tui/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 4c50db2bfd97..3033b0c55135 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -3385,7 +3385,7 @@ fn ghostty_helper_process_table_snapshot() -> Option { .stdout(Stdio::piped()) .stderr(Stdio::null()) .process_group(0); - let Ok(mut child) = command.spawn() else { + let Ok(mut child) = cmux_tui_process::spawn(&mut command) else { return None; }; let Some(stdout) = child.stdout.take() else { From 13d4509e3cfea4a6219b69966ae1481ec6f94ca2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:28:31 -0700 Subject: [PATCH 375/521] fix(tui): retain daemon shutdown receipts --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 35 +++++++----- cmux-tui/crates/cmux-tui-core/src/server.rs | 5 +- cmux-tui/crates/cmux-tui-core/src/surface.rs | 58 ++++++++++++++++++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 10 ++-- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index dd384d327b8a..ed8a524041c8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -401,6 +401,14 @@ impl ShutdownOwnerLedger { Some(self.stage(shutdown_owner_key(surface.id, &owner), owner)) } + fn stage_daemon_owned_local_surface( + &self, + surface: &Arc, + ) -> Option<(ShutdownOwnerKey, Arc)> { + let owner = surface.daemon_shutdown_owner()?; + Some(self.stage(ShutdownOwnerKey::Surface(surface.id), owner)) + } + fn snapshot(&self) -> Vec<(ShutdownOwnerKey, Arc)> { self.owners .lock() @@ -9217,6 +9225,7 @@ impl Mux { .context("drain unpublished child reaper ownership for daemon exit")?; let surfaces = unique_surface_runtimes(&self.state.lock().unwrap()); for surface in surfaces { + let _ = self.shutdown_owners.stage_daemon_owned_local_surface(&surface); surface.shutdown_for_daemon(); } self.terminate_staged_shutdown_owners_until(deadline); @@ -9592,33 +9601,33 @@ impl Mux { let browser_runtime_failed = self.shutdown_browser_runtimes_until(deadline); let retained = self.shutdown_owners.snapshot(); - let mut failed_surface_ids = retained + let mut failed_surfaces = retained .iter() - .filter_map(|(key, _)| match key { - ShutdownOwnerKey::Surface(surface) => Some(*surface), + .filter_map(|(key, owner)| match key { + ShutdownOwnerKey::Surface(_) => { + Some(format!("key={key:?} phase={}", owner.retained_shutdown_phase())) + } ShutdownOwnerKey::Hosted { .. } => None, }) .collect::>(); - failed_surface_ids.sort_unstable(); + failed_surfaces.sort(); let mut failed_hosts = retained .iter() - .filter_map(|(key, _)| match key { - ShutdownOwnerKey::Hosted { terminal_id, .. } => Some(terminal_id.clone()), + .filter_map(|(key, owner)| match key { + ShutdownOwnerKey::Hosted { .. } => { + Some(format!("key={key:?} phase={}", owner.retained_shutdown_phase())) + } ShutdownOwnerKey::Surface(_) => None, }) .collect::>(); failed_hosts.sort(); let mut failures = Vec::new(); - if !failed_surface_ids.is_empty() { + if !failed_surfaces.is_empty() { failures.push(format!( "could not terminate {} surface process(es): {}", - failed_surface_ids.len(), - failed_surface_ids - .into_iter() - .map(|surface| surface.to_string()) - .collect::>() - .join(", ") + failed_surfaces.len(), + failed_surfaces.join(", ") )); } if browser_runtime_failed { diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 67f88fb93265..775760beec25 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -16546,8 +16546,11 @@ mod tests { let error = handle_command(&mux, client, Command::Shutdown, &writer).unwrap_err(); assert_eq!(error.to_string(), SERVER_SHUTDOWN_INCOMPLETE_ERROR); + let diagnostic = format!("{error:#}"); assert!( - format!("{error:#}").contains("could not terminate 1 surface process"), + diagnostic.contains("could not terminate 1 surface process") + && diagnostic.contains("key=Surface(") + && diagnostic.contains("phase=untracked-local-kill-unconfirmed"), "shutdown diagnostic discarded its retained owner: {error:#}" ); } diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index b7df2742f91b..e0e1166f6532 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -1374,6 +1374,14 @@ impl LocalProcess { Self::Untracked(killer) => killer.lock().unwrap().kill().is_ok(), } } + + fn retained_shutdown_phase(&self) -> &'static str { + match self { + Self::Owned(process) => process.retained_shutdown_phase(), + #[cfg(test)] + Self::Untracked(_) => "untracked-local-kill-unconfirmed", + } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum PtyLifetime { @@ -1542,6 +1550,15 @@ impl SurfaceShutdownOwner { } } + pub(crate) fn retained_shutdown_phase(&self) -> &'static str { + match &self.kind { + SurfaceShutdownOwnerKind::Local(process) => process.retained_shutdown_phase(), + #[cfg(unix)] + SurfaceShutdownOwnerKind::Hosted(_) => "host-termination-confirmation-unfinished", + SurfaceShutdownOwnerKind::Browser(_) => "browser-target-close-unconfirmed", + } + } + #[cfg(unix)] pub(crate) fn hosted( record: crate::terminal_host_runtime::TerminalHostRecord, @@ -1893,6 +1910,36 @@ impl LocalPtyProcess { *exited } + fn retained_shutdown_phase(&self) -> &'static str { + if *self.exited.0.lock().unwrap() { + return "local-exit-observed-owner-retained"; + } + #[cfg(unix)] + { + if self.child_wait_ownership_lost.load(Ordering::Acquire) { + return "local-child-wait-ownership-lost"; + } + let child_reaped = self.child_reaped.load(Ordering::Acquire); + let group_escalation_complete = self.group_escalation_complete.load(Ordering::Acquire); + if child_reaped && group_escalation_complete { + return "local-termination-complete-owner-retained"; + } + if child_reaped { + return "local-session-drain-unconfirmed"; + } + if group_escalation_complete { + return "local-child-reap-unconfirmed"; + } + if self.child_waitable.load(Ordering::Acquire) { + return "local-session-escalation-unconfirmed"; + } + if self.termination_started.load(Ordering::Acquire) { + return "local-child-exit-unconfirmed"; + } + } + "local-termination-not-active" + } + #[cfg(unix)] fn observe_child_wait_state_locked( &self, @@ -6005,6 +6052,17 @@ impl Surface { } } + pub(crate) fn daemon_shutdown_owner(&self) -> Option { + let Surface::Pty(pty) = self else { return None }; + if pty.lifetime != PtyLifetime::DaemonOwned { + return None; + } + pty.local_process + .as_deref() + .cloned() + .map(|process| SurfaceShutdownOwner { kind: SurfaceShutdownOwnerKind::Local(process) }) + } + #[cfg(test)] pub(crate) fn terminate_for_server_shutdown(&self, deadline: Instant) -> bool { self.shutdown_owner().is_none_or(|owner| owner.terminate_until(deadline)) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 2ed313d7ede2..b393fea659fa 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -2915,12 +2915,10 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { let remote_shutdown_started_path = dir.join("remote-shutdown-started"); let remote_shutdown_release_path = dir.join("remote-shutdown-release"); let remote_shutdown_complete_path = dir.join("remote-shutdown-complete"); - let server_exit_ready_path = dir.join("server-exit-ready"); let mut remote_started = TestFifoSignal::create(&remote_started_path); let mut remote_shutdown_started = TestFifoSignal::create(&remote_shutdown_started_path); let mut remote_shutdown_release = TestFifoSignal::create(&remote_shutdown_release_path); let mut remote_shutdown_complete = TestFifoSignal::create(&remote_shutdown_complete_path); - let mut server_exit_ready = TestFifoSignal::create(&server_exit_ready_path); let mut server = PtyChild::start_with_env( &[ "--socket", @@ -2944,7 +2942,6 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { "CMUX_TUI_TEST_REMOTE_SHUTDOWN_COMPLETE_SIGNAL", remote_shutdown_complete_path.as_os_str(), ), - ("CMUX_TUI_TEST_SERVER_EXIT_READY_SIGNAL", server_exit_ready_path.as_os_str()), ], ); wait_for_socket_path(&socket); @@ -2960,8 +2957,11 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { remote_shutdown_release.send(); remote_shutdown_complete.receive(Duration::from_secs(5)); - server_exit_ready.receive(Duration::from_secs(5)); - let status = server.child.wait().unwrap(); + let status = wait_for_child_exit( + native_pty_child(server.child.as_mut()), + Duration::from_secs(5), + "server remained alive after remote runtime cleanup completed", + ); assert!(status.success(), "server exited with {status}"); drop(server); fs::remove_dir_all(dir).unwrap(); From 694a8178f12cdace9caa5706a2349207fa6178df Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 05:41:02 -0700 Subject: [PATCH 376/521] fix(tui): retain local shutdown ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 18 ++++-- cmux-tui/crates/cmux-tui-core/src/surface.rs | 15 +++-- .../crates/cmux-tui/src/session/remote.rs | 8 ++- cmux-tui/crates/cmux-tui/tests/cli.rs | 61 ++++++++++++++++--- 4 files changed, 78 insertions(+), 24 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index ed8a524041c8..362ea5371658 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -401,11 +401,11 @@ impl ShutdownOwnerLedger { Some(self.stage(shutdown_owner_key(surface.id, &owner), owner)) } - fn stage_daemon_owned_local_surface( + fn stage_local_process_surface( &self, surface: &Arc, ) -> Option<(ShutdownOwnerKey, Arc)> { - let owner = surface.daemon_shutdown_owner()?; + let owner = surface.local_process_shutdown_owner()?; Some(self.stage(ShutdownOwnerKey::Surface(surface.id), owner)) } @@ -9225,7 +9225,7 @@ impl Mux { .context("drain unpublished child reaper ownership for daemon exit")?; let surfaces = unique_surface_runtimes(&self.state.lock().unwrap()); for surface in surfaces { - let _ = self.shutdown_owners.stage_daemon_owned_local_surface(&surface); + let _ = self.shutdown_owners.stage_local_process_surface(&surface); surface.shutdown_for_daemon(); } self.terminate_staged_shutdown_owners_until(deadline); @@ -9641,7 +9641,17 @@ impl Mux { )); } if !failures.is_empty() { - anyhow::bail!("{}", failures.join("; ")); + let diagnostic = failures.join("; "); + #[cfg(debug_assertions)] + if let Some(path) = std::env::var_os("CMUX_TUI_TEST_SHUTDOWN_OWNER_DIAGNOSTIC") { + std::fs::write(&path, diagnostic.as_bytes()).with_context(|| { + format!( + "write test shutdown owner diagnostic to {}", + Path::new(&path).display() + ) + })?; + } + anyhow::bail!("{diagnostic}"); } cleanup_lifecycle.complete(); diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index e0e1166f6532..1dbbe2700a67 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -6052,15 +6052,14 @@ impl Surface { } } - pub(crate) fn daemon_shutdown_owner(&self) -> Option { + /// A local PTY process cannot survive its daemon, regardless of the + /// resource lifetime. Hosted runtimes have no local process owner here. + pub(crate) fn local_process_shutdown_owner(&self) -> Option { let Surface::Pty(pty) = self else { return None }; - if pty.lifetime != PtyLifetime::DaemonOwned { - return None; - } - pty.local_process - .as_deref() - .cloned() - .map(|process| SurfaceShutdownOwner { kind: SurfaceShutdownOwnerKind::Local(process) }) + let Some(LocalProcess::Owned(process)) = pty.local_process.as_deref() else { return None }; + Some(SurfaceShutdownOwner { + kind: SurfaceShutdownOwnerKind::Local(LocalProcess::Owned(process.clone())), + }) } #[cfg(test)] diff --git a/cmux-tui/crates/cmux-tui/src/session/remote.rs b/cmux-tui/crates/cmux-tui/src/session/remote.rs index bcee6b46d773..0e24362f864a 100644 --- a/cmux-tui/crates/cmux-tui/src/session/remote.rs +++ b/cmux-tui/crates/cmux-tui/src/session/remote.rs @@ -2582,8 +2582,12 @@ impl RemoteSession { return Err(RemoteRequestError::Transport(error).into()); } - if self.shutdown.load(Ordering::Acquire) { - self.pending.lock().unwrap().remove(&id); + // A routed response has already removed this id and owns the receiver. + // Shutdown cancels only a request whose pending entry we can still take; + // otherwise consume the routed response or shutdown sentinel below. + if self.shutdown.load(Ordering::Acquire) + && self.pending.lock().unwrap().remove(&id).is_some() + { return Err(RemoteRequestError::Shutdown.into()); } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b393fea659fa..2f0c81fe7973 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -97,6 +97,7 @@ struct HeadlessServer { socket: PathBuf, state: PathBuf, dir: PathBuf, + shutdown_owner_diagnostic: Option, } impl HeadlessServer { @@ -105,11 +106,15 @@ impl HeadlessServer { } fn start_with(name: &str, ephemeral: bool, environment: &[(&str, &str)]) -> Self { - Self::start_with_options(name, ephemeral, environment, None) + Self::start_with_options(name, ephemeral, environment, None, false) + } + + fn start_with_shutdown_owner_diagnostic(name: &str, ephemeral: bool) -> Self { + Self::start_with_options(name, ephemeral, &[], None, true) } fn start_with_config(name: &str, config_contents: Option<&str>) -> Self { - Self::start_with_options(name, false, &[], config_contents) + Self::start_with_options(name, false, &[], config_contents, false) } fn start_with_options( @@ -117,6 +122,7 @@ impl HeadlessServer { ephemeral: bool, environment: &[(&str, &str)], config_contents: Option<&str>, + capture_shutdown_owner_diagnostic: bool, ) -> Self { let dir = unique_temp_dir(name); fs::create_dir_all(&dir).unwrap(); @@ -137,12 +143,26 @@ impl HeadlessServer { for (name, value) in environment { command.env(name, value); } + let shutdown_owner_diagnostic = + capture_shutdown_owner_diagnostic.then(|| dir.join("shutdown-owner-diagnostic.txt")); + if let Some(path) = &shutdown_owner_diagnostic { + command.env("CMUX_TUI_TEST_SHUTDOWN_OWNER_DIAGNOSTIC", path); + } let child = command.stdout(Stdio::null()).stderr(Stdio::piped()).spawn().unwrap(); - let server = Self { child, socket, state, dir }; + let server = Self { child, socket, state, dir, shutdown_owner_diagnostic }; server.wait_for_socket(); server } + fn shutdown_owner_diagnostic(&self) -> String { + let Some(path) = &self.shutdown_owner_diagnostic else { + return "shutdown owner diagnostic was not requested".to_string(); + }; + fs::read_to_string(path).unwrap_or_else(|error| { + format!("shutdown owner diagnostic was not published at {}: {error}", path.display()) + }) + } + fn wait_for_socket(&self) { let deadline = Instant::now() + Duration::from_secs(15); while Instant::now() < deadline { @@ -3183,7 +3203,10 @@ fn cancelled_published_host_is_terminated_through_its_record() { #[cfg(unix)] #[test] fn server_stop_kills_ephemeral_pty_process_groups() { - let mut server = HeadlessServer::start_with("server-stop-ephemeral-process-group", true, &[]); + let mut server = HeadlessServer::start_with_shutdown_owner_diagnostic( + "server-stop-ephemeral-process-group", + true, + ); let descendant_pid_file = server.dir.join("ephemeral-descendant.pid"); let command = format!( "trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", @@ -3202,8 +3225,9 @@ fn server_stop_kills_ephemeral_pty_process_groups() { let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); let stop = cli(&server, &["--json", "server", "stop"]); + let shutdown_diagnostic = server.shutdown_owner_diagnostic(); - assert_success(&stop); + assert_success_with_context(&stop, &shutdown_diagnostic); assert!(server.child.wait().unwrap().success()); assert!(!process_is_active(direct_pid)); assert!(!process_group_is_active(direct_pid)); @@ -3213,8 +3237,10 @@ fn server_stop_kills_ephemeral_pty_process_groups() { #[cfg(unix)] #[test] fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { - let mut server = - HeadlessServer::start_with("server-stop-ephemeral-job-control-group", true, &[]); + let mut server = HeadlessServer::start_with_shutdown_owner_diagnostic( + "server-stop-ephemeral-job-control-group", + true, + ); let descendant_pid_file = server.dir.join("ephemeral-background-job.pid"); let command = format!( concat!( @@ -3249,8 +3275,9 @@ fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { let descendant_session = unsafe { libc::getsid(descendant_pid_raw) }; let stop = cli(&server, &["--json", "server", "stop"]); + let shutdown_diagnostic = server.shutdown_owner_diagnostic(); - assert_success(&stop); + assert_success_with_context(&stop, &shutdown_diagnostic); assert!(server.child.wait().unwrap().success()); let descendant_remained = process_is_active(descendant_pid); if descendant_remained { @@ -3280,7 +3307,8 @@ fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { #[cfg(unix)] #[test] fn server_stop_drains_many_hosted_panes_before_acknowledging() { - let mut server = HeadlessServer::start("server-stop-many-panes"); + let mut server = + HeadlessServer::start_with_shutdown_owner_diagnostic("server-stop-many-panes", false); let applied = raw_json( &server, serde_json::json!({ @@ -3312,8 +3340,9 @@ fn server_stop_drains_many_hosted_panes_before_acknowledging() { let stop_started = Instant::now(); let stop = cli(&server, &["--json", "server", "stop"]); + let shutdown_diagnostic = server.shutdown_owner_diagnostic(); - assert_success(&stop); + assert_success_with_context(&stop, &shutdown_diagnostic); assert!(stop_started.elapsed() < Duration::from_secs(5)); assert!(server.child.wait().unwrap().success()); assert!(!host_pids.iter().copied().any(process_is_active)); @@ -4885,6 +4914,18 @@ fn assert_success(output: &Output) { ); } +#[track_caller] +fn assert_success_with_context(output: &Output, context: &str) { + assert!( + output.status.success(), + "expected success, got status {:?}\nstdout:\n{}\nstderr:\n{}\ncontext:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + context, + ); +} + fn find_session_database(state: &std::path::Path, session: &str) -> PathBuf { fs::read_dir(state) .unwrap() From 13aafefa3ac5f22c7c83874fe7e828ccf1164fc7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 06:16:34 -0700 Subject: [PATCH 377/521] fix(tui): complete shutdown diagnostic fixture --- cmux-tui/crates/cmux-tui/tests/cli.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 2f0c81fe7973..046da498c6cd 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -1110,7 +1110,7 @@ fn explicit_socket_keeps_state_in_platform_root() { .stderr(Stdio::piped()) .spawn() .unwrap(); - let server = HeadlessServer { child, socket, state, dir }; + let server = HeadlessServer { child, socket, state, dir, shutdown_owner_diagnostic: None }; server.wait_for_socket(); let registry_exists = || { From 6b800943690928a306865c9d1ff9e4439c6a500e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:26:58 -0700 Subject: [PATCH 378/521] test(tui): capture complete shutdown failure --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 362ea5371658..0f9d529238ad 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10,7 +10,6 @@ pub(crate) use resource_content::ResourceEffectProjection; use public_projections::{RestoredPublicProjections, restore_public_projections}; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; -use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, Weak}; @@ -9390,7 +9389,14 @@ impl Mux { /// closed so a retry cannot race a new PTY into the session. pub fn close_all_surfaces_for_shutdown(&self) -> anyhow::Result { let deadline = Instant::now() + crate::server::SERVER_SHUTDOWN_TIMEOUT; - self.close_all_surfaces_for_shutdown_until(deadline) + let result = self.close_all_surfaces_for_shutdown_until(deadline); + #[cfg(debug_assertions)] + if let Err(error) = &result + && let Some(path) = std::env::var_os("CMUX_TUI_TEST_SHUTDOWN_OWNER_DIAGNOSTIC") + { + let _ = std::fs::write(path, format!("{error:#}")); + } + result } fn close_all_surfaces_for_shutdown_until(&self, deadline: Instant) -> anyhow::Result { @@ -9642,15 +9648,6 @@ impl Mux { } if !failures.is_empty() { let diagnostic = failures.join("; "); - #[cfg(debug_assertions)] - if let Some(path) = std::env::var_os("CMUX_TUI_TEST_SHUTDOWN_OWNER_DIAGNOSTIC") { - std::fs::write(&path, diagnostic.as_bytes()).with_context(|| { - format!( - "write test shutdown owner diagnostic to {}", - Path::new(&path).display() - ) - })?; - } anyhow::bail!("{diagnostic}"); } From 72f75ea5a8162394fdc80eddb75992fe7ade45cc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:54:51 -0700 Subject: [PATCH 379/521] fix(tui): restore shared path import --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 0f9d529238ad..38e45b9d0d5f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -10,6 +10,7 @@ pub(crate) use resource_content::ResourceEffectProjection; use public_projections::{RestoredPublicProjections, restore_public_projections}; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; +use std::path::Path; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{Receiver, SyncSender}; use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, Weak}; From 40d0ffd38e4ac65353132ec562af3859cf17ec4a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:24:06 -0700 Subject: [PATCH 380/521] fix(tui): clear terminal owners on shutdown --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 30 +++++++++++++----------- cmux-tui/crates/cmux-tui/tests/cli.rs | 3 --- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 38e45b9d0d5f..5d7ba368897a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9428,7 +9428,7 @@ impl Mux { let terminal_host_records = { let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); let capacity = self.shutdown_owner_capacity(); - let (required, surface_owner_keys) = { + let (required, runtime_owner_keys) = { let state = self.state.lock().unwrap(); let required = state .surfaces @@ -9436,9 +9436,8 @@ impl Mux { .filter(|surface| !surface.is_dead()) .filter_map(|surface| surface.terminal_host_identity()) .collect::>(); - let surface_owner_keys = state - .surfaces - .values() + let runtime_owner_keys = unique_surface_runtimes(&state) + .into_iter() .filter_map(|surface| match surface.shutdown_owner_identity()? { SurfaceShutdownOwnerIdentity::Surface => { Some(ShutdownOwnerKey::Surface(surface.id)) @@ -9448,7 +9447,7 @@ impl Mux { } }) .collect::>(); - (required, surface_owner_keys) + (required, runtime_owner_keys) }; let mut owner_keys = self .shutdown_owners @@ -9456,7 +9455,7 @@ impl Mux { .into_iter() .map(|(key, _)| key) .collect::>(); - owner_keys.extend(surface_owner_keys); + owner_keys.extend(runtime_owner_keys); let records = match root { Some(root) => { let records = self @@ -9501,7 +9500,7 @@ impl Mux { records }; - let (surfaces, retained_count, tree_changed, closed_public_ids) = { + let (owners, closed_count, tree_changed, closed_public_ids) = { let mutation = WorkspaceMutation::local("cmux-tui-shutdown"); let operation = "server.stop"; let fingerprint = serde_json::json!({"operation":operation}); @@ -9530,12 +9529,15 @@ impl Mux { let result = (|| -> anyhow::Result<_> { let mut state = self.state.lock().unwrap(); let retained_count = self.shutdown_owners.len(); + let closed_count = state.surfaces.len() + retained_count; let tree_changed = !state.surfaces.is_empty() || !state.panes.is_empty() || state.workspaces.iter().any(|workspace| !workspace.screens.is_empty()); - let surfaces = state.surfaces.values().cloned().collect::>(); + let owners = unique_surface_runtimes(&state); let mut projected = state.clone(); projected.surfaces.clear(); + projected.terminal_catalog.clear(); + projected.terminal_catalog_by_runtime.clear(); for workspace in &mut projected.workspaces { workspace.screens.clear(); workspace.active_screen = 0; @@ -9545,6 +9547,7 @@ impl Mux { } projected.panes.clear(); projected.split_screens.clear(); + projected.rebuild_resource_indexes(); let projection = self.resource_effect_projection_locked( ®istry, &mut projected, @@ -9561,13 +9564,13 @@ impl Mux { None, )?; projected.resource_revision = close.resource.revision; - for surface in &surfaces { + for surface in &owners { let _ = self.shutdown_owners.stage_surface(surface); } *state = projected; - Ok((surfaces, retained_count, tree_changed, close.terminal_batch)) + Ok((owners, closed_count, tree_changed, close.terminal_batch)) })(); - let (surfaces, retained_count, tree_changed, terminal_batch) = match result { + let (owners, closed_count, tree_changed, terminal_batch) = match result { Ok(result) => result, Err(error) => { let _ = registry.mark_resource_effect_indeterminate(&mutation.id); @@ -9578,7 +9581,7 @@ impl Mux { if terminal_batch.closed != 0 { self.emit_terminal_registry_changed(®istry, terminal_batch.revision); } - (surfaces, retained_count, tree_changed, closed_public_ids) + (owners, closed_count, tree_changed, closed_public_ids) }; self.notify_terminal_exit_waiters(closed_public_ids); self.publish_resource_event(); @@ -9590,8 +9593,7 @@ impl Mux { self.emit(MuxEvent::TreeChanged); } - let closed_count = surfaces.len() + retained_count; - drop(surfaces); + drop(owners); #[cfg(unix)] for (record_path, record) in terminal_host_records { let owner = SurfaceShutdownOwner::hosted(record, record_path); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 046da498c6cd..0a8cb0b8ade4 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -493,9 +493,6 @@ fn wait_for_pid_file(path: &std::path::Path, timeout: Duration) -> u32 { fs::read_to_string(path).ok()?.trim().parse().ok() } - if let Some(pid) = read_pid(path) { - return pid; - } let deadline = Instant::now() + timeout; #[cfg(target_os = "linux")] From 7923beee84eb420db47430a46f5182526c074bf5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 10:10:59 -0700 Subject: [PATCH 381/521] fix(macOS): require teardown ownership admission --- ...faceRuntimePreparedTeardownExecution.swift | 17 +++++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 64 +++------------- ...lSurfaceRuntimeTeardownExecutionLane.swift | 18 ----- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 12 +++ ...rfaceRuntimeTeardownCoordinatorTests.swift | 76 ++++++++++++++++--- ...SurfaceTeardownCallbackLifetimeTests.swift | 6 +- 6 files changed, 111 insertions(+), 82 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimePreparedTeardownExecution.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimePreparedTeardownExecution.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimePreparedTeardownExecution.swift new file mode 100644 index 000000000000..647627182bb4 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimePreparedTeardownExecution.swift @@ -0,0 +1,17 @@ +/// A native-free Task that cannot run before its coordinator stores the handle. +nonisolated struct TerminalSurfaceRuntimePreparedTeardownExecution: Sendable { + let task: Task + private let startGate: TerminalSurfaceRuntimeTeardownStartGate + + init( + task: Task, + startGate: TerminalSurfaceRuntimeTeardownStartGate + ) { + self.task = task + self.startGate = startGate + } + + func start() { + startGate.start() + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index d2c88afe7940..f88110c526f4 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -185,45 +185,6 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { request.read() } - /// Queues a native-surface free from any isolation (the surface model's - /// `deinit` is nonisolated and cannot await). - /// - /// - Parameters: - /// - id: The owning surface id. - /// - workspaceId: The owning workspace id. - /// - reason: The teardown reason, for diagnostics. - /// - surface: The native surface pointer, already removed from all - /// main-thread owner state. - /// - callbackContext: The retained callback context released on the - /// main actor after the free completes. - /// - freeSurface: The free operation; defaults to - /// `ghostty_surface_free`. - /// - Returns: A ticket that completes after the native free and userdata - /// releases, or `nil` when bounded ownership admission is unavailable. - /// On `nil`, the caller retains the native pointer and callback userdata. - @discardableResult - public nonisolated func enqueueRuntimeTeardown( - id: UUID, - workspaceId: UUID, - reason: String, - surface: ghostty_surface_t, - callbackContext: Unmanaged?, - freeSurface: @escaping @Sendable (ghostty_surface_t) -> Void = { surface in - ghostty_surface_free(surface) - } - ) -> TerminalSurfaceRuntimeTeardownTicket? { - enqueueRuntimeTeardown( - id: id, - workspaceId: workspaceId, - reason: reason, - surface: surface, - callbackContext: callbackContext, - manualIOContext: nil, - byteTeeLease: nil, - freeSurface: freeSurface - ) - } - /// Queues a native-surface free that also transports the surface's other /// retained callback userdata. /// @@ -250,9 +211,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. /// - Returns: A ticket that completes after the native free and userdata - /// releases, or `nil` when bounded ownership admission is degraded or - /// exhausted. On `nil`, the caller retains the native pointer and its - /// callback userdata. + /// releases, or `nil` when the supplied ownership reservation is invalid + /// or the bounded submission ingress rejects the request. On `nil`, the + /// caller retains the native pointer and its callback userdata. @discardableResult nonisolated func enqueueRuntimeTeardown( id: UUID, @@ -263,7 +224,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { manualIOContext: Unmanaged?, byteTeeLease: (any TerminalByteTeeLease)?, runtimeOwnershipReservation: - TerminalSurfaceRuntimeOwnershipReservation? = nil, + TerminalSurfaceRuntimeOwnershipReservation, executionLane: TerminalSurfaceRuntimeTeardownExecutionLane = .boundedClose, isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? = nil, @@ -271,13 +232,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ghostty_surface_free(surface) } ) -> TerminalSurfaceRuntimeTeardownTicket? { - let admittedRuntimeOwnership: TerminalSurfaceRuntimeOwnershipReservation - if let existing = runtimeOwnershipReservation { - guard runtimeOwnershipAdmission.contains(existing) else { return nil } - admittedRuntimeOwnership = existing - } else { - guard let admitted = runtimeOwnershipAdmission.reserve() else { return nil } - admittedRuntimeOwnership = admitted + guard runtimeOwnershipAdmission.contains( + runtimeOwnershipReservation + ) else { + return nil } let completion = TerminalSurfaceRuntimeTeardownCompletion() let ticket = TerminalSurfaceRuntimeTeardownTicket(completion: completion) @@ -290,7 +248,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { callbackContext: callbackContext, manualIOContext: manualIOContext, byteTeeLease: byteTeeLease, - runtimeOwnershipReservation: admittedRuntimeOwnership, + runtimeOwnershipReservation: runtimeOwnershipReservation, freeSurface: freeSurface, completion: completion ) @@ -303,10 +261,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { case .enqueued: return ticket case .dropped, .terminated: - runtimeOwnershipAdmission.release(admittedRuntimeOwnership) + runtimeOwnershipAdmission.release(runtimeOwnershipReservation) return nil @unknown default: - runtimeOwnershipAdmission.release(admittedRuntimeOwnership) + runtimeOwnershipAdmission.release(runtimeOwnershipReservation) return nil } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift index 92f378d54ce3..ff82b7cbb82c 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownExecutionLane.swift @@ -40,21 +40,3 @@ nonisolated enum TerminalSurfaceRuntimeTeardownExecutionLane: Sendable { ) } } - -/// A native-free Task that cannot run before its coordinator stores the handle. -nonisolated struct TerminalSurfaceRuntimePreparedTeardownExecution: Sendable { - let task: Task - private let startGate: TerminalSurfaceRuntimeTeardownStartGate - - fileprivate init( - task: Task, - startGate: TerminalSurfaceRuntimeTeardownStartGate - ) { - self.task = task - self.startGate = startGate - } - - func start() { - startGate.start() - } -} diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index f07ca3618117..1423de920115 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -198,6 +198,9 @@ import CmuxTerminalCore UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) } defer { for retainedSurface in retainedSurfaces { retainedSurface.deallocate() } } + let runtimeReservations = try (0...makeStream() let releaseFrees = DispatchSemaphore(value: 0) defer { @@ -213,6 +216,9 @@ import CmuxTerminalCore reason: "test.creationRetry.\(index)", surface: retainedSurface, callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[index], freeSurface: { _ in freeStarted.continuation.yield() releaseFrees.wait() @@ -276,6 +282,9 @@ import CmuxTerminalCore retainedSurface.deallocate() } } + let runtimeReservations = try (0...makeStream() let releaseFrees = DispatchSemaphore(value: 0) defer { @@ -293,6 +302,9 @@ import CmuxTerminalCore reason: "test.pacedCreationRetry.\(index)", surface: retainedSurface, callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[index], freeSurface: { _ in freeStarted.continuation.yield() releaseFrees.wait() diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 3a99d8617907..7cf2dc71a7ff 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -90,6 +90,9 @@ private func requireTeardownTicket( let recorder = FreedSurfaceRecorder() let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) defer { surface.deallocate() } + let runtimeReservation = try #require( + coordinator.reserveRuntimeSurfaceOwnership() + ) let ticket = try requireTeardownTicket( coordinator.enqueueRuntimeTeardown( @@ -98,6 +101,9 @@ private func requireTeardownTicket( reason: "test", surface: surface, callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservation, freeSurface: { pointer in recorder.record(UInt(bitPattern: pointer)) } @@ -287,6 +293,9 @@ private func requireTeardownTicket( releaseFirstFree.signal() firstFreeStarted.continuation.finish() } + let firstReservation = try #require( + coordinator.reserveRuntimeSurfaceOwnership() + ) let firstTicket = try requireTeardownTicket( coordinator.enqueueRuntimeTeardown( @@ -295,6 +304,9 @@ private func requireTeardownTicket( reason: "test.stuckClose", surface: surfaces[0], callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: firstReservation, freeSurface: { _ in firstFreeStarted.continuation.yield() releaseFirstFree.wait() @@ -359,6 +371,9 @@ private func requireTeardownTicket( let isolatedReservation = try #require( await coordinator.reserveIsolatedHibernationTeardown() ) + let runtimeReservations = try (0...makeStream() let releaseIsolatedFree = DispatchSemaphore(value: 0) let secondIsolatedFreeCount = OSAllocatedUnfairLock(initialState: 0) @@ -927,6 +969,7 @@ private func requireTeardownTicket( callbackContext: nil, manualIOContext: nil, byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[0], executionLane: .isolatedHibernation, isolatedHibernationReservation: isolatedReservation, freeSurface: { _ in @@ -951,6 +994,7 @@ private func requireTeardownTicket( callbackContext: nil, manualIOContext: nil, byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[1], executionLane: .isolatedHibernation, isolatedHibernationReservation: secondReservation, freeSurface: { _ in @@ -965,6 +1009,9 @@ private func requireTeardownTicket( reason: "test.boundedClose", surface: closeSurface, callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[2], freeSurface: { _ in closeFreeCount.withLock { $0 += 1 } } @@ -1003,6 +1050,9 @@ private func requireTeardownTicket( releaseIsolatedFree.signal() isolatedFreeStarted.continuation.finish() } + let runtimeReservations = try (0.. Date: Mon, 10 Aug 2026 10:11:07 -0700 Subject: [PATCH 382/521] fix(tui): close durable terminals atomically --- .../src/mux/resource_topology.rs | 165 ++++++++++++++---- .../src/resource_router/content.rs | 25 ++- cmux-tui/crates/cmux-tui/tests/cli.rs | 47 ++++- .../cmux-tui/tests/terminal_host_recovery.rs | 33 +++- 4 files changed, 204 insertions(+), 66 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index b349e9e2fb95..25d48fbd8658 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2143,12 +2143,21 @@ impl Mux { .workspace .context("topology close target has no workspace")?; self.commit_resource_close_with( - operation, Some(workspace), idempotency_key, operation_name, fingerprint, - move |state| self.effect_slots_in_state(state, &path), + move |registry, state, notifications| { + let slots = self.effect_slots_in_state(state, &path)?; + let plan = self.resource_close_plan_locked( + operation, + slots, + registry, + state, + notifications, + )?; + Ok((slots, plan)) + }, ) } @@ -2169,12 +2178,11 @@ impl Mux { self.surface_workspace(slot).context("content close target has no workspace")?; let content_id = content_id.clone(); let committed = self.commit_resource_close_with( - ResourceOperation::TabClose, Some(workspace), idempotency_key, operation_name, fingerprint, - move |state| { + move |registry, state, notifications| { anyhow::ensure!( state.resource_indexes.content_ids.get(&slot) == Some(&content_id), "content identity changed before close" @@ -2182,12 +2190,20 @@ impl Mux { let pane = state.pane_of(slot).context("content has no pane")?; let (workspace_index, screen_index) = state.screen_of(pane).context("content pane has no screen")?; - Ok(EffectSlots { + let slots = EffectSlots { workspace: Some(state.workspaces[workspace_index].id), screen: Some(state.workspaces[workspace_index].screens[screen_index].id), pane: Some(pane), tab: Some(slot), - }) + }; + let plan = self.resource_close_plan_locked( + ResourceOperation::TabClose, + slots, + registry, + state, + notifications, + )?; + Ok((slots, plan)) }, )?; drop(_creation_fence); @@ -2195,9 +2211,42 @@ impl Mux { self.finish_resource_close(committed) } + /// A terminal remains a session-owned resource after its last view and + /// runtime detach. Resolve an exact detached terminal through the durable + /// public-to-host relationship while preserving normal selector errors for + /// scoped, current, and name selectors. + pub(crate) fn resolve_resource_terminal_close_id( + &self, + selectors: &ResourceSelectors, + ) -> anyhow::Result { + let missing = match self.resolve_resource_path(ResourceTarget::Terminal, selectors) { + Ok(path) => { + return path.terminal.context("resolved terminal close omitted its public id"); + } + Err(error) if error.code == "selector.not_found" => error, + Err(error) => return Err(anyhow::Error::new(error)), + }; + if selectors.workspace.is_some() + || selectors.screen.is_some() + || selectors.pane.is_some() + || selectors.tab.is_some() + { + return Err(anyhow::Error::new(missing)); + } + let raw = selectors.terminal.as_deref().context("terminal close omitted its selector")?; + let public_id = match Selector::parse(raw).map_err(anyhow::Error::new)? { + Selector::Id(id) => TerminalPublicId::parse(id).map_err(anyhow::Error::new)?, + Selector::Current | Selector::Name(_) => return Err(anyhow::Error::new(missing)), + }; + if self.workspace_registry.lock().unwrap().live_terminal_host_id(&public_id)?.is_none() { + return Err(anyhow::Error::new(missing)); + } + Ok(public_id) + } + pub(crate) fn commit_resource_terminal_close_effect( &self, - surface: SurfaceId, + terminal_id: &TerminalPublicId, idempotency_key: &str, operation_name: &str, fingerprint: &Value, @@ -2205,17 +2254,24 @@ impl Mux { let _creation_handoff = self.resource_creation_handoff.lock().unwrap(); let _creation_fence = self.resource_creation_execution.lock().unwrap(); let committed = self.commit_resource_close_with( - ResourceOperation::TerminalClose, None, idempotency_key, operation_name, fingerprint, - move |state| { - anyhow::ensure!( - state.terminal_runtime_by_id(surface).is_some(), - "terminal disappeared" - ); - Ok(EffectSlots { workspace: None, screen: None, pane: None, tab: Some(surface) }) + move |registry, state, _notifications| { + let host_id = registry + .live_terminal_host_id(terminal_id)? + .with_context(|| format!("terminal {terminal_id} is not live"))?; + let terminal = registry + .terminal_record(&host_id)? + .with_context(|| format!("terminal {terminal_id} has no durable host"))?; + let plan = self.resource_terminal_close_plan_locked( + &host_id, + terminal.incarnation.as_deref(), + terminal_id, + state, + )?; + Ok((EffectSlots { workspace: None, screen: None, pane: None, tab: None }, plan)) }, )?; drop(_creation_fence); @@ -2413,15 +2469,17 @@ impl Mux { Ok(true) } - #[allow(clippy::too_many_arguments)] fn commit_resource_close_with( &self, - operation: ResourceOperation, workspace: Option, idempotency_key: &str, operation_name: &str, fingerprint: &Value, - resolve_slots: impl FnOnce(&State) -> anyhow::Result, + resolve_plan: impl FnOnce( + &WorkspaceRegistry, + &State, + &HashMap, + ) -> anyhow::Result<(EffectSlots, ResourceClosePlan)>, ) -> anyhow::Result { let terminal_host_root = self.surface_options.lock().unwrap().terminal_host_root.clone(); let lifecycle = workspace.map(|workspace| self.workspace_lifecycle(workspace)); @@ -2429,15 +2487,13 @@ impl Mux { let notifications = self.surface_notifications(); let mut registry = self.workspace_registry.lock().unwrap(); let mut state = self.state.lock().unwrap(); - let slots = resolve_slots(&state)?; + let (slots, mut plan) = resolve_plan(®istry, &state, ¬ifications)?; if let Some(workspace) = workspace { anyhow::ensure!( slots.workspace == Some(workspace), "topology close target changed workspaces before commit" ); } - let mut plan = - self.resource_close_plan_locked(operation, slots, ®istry, &state, ¬ifications)?; let closed_workspace_key = plan.workspace_close.as_ref().map(|close| close.workspace_key.clone()); let workspace_host_identities = if let Some(workspace_key) = closed_workspace_key.as_deref() @@ -2587,17 +2643,7 @@ impl Mux { state: &State, notifications: &HashMap, ) -> anyhow::Result { - let selection_before = active_tree_selection(state); - let mut projected = state.clone(); - let ResourceCloseInputs { - surface_ids, - mut delta, - changed_screens, - workspace_metadata, - terminal_runtime, - terminal_batch, - terminal_public_id, - } = match operation { + let inputs = match operation { ResourceOperation::WorkspaceClose => { let workspace = slots.workspace.context("workspace disappeared")?; let index = state.workspace_index(workspace).context("workspace disappeared")?; @@ -2701,16 +2747,65 @@ impl Mux { _ => anyhow::bail!("operation is not a topology close"), }; + self.resource_close_plan_from_inputs_locked(inputs, state) + } + + fn resource_terminal_close_plan_locked( + &self, + terminal_id: &str, + terminal_incarnation: Option<&str>, + public_id: &TerminalPublicId, + state: &State, + ) -> anyhow::Result { + let placements = + state.placements_of_content(&ContentPublicId::Terminal(public_id.clone())).to_vec(); + let changed_screens = unique_screen_ids( + placements.iter().filter_map(|surface| surface_screen_id(state, *surface)), + ); + self.resource_close_plan_from_inputs_locked( + ResourceCloseInputs { + surface_ids: placements, + changed_screens, + terminal_runtime: state.terminal_catalog.get(public_id).cloned(), + terminal_batch: vec![( + terminal_id.to_string(), + terminal_incarnation.map(str::to_owned), + )], + terminal_public_id: Some(public_id.clone()), + ..Default::default() + }, + state, + ) + } + + fn resource_close_plan_from_inputs_locked( + &self, + inputs: ResourceCloseInputs, + state: &State, + ) -> anyhow::Result { + let selection_before = active_tree_selection(state); + let mut projected = state.clone(); + let ResourceCloseInputs { + surface_ids, + mut delta, + changed_screens, + workspace_metadata, + terminal_runtime, + terminal_batch, + terminal_public_id, + } = inputs; + let mut removed = Vec::new(); let mut split_index_changed = false; if let Some(public_id) = &terminal_public_id { let (removed_runtime, terminal_views, changed) = remove_terminal_content_from_state(self, &mut projected, public_id); anyhow::ensure!( - removed_runtime - .as_ref() - .zip(terminal_runtime.as_ref()) - .is_some_and(|(removed, planned)| removed.shares_terminal_runtime(planned)), + match (removed_runtime.as_ref(), terminal_runtime.as_ref()) { + (Some(removed), Some(planned)) => removed.shares_terminal_runtime(planned), + (None, None) => true, + _ => false, + }, "terminal close lost its catalog runtime" ); removed = terminal_views; diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs index 793b405cc48d..6c63f20c3941 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_router/content.rs @@ -327,9 +327,8 @@ fn terminal_effect(mux: &Arc, request: ParsedResourceRequest) -> Result"))? + mux.resolve_resource_terminal_close_id(&request.selectors) + .map_err(resource_operation_error)? } else { resolve_terminal_surface(mux, &request.selectors)?.0 }; @@ -358,6 +357,15 @@ fn execute_terminal_effect( Ok(fields) => fields.clone(), Err(error) => return effects::commit_known_failure(mux, prepared, error), }; + if prepared.operation == "terminal.close" { + let commit = mux.commit_resource_terminal_close_effect( + &terminal_id, + &prepared.idempotency_key, + &prepared.operation, + &prepared.fingerprint, + ); + return finish_projection_commit(mux, prepared, commit); + } let Some(surface_id) = mux.resource_surface_for_terminal(&terminal_id) else { return effects::commit_known_failure( mux, @@ -389,7 +397,6 @@ fn execute_terminal_effect( surface.clear_history().map_err(|error| ActionFailure::Indeterminate(error.to_string())) } "terminal.viewport.scroll" => terminal_scroll_viewport(mux, &surface, &fields), - "terminal.close" => Ok(()), operation => Err(ActionFailure::Known(ResourceError::operation_failed( operation, "stored terminal effect operation is invalid", @@ -400,16 +407,6 @@ fn execute_terminal_effect( return finish_action_failure(mux, prepared, failure); } - if prepared.operation == "terminal.close" { - let commit = mux.commit_resource_terminal_close_effect( - surface_id, - &prepared.idempotency_key, - &prepared.operation, - &prepared.fingerprint, - ); - return finish_projection_commit(mux, prepared, commit); - } - debug_assert!(effects::receipt_only_operation(&prepared.operation)); effects::commit_success_without_changes(mux, prepared, json!({})) } diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 0a8cb0b8ade4..79187d3e1205 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -605,6 +605,7 @@ struct LegacyServerProcess { socket: PathBuf, dir: PathBuf, descendant_pid_file: PathBuf, + descendant_pid_published: Option, } #[cfg(unix)] @@ -623,6 +624,9 @@ impl LegacyServerProcess { fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let descendant_pid_file = dir.join("descendant.pid"); + let descendant_pid_published_path = dir.join("descendant-pid-published"); + let descendant_pid_published = (scenario == "reparent-on-close") + .then(|| TestFifoSignal::create(&descendant_pid_published_path)); let ready_path = dir.join("ready"); let mut ready = TestFifoSignal::create(&ready_path); let mut command = Command::new(std::env::current_exe().unwrap()); @@ -634,6 +638,12 @@ impl LegacyServerProcess { .env("CMUX_TUI_TEST_LEGACY_READY_SIGNAL", &ready_path) .stdout(Stdio::null()) .stderr(Stdio::piped()); + if descendant_pid_published.is_some() { + command.env( + "CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_PUBLISHED_SIGNAL", + &descendant_pid_published_path, + ); + } if let Some(reported_pid) = reported_pid { command.env("CMUX_TUI_TEST_LEGACY_REPORTED_PID", reported_pid.to_string()); } @@ -642,12 +652,20 @@ impl LegacyServerProcess { } let child = command.spawn().unwrap(); ready.receive(Duration::from_secs(15)); - Self { child, socket, dir, descendant_pid_file } + Self { child, socket, dir, descendant_pid_file, descendant_pid_published } } fn descendant_pid(&self) -> Option { fs::read_to_string(&self.descendant_pid_file).ok()?.trim().parse().ok() } + + fn receive_descendant_pid(&mut self, timeout: Duration) -> u32 { + self.descendant_pid_published + .as_mut() + .expect("legacy descendant PID receipt was not configured") + .receive(timeout); + self.descendant_pid().expect("legacy descendant PID receipt preceded its complete file") + } } #[cfg(unix)] @@ -770,7 +788,8 @@ fn legacy_server_process_helper() { concat!( "set -m; trap '' HUP; ", "(trap '' HUP; while :; do sleep 60; done) & ", - "printf '%s' \"$!\" > \"$CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE\"; wait" + "printf '%s' \"$!\" > \"$CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_FILE\"; ", + "printf '1' > \"$CMUX_TUI_TEST_LEGACY_DESCENDANT_PID_PUBLISHED_SIGNAL\"; wait" ), ]) .stdout(Stdio::null()) @@ -3205,9 +3224,12 @@ fn server_stop_kills_ephemeral_pty_process_groups() { true, ); let descendant_pid_file = server.dir.join("ephemeral-descendant.pid"); + let descendant_pid_published_path = server.dir.join("ephemeral-descendant-pid-published"); + let mut descendant_pid_published = TestFifoSignal::create(&descendant_pid_published_path); let command = format!( - "trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", - descendant_pid_file.display() + "trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; printf 1 > {}; wait", + descendant_pid_file.display(), + descendant_pid_published_path.display() ); let surface = raw_json(&server, serde_json::json!({"id":"ephemeral-run","cmd":"run","command":command})) @@ -3219,7 +3241,9 @@ fn server_stop_kills_ephemeral_pty_process_groups() { serde_json::json!({"id":"ephemeral-process","cmd":"process-info","surface":surface}), ); let direct_pid = u32::try_from(info["pid"].as_u64().unwrap()).unwrap(); - let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + descendant_pid_published.receive(Duration::from_secs(5)); + let descendant_pid = + fs::read_to_string(&descendant_pid_file).unwrap().trim().parse::().unwrap(); let stop = cli(&server, &["--json", "server", "stop"]); let shutdown_diagnostic = server.shutdown_owner_diagnostic(); @@ -3239,14 +3263,17 @@ fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { true, ); let descendant_pid_file = server.dir.join("ephemeral-background-job.pid"); + let descendant_pid_published_path = server.dir.join("ephemeral-background-job-pid-published"); + let mut descendant_pid_published = TestFifoSignal::create(&descendant_pid_published_path); let command = format!( concat!( "exec /bin/bash --noprofile --norc -i -c '", "trap \"\" HUP; ", "(trap \"\" HUP; while :; do sleep 60; done) & ", - "printf \"%s\" \"$!\" > {}; wait'", + "printf \"%s\" \"$!\" > {}; printf \"1\" > {}; wait'", ), - descendant_pid_file.display() + descendant_pid_file.display(), + descendant_pid_published_path.display() ); let surface = raw_json( &server, @@ -3259,7 +3286,9 @@ fn server_stop_kills_background_jobs_in_separate_pty_process_groups() { serde_json::json!({"id":"job-control-process","cmd":"process-info","surface":surface}), ); let direct_pid = u32::try_from(info["pid"].as_u64().unwrap()).unwrap(); - let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + descendant_pid_published.receive(Duration::from_secs(5)); + let descendant_pid = + fs::read_to_string(&descendant_pid_file).unwrap().trim().parse::().unwrap(); let direct_pid_raw = libc::pid_t::try_from(direct_pid).unwrap(); let descendant_pid_raw = libc::pid_t::try_from(descendant_pid).unwrap(); // SAFETY: both fixture processes are live and owned by this test. @@ -3500,7 +3529,7 @@ fn published_host_launch_errors_reap_the_complete_pty_session() { fn legacy_stop_retains_pty_ownership_captured_before_surface_close() { let mut server = LegacyServerProcess::start("legacy-server-reparent-on-close", "reparent-on-close", None); - let descendant_pid = wait_for_pid_file(&server.descendant_pid_file, Duration::from_secs(5)); + let descendant_pid = server.receive_descendant_pid(Duration::from_secs(5)); let output = Command::new(bin()) .args(["server", "stop", "--socket"]) diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 1247eca8c31a..911653918cc7 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -2044,20 +2044,37 @@ fn client_reserved_short_lived_create_replays_its_durable_exit_without_topology( let first = request(&harness.socket, create.clone()); assert_eq!(first["terminal_id"], terminal_id); assert_eq!(first["replayed"], false); - assert_eq!(first["already_exited"], true); - assert_eq!(first["lifecycle"], "exited"); - assert_eq!(first["exit"]["outcome"], serde_json::json!({"kind":"exit","code":17})); - assert_eq!(first["surface"], serde_json::Value::Null); - assert_eq!(first["pane"], serde_json::Value::Null); - assert_eq!(first["screen"], serde_json::Value::Null); - assert_eq!(first["workspace"], serde_json::Value::Null); + let already_exited = first["already_exited"].as_bool().unwrap(); + assert_eq!(first["lifecycle"], if already_exited { "exited" } else { "running" }); + for field in ["surface", "pane", "screen", "workspace"] { + assert_eq!(first[field].is_null(), already_exited, "unexpected {field}: {first}"); + } + if already_exited { + assert_eq!(first["exit"]["outcome"], serde_json::json!({"kind":"exit","code":17})); + } else { + assert!(first["exit"].is_null()); + let waited = resource_request( + &harness.socket, + "reserved-short-lived-create-wait", + "terminal.wait_exit", + serde_json::json!({ + "machine":"current", + "session":"current", + "terminal":terminal_id, + "timeout_ms":"5000", + }), + None, + ); + assert_eq!(waited["state"], "exited", "terminal did not exit: {waited}"); + assert_eq!(waited["outcome"], serde_json::json!({"kind":"exit","code":17})); + } let retry = request(&harness.socket, create); assert_eq!(retry["replayed"], true); assert_eq!(retry["terminal_id"], terminal_id); assert_eq!(retry["already_exited"], true); assert_eq!(retry["lifecycle"], "exited"); - assert_eq!(retry["exit"], first["exit"]); + assert_eq!(retry["exit"]["outcome"], serde_json::json!({"kind":"exit","code":17})); assert_eq!(retry["surface"], serde_json::Value::Null); assert_eq!(retry["pane"], serde_json::Value::Null); assert_eq!(retry["screen"], serde_json::Value::Null); From 843f5612f57083d25bab210e228e8edaf4253f76 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:09:23 -0700 Subject: [PATCH 383/521] test(macOS): preserve teardown ingress for owners --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 7cf2dc71a7ff..e4ba8b526d06 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -360,10 +360,11 @@ private func requireTeardownTicket( } @MainActor - @Test func boundedIngressDropsNewestSubmissionWithoutLeakingOwnershipOrWaiter() async throws { + @Test func ownershipReservationsKeepIngressCapacityForEveryNativeFree() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 ) + let recorder = FreedSurfaceRecorder() let surfaces = (0..<2).map { _ in UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) } @@ -386,37 +387,43 @@ private func requireTeardownTicket( runtimeOwnershipReservation: runtimeReservations[0], executionLane: .isolatedHibernation, isolatedHibernationReservation: isolatedReservation, - freeSurface: { _ in } + freeSurface: { pointer in + recorder.record(UInt(bitPattern: pointer)) + } ) ) - // While this test owns MainActor, the consumer either has the admitted - // request buffered or waits for MainActor to validate its reservation. - // Two owner-free messages therefore fill the two-element buffer in - // either schedule. + // While this test owns MainActor, the consumer waits for MainActor to + // validate the isolated reservation. Control submissions must not use + // the ingress capacity already reserved for the second owned surface. coordinator.cancelAllRuntimeTeardowns() coordinator.cancelAllRuntimeTeardowns() - let droppedTicket = coordinator.enqueueRuntimeTeardown( - id: UUID(), - workspaceId: UUID(), - reason: "test.dropNewestIngressSubmission", - surface: surfaces[1], - callbackContext: nil, - manualIOContext: nil, - byteTeeLease: nil, - runtimeOwnershipReservation: runtimeReservations[1], - freeSurface: { _ in } + let secondTicket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.preserveOwnedIngressSubmission", + surface: surfaces[1], + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservations[1], + freeSurface: { pointer in + recorder.record(UInt(bitPattern: pointer)) + } + ) ) - #expect(droppedTicket == nil) - #expect(coordinator.debugRuntimeSurfaceOwnerCount == 1) + #expect(coordinator.debugRuntimeSurfaceOwnerCount == 2) + #expect(await admittedTicket.wait(timeout: nil)) + #expect(await secondTicket.wait(timeout: nil)) + await recorder.waitForFreeCount(2) + #expect(recorder.freed.count == 2) #expect( - await coordinator.cancelRuntimeTeardown(ticketID: UUID()) == false + Set(recorder.freed) + == Set(surfaces.map { UInt(bitPattern: $0) }) ) - #expect(coordinator.debugRuntimeSurfaceOwnerCount == 1) - - #expect(await admittedTicket.wait(timeout: nil)) #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) } From 44ddaf684d578c96f3d9d3f7db745de0eab46849 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:11:31 -0700 Subject: [PATCH 384/521] fix(macOS): reserve teardown ingress with owners --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 100 ++++++++++++++++-- ...urfaceRuntimeOwnershipAdmissionState.swift | 12 +++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 73 ++++++++++--- 3 files changed, 158 insertions(+), 27 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 05c9006a56be..a24e9eaf8891 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -8,6 +8,8 @@ typealias TerminalSurfaceRuntimeOwnershipRecovery = /// /// Creation, public teardown, and deinit cannot suspend while transferring a /// native pointer, so this ledger uses one short unfair-lock critical section. +/// Each ownership reservation also holds one submission-ingress slot until the +/// teardown submission is dequeued or the unused ownership is released. /// Native work never runs under the lock. A fully stalled close pool degrades /// admission until a worker returns, preventing repeated create/close cycles /// from growing an unbounded retained teardown backlog. @@ -29,12 +31,11 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { state.withLock { state in guard !state.closeTeardownDegraded, - state.reservationIDs.count < maximumOwnerCount else { + state.reservationIDs.count < maximumOwnerCount, + state.ingressReservationIDs.count < maximumOwnerCount else { return nil } - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) - return reservation + return reserveOwnershipAndIngress(in: &state) } } @@ -44,12 +45,11 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { state.withLock { state in if !state.closeTeardownDegraded, - state.reservationIDs.count < maximumOwnerCount + state.reservationIDs.count < maximumOwnerCount, + state.ingressReservationIDs.count < maximumOwnerCount { _ = removeRecoveryAction(recoveryID, from: &state) - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) - return .reserved(reservation) + return .reserved(reserveOwnershipAndIngress(in: &state)) } if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery @@ -71,9 +71,79 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.withLock { $0.reservationIDs.contains(reservation.id) } } + func claimIngressReservation( + for reservation: TerminalSurfaceRuntimeOwnershipReservation + ) -> TerminalSurfaceRuntimeTeardownIngressReservation? { + state.withLock { state in + guard state.reservationIDs.contains(reservation.id), + state.unclaimedOwnershipIngressReservationIDs.remove( + reservation.id + ) != nil else { + return nil + } + return TerminalSurfaceRuntimeTeardownIngressReservation( + id: reservation.id + ) + } + } + + func reserveControlIngress() + -> TerminalSurfaceRuntimeTeardownIngressReservation? { + let result: ( + TerminalSurfaceRuntimeTeardownIngressReservation?, + TerminalSurfaceRuntimeOwnershipRecoveryGrant? + ) = state.withLock { state in + if let recoveryGrant = takeNextRecoveryGrant(from: &state) { + return (nil, recoveryGrant) + } + guard state.ingressReservationIDs.count < maximumOwnerCount else { + return (nil, nil) + } + let reservation = TerminalSurfaceRuntimeTeardownIngressReservation() + state.ingressReservationIDs.insert(reservation.id) + return (reservation, nil) + } + schedule(result.1) + return result.0 + } + + func releaseIngress( + _ reservation: TerminalSurfaceRuntimeTeardownIngressReservation + ) { + let recoveryGrant = state.withLock { state in + _ = state.ingressReservationIDs.remove(reservation.id) + _ = state.unclaimedOwnershipIngressReservationIDs.remove( + reservation.id + ) + return takeNextRecoveryGrant(from: &state) + } + schedule(recoveryGrant) + } + + func releaseFailedSubmission( + ownership reservation: TerminalSurfaceRuntimeOwnershipReservation, + ingress ingressReservation: + TerminalSurfaceRuntimeTeardownIngressReservation + ) { + let recoveryGrant = state.withLock { state in + _ = state.reservationIDs.remove(reservation.id) + _ = state.ingressReservationIDs.remove(ingressReservation.id) + _ = state.unclaimedOwnershipIngressReservationIDs.remove( + reservation.id + ) + return takeNextRecoveryGrant(from: &state) + } + schedule(recoveryGrant) + } + func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { let recoveryGrant = state.withLock { state in _ = state.reservationIDs.remove(reservation.id) + if state.unclaimedOwnershipIngressReservationIDs.remove( + reservation.id + ) != nil { + _ = state.ingressReservationIDs.remove(reservation.id) + } return takeNextRecoveryGrant(from: &state) } schedule(recoveryGrant) @@ -135,12 +205,23 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.recoveryTailID = recoveryID } + private func reserveOwnershipAndIngress( + in state: inout TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> TerminalSurfaceRuntimeOwnershipReservation { + let reservation = TerminalSurfaceRuntimeOwnershipReservation() + state.reservationIDs.insert(reservation.id) + state.ingressReservationIDs.insert(reservation.id) + state.unclaimedOwnershipIngressReservationIDs.insert(reservation.id) + return reservation + } + private func takeNextRecoveryGrant( from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) -> TerminalSurfaceRuntimeOwnershipRecoveryGrant? { guard !state.closeTeardownDegraded, !state.recoveryGrantIsScheduled, state.reservationIDs.count < maximumOwnerCount, + state.ingressReservationIDs.count < maximumOwnerCount, let recoveryID = state.recoveryHeadID, let recoveryAction = removeRecoveryAction( recoveryID, @@ -148,8 +229,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) else { return nil } - let reservation = TerminalSurfaceRuntimeOwnershipReservation() - state.reservationIDs.insert(reservation.id) + let reservation = reserveOwnershipAndIngress(in: &state) state.recoveryGrantIsScheduled = true return TerminalSurfaceRuntimeOwnershipRecoveryGrant( action: recoveryAction, diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index e717f9a63f3d..c187f55cd5fc 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -1,7 +1,19 @@ import Foundation +struct TerminalSurfaceRuntimeTeardownIngressReservation: Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} + struct TerminalSurfaceRuntimeOwnershipAdmissionState { var reservationIDs: Set = [] + /// Ingress slots held by live owners or by ordered control submissions. + var ingressReservationIDs: Set = [] + /// Owner slots that have not transferred to an enqueue submission. + var unclaimedOwnershipIngressReservationIDs: Set = [] var closeTeardownDegraded = false var recoveryEntriesByID: [UUID: TerminalSurfaceRuntimeOwnershipRecoveryEntry] = [:] diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index f88110c526f4..dd4ec89aa4b8 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -9,13 +9,17 @@ private enum TerminalSurfaceRuntimeTeardownSubmission: Sendable { case enqueue( request: TerminalSurfaceRuntimeTeardownRequest, executionLane: TerminalSurfaceRuntimeTeardownExecutionLane, - isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation? + isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation?, + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation ) case cancel( ticketID: UUID, - result: AsyncStream.Continuation + result: AsyncStream.Continuation, + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation + ) + case cancelAll( + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation ) - case cancelAll } private struct TerminalSurfaceRuntimeActiveTeardown: Sendable { @@ -211,9 +215,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// - freeSurface: The free operation; defaults to /// `ghostty_surface_free`. /// - Returns: A ticket that completes after the native free and userdata - /// releases, or `nil` when the supplied ownership reservation is invalid - /// or the bounded submission ingress rejects the request. On `nil`, the - /// caller retains the native pointer and its callback userdata. + /// releases, or `nil` when the supplied ownership reservation is invalid, + /// was already transferred, or the submission stream has terminated. On + /// `nil`, the caller retains the native pointer and its callback userdata. @discardableResult nonisolated func enqueueRuntimeTeardown( id: UUID, @@ -232,9 +236,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ghostty_surface_free(surface) } ) -> TerminalSurfaceRuntimeTeardownTicket? { - guard runtimeOwnershipAdmission.contains( - runtimeOwnershipReservation - ) else { + guard let ingressReservation = + runtimeOwnershipAdmission.claimIngressReservation( + for: runtimeOwnershipReservation + ) else { return nil } let completion = TerminalSurfaceRuntimeTeardownCompletion() @@ -255,16 +260,23 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { let submission = TerminalSurfaceRuntimeTeardownSubmission.enqueue( request: request, executionLane: executionLane, - isolatedHibernationReservation: isolatedHibernationReservation + isolatedHibernationReservation: isolatedHibernationReservation, + ingressReservation: ingressReservation ) switch submissionContinuation.yield(submission) { case .enqueued: return ticket case .dropped, .terminated: - runtimeOwnershipAdmission.release(runtimeOwnershipReservation) + runtimeOwnershipAdmission.releaseFailedSubmission( + ownership: runtimeOwnershipReservation, + ingress: ingressReservation + ) return nil @unknown default: - runtimeOwnershipAdmission.release(runtimeOwnershipReservation) + runtimeOwnershipAdmission.releaseFailedSubmission( + ownership: runtimeOwnershipReservation, + ingress: ingressReservation + ) return nil } } @@ -276,26 +288,49 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// releases its reservation through the normal completion path. @discardableResult nonisolated func cancelRuntimeTeardown(ticketID: UUID) async -> Bool { + guard let ingressReservation = + runtimeOwnershipAdmission.reserveControlIngress() else { + return false + } let result = AsyncStream.makeStream( bufferingPolicy: .bufferingNewest(1) ) switch submissionContinuation.yield( - .cancel(ticketID: ticketID, result: result.continuation) + .cancel( + ticketID: ticketID, + result: result.continuation, + ingressReservation: ingressReservation + ) ) { case .enqueued: var iterator = result.stream.makeAsyncIterator() return await iterator.next() ?? false case .dropped, .terminated: + runtimeOwnershipAdmission.releaseIngress(ingressReservation) result.continuation.finish() return false @unknown default: + runtimeOwnershipAdmission.releaseIngress(ingressReservation) result.continuation.finish() return false } } nonisolated func cancelAllRuntimeTeardowns() { - _ = submissionContinuation.yield(.cancelAll) + guard let ingressReservation = + runtimeOwnershipAdmission.reserveControlIngress() else { + return + } + switch submissionContinuation.yield( + .cancelAll(ingressReservation: ingressReservation) + ) { + case .enqueued: + break + case .dropped, .terminated: + runtimeOwnershipAdmission.releaseIngress(ingressReservation) + @unknown default: + runtimeOwnershipAdmission.releaseIngress(ingressReservation) + } } private func receive( @@ -305,18 +340,22 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { case .enqueue( let request, let executionLane, - let isolatedHibernationReservation + let isolatedHibernationReservation, + let ingressReservation ): + runtimeOwnershipAdmission.releaseIngress(ingressReservation) await enqueue( request, executionLane: executionLane, isolatedHibernationReservation: isolatedHibernationReservation ) - case .cancel(let ticketID, let result): + case .cancel(let ticketID, let result, let ingressReservation): + runtimeOwnershipAdmission.releaseIngress(ingressReservation) result.yield(cancelStoredTeardown(ticketID: ticketID)) result.finish() - case .cancelAll: + case .cancelAll(let ingressReservation): + runtimeOwnershipAdmission.releaseIngress(ingressReservation) cancelAllStoredTeardowns() } } From c60464972f869cc040fab118fb770ad11f3a06f5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:13:45 -0700 Subject: [PATCH 385/521] fix(tui): repair merged runtime structure --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 1 - cmux-tui/crates/cmux-tui-core/src/mux.rs | 12091 ++++++++-------- .../src/resource_router/effects.rs | 2 +- .../src/terminal_host_runtime.rs | 3 +- 4 files changed, 5956 insertions(+), 6141 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index fc60ddc8d4de..714b36adfc54 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -1,5 +1,4 @@ use std::collections::{HashMap, HashSet, VecDeque}; -use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError, sync_channel}; use std::sync::{Arc, Condvar, Mutex, Weak}; diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 60c80b8f8bf7..337ec9c87be2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -2093,8 +2093,7 @@ struct ShutdownCoordinatorGuard<'a> { impl Drop for ShutdownCoordinatorGuard<'_> { fn drop(&mut self) { - let mut state = - self.coordinator.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let mut state = self.coordinator.state.lock().unwrap_or_else(PoisonError::into_inner); state.poisoned |= std::thread::panicking(); state.held = false; self.coordinator.available.notify_one(); @@ -2285,7 +2284,7 @@ impl BrowserRuntimeSlot { } fn take_on_drop(&mut self) -> Option> { - let state = self.state.get_mut().unwrap_or_else(std::sync::PoisonError::into_inner); + let state = self.state.get_mut().unwrap_or_else(PoisonError::into_inner); match std::mem::take(state) { BrowserRuntimeSlotState::Ready(runtime) => Some(runtime), BrowserRuntimeSlotState::Stopping(runtime) => runtime, @@ -3493,3475 +3492,3450 @@ impl Mux { ); } } - Ok(()) } + Ok(()) + } - #[cfg(unix)] - fn restored_terminal_binding( - &self, - terminal_id: &str, - ) -> anyhow::Result> { - let registry = self.workspace_registry.lock().unwrap(); - let Some(public_id) = registry.terminal_resource_id(terminal_id)? else { - return Ok(None); - }; - drop(registry); - let state = self.state.lock().unwrap(); - let content_id = ContentPublicId::Terminal(public_id.clone()); - let placements = state - .placements_of_content(&content_id) - .iter() - .map(|slot| { - let tab_id = - state.resource_indexes.tab_ids.get(slot).cloned().with_context(|| { - format!("restored terminal placement {slot} has no tab identity") - })?; - Ok((*slot, TabResourceIdentity::new(tab_id, content_id.clone()))) - }) - .collect::>>()?; - Ok(Some(RestoredTerminalBinding { public_id, placements })) - } + #[cfg(unix)] + fn restored_terminal_binding( + &self, + terminal_id: &str, + ) -> anyhow::Result> { + let registry = self.workspace_registry.lock().unwrap(); + let Some(public_id) = registry.terminal_resource_id(terminal_id)? else { + return Ok(None); + }; + drop(registry); + let state = self.state.lock().unwrap(); + let content_id = ContentPublicId::Terminal(public_id.clone()); + let placements = state + .placements_of_content(&content_id) + .iter() + .map(|slot| { + let tab_id = + state.resource_indexes.tab_ids.get(slot).cloned().with_context(|| { + format!("restored terminal placement {slot} has no tab identity") + })?; + Ok((*slot, TabResourceIdentity::new(tab_id, content_id.clone()))) + }) + .collect::>>()?; + Ok(Some(RestoredTerminalBinding { public_id, placements })) + } - #[cfg(unix)] - fn adopt_restored_terminal( - self: &Arc, - binding: Option, - options: &SurfaceOptions, - record: &crate::terminal_host_runtime::TerminalHostRecord, - record_path: &Path, - ) -> anyhow::Result> { - let id = binding - .as_ref() - .and_then(|binding| binding.placements.first().map(|(slot, _)| *slot)) - .unwrap_or_else(|| self.next_id()); - match binding { - Some(binding) if !binding.placements.is_empty() => { - let (_, identity) = binding.placements.first().expect("checked placement"); - Surface::adopt_hosted_with_resource_identity( - id, - options.clone(), - Arc::downgrade(self), - record.clone(), - record_path.to_path_buf(), - identity.clone(), - ) - } - Some(binding) => Surface::adopt_hosted_with_terminal_public_id( - id, - options.clone(), - Arc::downgrade(self), - record.clone(), - record_path.to_path_buf(), - binding.public_id, - ), - None => Surface::adopt_hosted( + #[cfg(unix)] + fn adopt_restored_terminal( + self: &Arc, + binding: Option, + options: &SurfaceOptions, + record: &crate::terminal_host_runtime::TerminalHostRecord, + record_path: &Path, + ) -> anyhow::Result> { + let id = binding + .as_ref() + .and_then(|binding| binding.placements.first().map(|(slot, _)| *slot)) + .unwrap_or_else(|| self.next_id()); + match binding { + Some(binding) if !binding.placements.is_empty() => { + let (_, identity) = binding.placements.first().expect("checked placement"); + Surface::adopt_hosted_with_resource_identity( id, options.clone(), Arc::downgrade(self), record.clone(), record_path.to_path_buf(), - ), + identity.clone(), + ) } + Some(binding) => Surface::adopt_hosted_with_terminal_public_id( + id, + options.clone(), + Arc::downgrade(self), + record.clone(), + record_path.to_path_buf(), + binding.public_id, + ), + None => Surface::adopt_hosted( + id, + options.clone(), + Arc::downgrade(self), + record.clone(), + record_path.to_path_buf(), + ), } + } - #[cfg(unix)] - fn adopt_terminal_hosts(self: &Arc) -> anyhow::Result<()> { - let options = self.surface_options.lock().unwrap().clone(); - let exit_records = match options.terminal_host_root.as_deref() { - Some(root) => crate::terminal_host_runtime::load_terminal_host_exit_records(root)?, - None => Vec::new(), - }; - let records = match options.terminal_host_root.as_deref() { - Some(root) => crate::terminal_host_runtime::load_terminal_host_records(root)?, - None => Vec::new(), + #[cfg(unix)] + fn adopt_terminal_hosts(self: &Arc) -> anyhow::Result<()> { + let options = self.surface_options.lock().unwrap().clone(); + let exit_records = match options.terminal_host_root.as_deref() { + Some(root) => crate::terminal_host_runtime::load_terminal_host_exit_records(root)?, + None => Vec::new(), + }; + let records = match options.terminal_host_root.as_deref() { + Some(root) => crate::terminal_host_runtime::load_terminal_host_records(root)?, + None => Vec::new(), + }; + let mut handled_terminals = HashSet::new(); + // Sidecars are host-owned write-ahead completion records. Reconcile + // them before live discovery records so a daemon crash after host + // completion cannot collapse the exact status into "host missing". + for (exit_path, record) in exit_records { + let Some(terminal) = + self.workspace_registry.lock().unwrap().terminal_record(&record.terminal_id)? + else { + continue; }; - let mut handled_terminals = HashSet::new(); - // Sidecars are host-owned write-ahead completion records. Reconcile - // them before live discovery records so a daemon crash after host - // completion cannot collapse the exact status into "host missing". - for (exit_path, record) in exit_records { - let Some(terminal) = - self.workspace_registry.lock().unwrap().terminal_record(&record.terminal_id)? - else { - continue; - }; - if terminal.lifecycle == TerminalLifecycle::Tombstoned { - let _ = crate::terminal_host_runtime::acknowledge_terminal_host_exit_record( - &exit_path, &record, - )?; - handled_terminals.insert(record.terminal_id); - continue; - } - if terminal - .incarnation - .as_deref() - .is_some_and(|incarnation| incarnation != record.incarnation) - { - // Retain evidence from the non-current incarnation. It must - // never be allowed to terminate a replacement process. - continue; - } - self.persist_terminal_exit( - &record.terminal_id, - Some(&record.incarnation), - &record.exit, - )?; - self.detach_exited_terminal_topology(&record.terminal_id)?; + if terminal.lifecycle == TerminalLifecycle::Tombstoned { let _ = crate::terminal_host_runtime::acknowledge_terminal_host_exit_record( &exit_path, &record, )?; handled_terminals.insert(record.terminal_id); + continue; + } + if terminal + .incarnation + .as_deref() + .is_some_and(|incarnation| incarnation != record.incarnation) + { + // Retain evidence from the non-current incarnation. It must + // never be allowed to terminate a replacement process. + continue; + } + self.persist_terminal_exit( + &record.terminal_id, + Some(&record.incarnation), + &record.exit, + )?; + self.detach_exited_terminal_topology(&record.terminal_id)?; + let _ = crate::terminal_host_runtime::acknowledge_terminal_host_exit_record( + &exit_path, &record, + )?; + handled_terminals.insert(record.terminal_id); + } + for (record_path, record) in records { + let terminal_id = record.terminal_id.clone(); + if handled_terminals.contains(&terminal_id) { + if !cleanup_terminal_host_record(&record, &record_path) { + self.schedule_terminal_adoption(options.clone(), record, record_path); + } + continue; } - for (record_path, record) in records { - let terminal_id = record.terminal_id.clone(); - if handled_terminals.contains(&terminal_id) { + let mut terminal = + self.workspace_registry.lock().unwrap().terminal_record(&terminal_id)?; + if terminal.is_none() { + // One-release migration path for hosts launched before SQLite + // became placement authority. Never trust the JSON hint when + // its workspace no longer exists. + let can_import = !record.workspace_key.is_empty() + && self.state.lock().unwrap().workspace_by_key(&record.workspace_key).is_some(); + if can_import { + let imported = RegistryTerminal { + terminal_id: terminal_id.clone(), + workspace_key: record.workspace_key.clone(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({"legacy_import":true}), + exit: None, + }; + let mut registry = self.workspace_registry.lock().unwrap(); + let revision = commit_terminal_transition( + &mut registry, + "terminal-imported", + "import-legacy-terminal", + &imported, + )?; + self.emit_terminal_registry_changed(®istry, revision); + terminal = Some(imported); + } else { if !cleanup_terminal_host_record(&record, &record_path) { self.schedule_terminal_adoption(options.clone(), record, record_path); } continue; } - let mut terminal = - self.workspace_registry.lock().unwrap().terminal_record(&terminal_id)?; - if terminal.is_none() { - // One-release migration path for hosts launched before SQLite - // became placement authority. Never trust the JSON hint when - // its workspace no longer exists. - let can_import = !record.workspace_key.is_empty() - && self - .state - .lock() - .unwrap() - .workspace_by_key(&record.workspace_key) - .is_some(); - if can_import { - let imported = RegistryTerminal { - terminal_id: terminal_id.clone(), - workspace_key: record.workspace_key.clone(), - incarnation: None, - lifecycle: TerminalLifecycle::Launching, - launch_spec: serde_json::json!({"legacy_import":true}), - exit: None, - }; - let mut registry = self.workspace_registry.lock().unwrap(); - let revision = commit_terminal_transition( - &mut registry, - "terminal-imported", - "import-legacy-terminal", - &imported, - )?; - self.emit_terminal_registry_changed(®istry, revision); - terminal = Some(imported); - } else { - if !cleanup_terminal_host_record(&record, &record_path) { - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; - } + } + let terminal = terminal.expect("terminal imported or loaded"); + if terminal.lifecycle == TerminalLifecycle::Tombstoned { + if !cleanup_terminal_host_record(&record, &record_path) { + self.schedule_terminal_adoption(options.clone(), record, record_path); } - let terminal = terminal.expect("terminal imported or loaded"); - if terminal.lifecycle == TerminalLifecycle::Tombstoned { - if !cleanup_terminal_host_record(&record, &record_path) { - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; + continue; + } + if terminal.lifecycle == TerminalLifecycle::Exited { + self.detach_exited_terminal_topology(&terminal.terminal_id)?; + handled_terminals.insert(terminal_id.clone()); + if !cleanup_terminal_host_record(&record, &record_path) { + self.schedule_terminal_adoption(options.clone(), record, record_path); } - if terminal.lifecycle == TerminalLifecycle::Exited { - self.detach_exited_terminal_topology(&terminal.terminal_id)?; - handled_terminals.insert(terminal_id.clone()); - if !cleanup_terminal_host_record(&record, &record_path) { - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; + continue; + } + if terminal.incarnation.as_deref().is_some_and(|value| value != record.incarnation) { + self.mark_terminal_exited_and_detach( + &terminal_id, + "terminal-incarnation-mismatch", + "host-incarnation-mismatch", + &options, + )?; + handled_terminals.insert(terminal_id.clone()); + if !cleanup_terminal_host_record(&record, &record_path) { + self.schedule_terminal_adoption(options.clone(), record, record_path); } - if terminal.incarnation.as_deref().is_some_and(|value| value != record.incarnation) - { - self.mark_terminal_exited_and_detach( - &terminal_id, - "terminal-incarnation-mismatch", - "host-incarnation-mismatch", - &options, - )?; + continue; + } + if let Err(error) = self.transition_terminal_lifecycle( + "terminal-adopting", + "adopt-terminal", + &terminal_id, + TerminalLifecycle::Adopting, + Some(&record.incarnation), + None, + ) { + let current = + self.workspace_registry.lock().unwrap().terminal_record(&terminal_id)?; + if current.as_ref().is_some_and(|terminal| { + matches!( + terminal.lifecycle, + TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned + ) + }) { + if current + .as_ref() + .is_some_and(|terminal| terminal.lifecycle == TerminalLifecycle::Exited) + { + self.detach_exited_terminal_topology(&terminal_id)?; + } handled_terminals.insert(terminal_id.clone()); if !cleanup_terminal_host_record(&record, &record_path) { self.schedule_terminal_adoption(options.clone(), record, record_path); } continue; } - if let Err(error) = self.transition_terminal_lifecycle( - "terminal-adopting", - "adopt-terminal", - &terminal_id, - TerminalLifecycle::Adopting, - Some(&record.incarnation), - None, - ) { - let current = - self.workspace_registry.lock().unwrap().terminal_record(&terminal_id)?; - if current.as_ref().is_some_and(|terminal| { - matches!( - terminal.lifecycle, - TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned - ) - }) { - if current - .as_ref() - .is_some_and(|terminal| terminal.lifecycle == TerminalLifecycle::Exited) - { - self.detach_exited_terminal_topology(&terminal_id)?; - } - handled_terminals.insert(terminal_id.clone()); - if !cleanup_terminal_host_record(&record, &record_path) { - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - continue; - } - return Err(error); - } - if terminal_host_record_liveness(&record_path, &record) - == TerminalHostLiveness::Dead - { - let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( - &record_path, - &record, - ); - self.mark_terminal_exited_and_detach( - &terminal_id, - "terminal-host-proven-dead", - "host-process-ended-before-adoption", - &options, - )?; - handled_terminals.insert(terminal_id.clone()); - continue; - } - // Host handshakes can consume their full timeout. Queue every - // live or indeterminate record before the control socket is - // published so startup time is independent of record count. - handled_terminals.insert(terminal_id); - self.schedule_terminal_adoption(options.clone(), record, record_path); - } - - // No launcher survives a daemon restart. A durable lifecycle row - // without a host-owned record therefore represents a closed crash - // window, not permission to spawn a replacement shell. - let snapshot = self.workspace_registry.lock().unwrap().terminal_snapshot()?; - for terminal in snapshot.terminals { - if terminal.lifecycle == TerminalLifecycle::Tombstoned - || handled_terminals.contains(&terminal.terminal_id) - { - continue; - } - if terminal.lifecycle == TerminalLifecycle::Exited { - self.detach_exited_terminal_topology(&terminal.terminal_id)?; - continue; - } - if let Some(root) = options.terminal_host_root.as_deref() { - // The directory scan deliberately tolerates malformed debris. - // Before converting a durable live row into record absence, - // exclude its launcher through the same transferable guard - // held until the child publishes. The exact canonical probe - // then either observes that publication, rejects unreadable - // ownership, or proves no child can publish after absence. - let _publication = - crate::terminal_host_runtime::acquire_terminal_host_publication( - root, - &terminal.terminal_id, - )?; - let exact_record = crate::terminal_host_runtime::load_terminal_host_record( - root, - &terminal.terminal_id, - ) - .with_context(|| { - format!( - "load exact terminal-host record for durable terminal {}", - terminal.terminal_id - ) - })?; - anyhow::ensure!( - exact_record.is_none(), - "terminal-host record appeared after adoption discovery for {}", - terminal.terminal_id - ); - } + return Err(error); + } + if terminal_host_record_liveness(&record_path, &record) == TerminalHostLiveness::Dead { + let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( + &record_path, + &record, + ); self.mark_terminal_exited_and_detach( - &terminal.terminal_id, - "terminal-record-missing", - "missing-host-record", + &terminal_id, + "terminal-host-proven-dead", + "host-process-ended-before-adoption", &options, )?; + handled_terminals.insert(terminal_id.clone()); + continue; } - Ok(()) + // Host handshakes can consume their full timeout. Queue every + // live or indeterminate record before the control socket is + // published so startup time is independent of record count. + handled_terminals.insert(terminal_id); + self.schedule_terminal_adoption(options.clone(), record, record_path); } - #[cfg(unix)] - fn mark_terminal_exited_and_detach( - self: &Arc, - terminal_id: &str, - _operation: &str, - reason: &str, - options: &SurfaceOptions, - ) -> anyhow::Result<()> { - let terminal = self.workspace_registry.lock().unwrap().terminal_record(terminal_id)?; - let Some(terminal) = terminal else { return Ok(()) }; - if terminal.lifecycle == TerminalLifecycle::Tombstoned { - return Ok(()); - } - let sidecar = options - .terminal_host_root - .as_ref() - .map(|root| root.join(format!("{terminal_id}.json"))) - .map(|record_path| { - crate::terminal_host_runtime::terminal_host_exit_record(&record_path) - }) - .transpose()? - .flatten() - .filter(|(_, record)| { - record.terminal_id == terminal_id - && terminal - .incarnation - .as_deref() - .is_none_or(|incarnation| incarnation == record.incarnation) - }); - if terminal.lifecycle != TerminalLifecycle::Exited { - let observed = sidecar - .as_ref() - .map(|(_, record)| record.exit.clone()) - .unwrap_or_else(|| TerminalExit::unknown(reason)); - let incarnation = sidecar - .as_ref() - .map(|(_, record)| record.incarnation.as_str()) - .or(terminal.incarnation.as_deref()); - self.persist_terminal_exit(terminal_id, incarnation, &observed)?; + // No launcher survives a daemon restart. A durable lifecycle row + // without a host-owned record therefore represents a closed crash + // window, not permission to spawn a replacement shell. + let snapshot = self.workspace_registry.lock().unwrap().terminal_snapshot()?; + for terminal in snapshot.terminals { + if terminal.lifecycle == TerminalLifecycle::Tombstoned + || handled_terminals.contains(&terminal.terminal_id) + { + continue; } - if let Some((path, record)) = sidecar { - let _ = crate::terminal_host_runtime::acknowledge_terminal_host_exit_record( - &path, &record, - )?; + if terminal.lifecycle == TerminalLifecycle::Exited { + self.detach_exited_terminal_topology(&terminal.terminal_id)?; + continue; } - self.detach_exited_terminal_topology(terminal_id)?; - Ok(()) - } - - #[cfg(unix)] - fn finish_terminal_adoption( - &self, - terminal_id: &str, - incarnation: &str, - surface: Arc, - owner_reservation: &mut SurfaceOwnerReservation<'_>, - ) -> anyhow::Result<()> { - let _creation = self.begin_surface_creation()?; - if surface.is_dead() { - self.persist_terminal_exit( - terminal_id, - Some(incarnation), - &TerminalExit::unknown("host-exited-during-adoption"), + if let Some(root) = options.terminal_host_root.as_deref() { + // The directory scan deliberately tolerates malformed debris. + // Before converting a durable live row into record absence, + // exclude its launcher through the same transferable guard + // held until the child publishes. The exact canonical probe + // then either observes that publication, rejects unreadable + // ownership, or proves no child can publish after absence. + let _publication = crate::terminal_host_runtime::acquire_terminal_host_publication( + root, + &terminal.terminal_id, )?; - anyhow::bail!("terminal host exited during adoption"); - } - let mut registry = self.workspace_registry.lock().unwrap(); - let mut state = self.state.lock().unwrap(); - let terminal = registry - .terminal_record(terminal_id)? - .ok_or_else(|| anyhow::anyhow!("terminal disappeared during adoption"))?; - anyhow::ensure!( - terminal.lifecycle == TerminalLifecycle::Adopting, - "terminal is no longer awaiting adoption" - ); - anyhow::ensure!( - terminal.incarnation.as_deref() == Some(incarnation), - "terminal incarnation changed during adoption" - ); - anyhow::ensure!( - surface.terminal_host_identity().is_some_and(|identity| { - identity.terminal_id == terminal_id && identity.incarnation == incarnation - }), - "adopted host identity does not match durable terminal" - ); - - let before = state.clone(); - let restored_public_id = surface.terminal_public_id().cloned(); - let has_restored_placements = restored_public_id.as_ref().is_some_and(|public_id| { - !state - .placements_of_content(&ContentPublicId::Terminal(public_id.clone())) - .is_empty() - }); - if has_restored_placements || surface.resource_identity().is_none() { - anyhow::ensure!( - !self.consume_terminal_adoption_insert_failure(), - "injected terminal adoption topology failure" - ); - insert_restored_terminal_runtime_checked(self, &mut state, surface)?; - } else { - // One-release import path for a host that predates public content - // identities. Give it a real initial placement so the normal - // resource projection can persist its generated identities. - let workspace_index = state - .workspaces - .iter() - .position(|workspace| workspace.key == terminal.workspace_key) - .ok_or_else(|| { - anyhow::anyhow!("terminal workspace disappeared during adoption") - })?; - let (pane_id, pane) = self.make_pane(surface.id)?; - let screen_id = self.next_id(); - let screen_public_id = ScreenPublicId::random()?; + let exact_record = crate::terminal_host_runtime::load_terminal_host_record( + root, + &terminal.terminal_id, + ) + .with_context(|| { + format!( + "load exact terminal-host record for durable terminal {}", + terminal.terminal_id + ) + })?; anyhow::ensure!( - !self.consume_terminal_adoption_insert_failure(), - "injected terminal adoption topology failure" + exact_record.is_none(), + "terminal-host record appeared after adoption discovery for {}", + terminal.terminal_id ); - insert_surface_with_active_reservation_checked(self, &mut state, surface)?; - { - let workspace = &mut state.workspaces[workspace_index]; - workspace.screens.push(Screen { - id: screen_id, - public_id: screen_public_id, - name: None, - root: Node::Leaf(pane_id), - active_pane: pane_id, - zoomed_pane: None, - zellij_auto_layout: Some(vec![pane_id]), - viewport_splits: Default::default(), - viewport_base_width: None, - layout_columns: Vec::new(), - layout_revision: 0, - layout_undo: Default::default(), - }); - workspace.active_screen = workspace.screens.len() - 1; - } - // Adoption materializes a live pane without stealing focus, but it - // must still advance the pane-set revision used by frontend focus - // history pruning. - state.insert_pane(pane); - state.rebuild_resource_indexes(); } + self.mark_terminal_exited_and_detach( + &terminal.terminal_id, + "terminal-record-missing", + "missing-host-record", + &options, + )?; + } + Ok(()) + } - let revision = match commit_terminal_lifecycle( - &mut registry, - "terminal-ready", - "terminal-adopted", - terminal_id, - TerminalLifecycle::Running, - Some(incarnation), - None, - ) { - Ok((_, revision)) => revision, - Err(error) => { - *state = before; - return Err(error); - } - }; - owner_reservation.release(); - drop(state); - self.emit_terminal_registry_changed(®istry, revision); - #[cfg(debug_assertions)] - if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL") { - let _ = std::fs::write(signal, b"1"); - } - Ok(()) + #[cfg(unix)] + fn mark_terminal_exited_and_detach( + self: &Arc, + terminal_id: &str, + _operation: &str, + reason: &str, + options: &SurfaceOptions, + ) -> anyhow::Result<()> { + let terminal = self.workspace_registry.lock().unwrap().terminal_record(terminal_id)?; + let Some(terminal) = terminal else { return Ok(()) }; + if terminal.lifecycle == TerminalLifecycle::Tombstoned { + return Ok(()); + } + let sidecar = options + .terminal_host_root + .as_ref() + .map(|root| root.join(format!("{terminal_id}.json"))) + .map(|record_path| { + crate::terminal_host_runtime::terminal_host_exit_record(&record_path) + }) + .transpose()? + .flatten() + .filter(|(_, record)| { + record.terminal_id == terminal_id + && terminal + .incarnation + .as_deref() + .is_none_or(|incarnation| incarnation == record.incarnation) + }); + if terminal.lifecycle != TerminalLifecycle::Exited { + let observed = sidecar + .as_ref() + .map(|(_, record)| record.exit.clone()) + .unwrap_or_else(|| TerminalExit::unknown(reason)); + let incarnation = sidecar + .as_ref() + .map(|(_, record)| record.incarnation.as_str()) + .or(terminal.incarnation.as_deref()); + self.persist_terminal_exit(terminal_id, incarnation, &observed)?; + } + if let Some((path, record)) = sidecar { + let _ = crate::terminal_host_runtime::acknowledge_terminal_host_exit_record( + &path, &record, + )?; } + self.detach_exited_terminal_topology(terminal_id)?; + Ok(()) + } - #[cfg(unix)] - fn ensure_terminal_adoption_workers(self: &Arc) -> std::io::Result<()> { - let coordinator = self.terminal_adoption_coordinator.clone(); - let (existing, missing) = { - let mut state = coordinator.state.lock().unwrap(); - if state.stopping { - return Err(std::io::Error::new( - std::io::ErrorKind::Interrupted, - "terminal adoption is stopping", - )); - } - if state.workers_running >= TERMINAL_ADOPTION_WORKERS { - return Ok(()); - } - let existing = state.workers_running; - let missing = TERMINAL_ADOPTION_WORKERS - existing; - state.workers_running += missing; - (existing, missing) - }; - let mut started = 0; - let mut first_error = None; - for worker in 0..missing { - let worker = existing + worker; - let mux = Arc::downgrade(self); - let worker_coordinator = coordinator.clone(); - match std::thread::Builder::new().name(format!("terminal-adoption-{worker}")).spawn( - move || { - Self::run_terminal_adoption_worker(mux, worker_coordinator); - }, - ) { - Ok(_) => started += 1, - Err(error) => { - let mut state = coordinator.state.lock().unwrap(); - state.workers_running = state.workers_running.saturating_sub(1); - coordinator.wake.notify_all(); - first_error.get_or_insert(error); - } - } - } - if existing + started > 0 { - Ok(()) - } else { - Err(first_error.expect("at least one worker spawn failed")) - } + #[cfg(unix)] + fn finish_terminal_adoption( + &self, + terminal_id: &str, + incarnation: &str, + surface: Arc, + owner_reservation: &mut SurfaceOwnerReservation<'_>, + ) -> anyhow::Result<()> { + let _creation = self.begin_surface_creation()?; + if surface.is_dead() { + self.persist_terminal_exit( + terminal_id, + Some(incarnation), + &TerminalExit::unknown("host-exited-during-adoption"), + )?; + anyhow::bail!("terminal host exited during adoption"); } + let mut registry = self.workspace_registry.lock().unwrap(); + let mut state = self.state.lock().unwrap(); + let terminal = registry + .terminal_record(terminal_id)? + .ok_or_else(|| anyhow::anyhow!("terminal disappeared during adoption"))?; + anyhow::ensure!( + terminal.lifecycle == TerminalLifecycle::Adopting, + "terminal is no longer awaiting adoption" + ); + anyhow::ensure!( + terminal.incarnation.as_deref() == Some(incarnation), + "terminal incarnation changed during adoption" + ); + anyhow::ensure!( + surface.terminal_host_identity().is_some_and(|identity| { + identity.terminal_id == terminal_id && identity.incarnation == incarnation + }), + "adopted host identity does not match durable terminal" + ); - #[cfg(unix)] - fn consume_terminal_adoption_insert_failure(&self) -> bool { - self.terminal_adoption_insert_failures - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() + let before = state.clone(); + let restored_public_id = surface.terminal_public_id().cloned(); + let has_restored_placements = restored_public_id.as_ref().is_some_and(|public_id| { + !state.placements_of_content(&ContentPublicId::Terminal(public_id.clone())).is_empty() + }); + if has_restored_placements || surface.resource_identity().is_none() { + anyhow::ensure!( + !self.consume_terminal_adoption_insert_failure(), + "injected terminal adoption topology failure" + ); + insert_restored_terminal_runtime_checked(self, &mut state, surface)?; + } else { + // One-release import path for a host that predates public content + // identities. Give it a real initial placement so the normal + // resource projection can persist its generated identities. + let workspace_index = state + .workspaces + .iter() + .position(|workspace| workspace.key == terminal.workspace_key) + .ok_or_else(|| anyhow::anyhow!("terminal workspace disappeared during adoption"))?; + let (pane_id, pane) = self.make_pane(surface.id)?; + let screen_id = self.next_id(); + let screen_public_id = ScreenPublicId::random()?; + anyhow::ensure!( + !self.consume_terminal_adoption_insert_failure(), + "injected terminal adoption topology failure" + ); + insert_surface_with_active_reservation_checked(self, &mut state, surface)?; + { + let workspace = &mut state.workspaces[workspace_index]; + workspace.screens.push(Screen { + id: screen_id, + public_id: screen_public_id, + name: None, + root: Node::Leaf(pane_id), + active_pane: pane_id, + zoomed_pane: None, + zellij_auto_layout: Some(vec![pane_id]), + viewport_splits: Default::default(), + viewport_base_width: None, + layout_columns: Vec::new(), + layout_revision: 0, + layout_undo: Default::default(), + }); + workspace.active_screen = workspace.screens.len() - 1; + } + // Adoption materializes a live pane without stealing focus, but it + // must still advance the pane-set revision used by frontend focus + // history pruning. + state.insert_pane(pane); + state.rebuild_resource_indexes(); } - #[cfg(unix)] - fn schedule_terminal_adoption( - self: &Arc, - options: SurfaceOptions, - record: crate::terminal_host_runtime::TerminalHostRecord, - record_path: std::path::PathBuf, + let revision = match commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "terminal-adopted", + terminal_id, + TerminalLifecycle::Running, + Some(incarnation), + None, ) { - if self.shutting_down.load(Ordering::Acquire) { - return; + Ok((_, revision)) => revision, + Err(error) => { + *state = before; + return Err(error); } - let terminal_id = record.terminal_id.clone(); - { - let mut tracked = self.terminal_adoptions.lock().unwrap(); - if tracked.contains(&terminal_id) { - return; + }; + owner_reservation.release(); + drop(state); + self.emit_terminal_registry_changed(®istry, revision); + #[cfg(debug_assertions)] + if let Some(signal) = std::env::var_os("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL") { + let _ = std::fs::write(signal, b"1"); + } + Ok(()) + } + + #[cfg(unix)] + fn ensure_terminal_adoption_workers(self: &Arc) -> std::io::Result<()> { + let coordinator = self.terminal_adoption_coordinator.clone(); + let (existing, missing) = { + let mut state = coordinator.state.lock().unwrap(); + if state.stopping { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "terminal adoption is stopping", + )); + } + if state.workers_running >= TERMINAL_ADOPTION_WORKERS { + return Ok(()); + } + let existing = state.workers_running; + let missing = TERMINAL_ADOPTION_WORKERS - existing; + state.workers_running += missing; + (existing, missing) + }; + let mut started = 0; + let mut first_error = None; + for worker in 0..missing { + let worker = existing + worker; + let mux = Arc::downgrade(self); + let worker_coordinator = coordinator.clone(); + match std::thread::Builder::new().name(format!("terminal-adoption-{worker}")).spawn( + move || { + Self::run_terminal_adoption_worker(mux, worker_coordinator); + }, + ) { + Ok(_) => started += 1, + Err(error) => { + let mut state = coordinator.state.lock().unwrap(); + state.workers_running = state.workers_running.saturating_sub(1); + coordinator.wake.notify_all(); + first_error.get_or_insert(error); } - tracked.insert(terminal_id.clone()); } + } + if existing + started > 0 { + Ok(()) + } else { + Err(first_error.expect("at least one worker spawn failed")) + } + } - let task = TerminalAdoptionTask { - options, - record, - record_path, - next_attempt: Instant::now() + Duration::from_millis(100), - delay: Duration::from_millis(100), - }; - let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); - if !state.enqueue(task) { - self.terminal_adoptions.lock().unwrap().remove(&terminal_id); - state.rescan_required = true; - state.next_rescan.get_or_insert_with(Instant::now); - self.terminal_adoption_coordinator.wake.notify_one(); + #[cfg(unix)] + fn consume_terminal_adoption_insert_failure(&self) -> bool { + self.terminal_adoption_insert_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| remaining.checked_sub(1)) + .is_ok() + } + + #[cfg(unix)] + fn schedule_terminal_adoption( + self: &Arc, + options: SurfaceOptions, + record: crate::terminal_host_runtime::TerminalHostRecord, + record_path: std::path::PathBuf, + ) { + if self.shutting_down.load(Ordering::Acquire) { + return; + } + let terminal_id = record.terminal_id.clone(); + { + let mut tracked = self.terminal_adoptions.lock().unwrap(); + if tracked.contains(&terminal_id) { return; } - drop(state); + tracked.insert(terminal_id.clone()); + } + + let task = TerminalAdoptionTask { + options, + record, + record_path, + next_attempt: Instant::now() + Duration::from_millis(100), + delay: Duration::from_millis(100), + }; + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + if !state.enqueue(task) { + self.terminal_adoptions.lock().unwrap().remove(&terminal_id); + state.rescan_required = true; + state.next_rescan.get_or_insert_with(Instant::now); self.terminal_adoption_coordinator.wake.notify_one(); - if self.ensure_terminal_adoption_workers().is_err() { - self.terminal_adoptions.lock().unwrap().remove(&terminal_id); - let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); - state.tasks.retain(|task| task.record.terminal_id != terminal_id); - state.deferred.retain(|task| task.record.terminal_id != terminal_id); - state.promote_deferred(); - state.rescan_required = true; - state.next_rescan.get_or_insert_with(Instant::now); - } + return; + } + drop(state); + self.terminal_adoption_coordinator.wake.notify_one(); + if self.ensure_terminal_adoption_workers().is_err() { + self.terminal_adoptions.lock().unwrap().remove(&terminal_id); + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + state.tasks.retain(|task| task.record.terminal_id != terminal_id); + state.deferred.retain(|task| task.record.terminal_id != terminal_id); + state.promote_deferred(); + state.rescan_required = true; + state.next_rescan.get_or_insert_with(Instant::now); } + } - #[cfg(unix)] - fn rescan_terminal_adoptions(self: &Arc) -> bool { - let options = self.surface_options.lock().unwrap().clone(); - let Some(root) = options.terminal_host_root.as_deref() else { return true }; - let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(root) else { + #[cfg(unix)] + fn rescan_terminal_adoptions(self: &Arc) -> bool { + let options = self.surface_options.lock().unwrap().clone(); + let Some(root) = options.terminal_host_root.as_deref() else { return true }; + let Ok(records) = crate::terminal_host_runtime::load_terminal_host_records(root) else { + return false; + }; + let registered = { + let registry = self.workspace_registry.lock().unwrap(); + let Ok(terminals) = registry.terminal_ids_including_tombstones() else { return false; }; - let registered = { - let registry = self.workspace_registry.lock().unwrap(); - let Ok(terminals) = registry.terminal_ids_including_tombstones() else { - return false; - }; - terminals.into_iter().collect::>() - }; - let live_incarnations = { - let state = self.state.lock().unwrap(); - let mut live = HashMap::>::new(); - for identity in state - .surfaces - .values() - .filter_map(|surface| surface.live_terminal_host_identity()) - { - live.entry(identity.terminal_id).or_default().insert(identity.incarnation); - } - live - }; - for (record_path, record) in records { - let already_live = registered.contains(&record.terminal_id) - && live_incarnations - .get(&record.terminal_id) - .is_some_and(|incarnations| incarnations.contains(&record.incarnation)); - if already_live { - continue; - } - self.schedule_terminal_adoption(options.clone(), record, record_path); + terminals.into_iter().collect::>() + }; + let live_incarnations = { + let state = self.state.lock().unwrap(); + let mut live = HashMap::>::new(); + for identity in + state.surfaces.values().filter_map(|surface| surface.live_terminal_host_identity()) + { + live.entry(identity.terminal_id).or_default().insert(identity.incarnation); } - true + live + }; + for (record_path, record) in records { + let already_live = registered.contains(&record.terminal_id) + && live_incarnations + .get(&record.terminal_id) + .is_some_and(|incarnations| incarnations.contains(&record.incarnation)); + if already_live { + continue; + } + self.schedule_terminal_adoption(options.clone(), record, record_path); } + true + } - #[cfg(unix)] - fn run_terminal_adoption_worker( - mux: Weak, - coordinator: Arc, - ) { - loop { - let stopping = coordinator.state.lock().unwrap().stopping; - if stopping { - Self::terminal_adoption_worker_stopped(&coordinator); - return; - } - let Some(mux) = mux.upgrade() else { - Self::terminal_adoption_worker_stopped(&coordinator); - return; - }; - if mux.shutting_down.load(Ordering::Acquire) { - mux.terminal_adoptions.lock().unwrap().clear(); - drop(mux); - Self::terminal_adoption_worker_stopped(&coordinator); - return; - } - - let mut state = coordinator.state.lock().unwrap(); - let now = Instant::now(); - if state.rescan_due(now) { - state.rescan_required = false; - state.next_rescan = None; - drop(state); - if !mux.rescan_terminal_adoptions() { - let mut state = coordinator.state.lock().unwrap(); - state.rescan_required = true; - state.next_rescan = Some(Instant::now() + state.rescan_delay); - state.rescan_delay = (state.rescan_delay * 2).min(Duration::from_secs(5)); - } else { - coordinator.state.lock().unwrap().rescan_delay = Duration::from_millis(100); - } - continue; - } + #[cfg(unix)] + fn run_terminal_adoption_worker( + mux: Weak, + coordinator: Arc, + ) { + loop { + let stopping = coordinator.state.lock().unwrap().stopping; + if stopping { + Self::terminal_adoption_worker_stopped(&coordinator); + return; + } + let Some(mux) = mux.upgrade() else { + Self::terminal_adoption_worker_stopped(&coordinator); + return; + }; + if mux.shutting_down.load(Ordering::Acquire) { + mux.terminal_adoptions.lock().unwrap().clear(); + drop(mux); + Self::terminal_adoption_worker_stopped(&coordinator); + return; + } - if let Some(index) = state.tasks.iter().position(|task| task.next_attempt <= now) { - let mut task = state.tasks.swap_remove(index); - state.in_flight += 1; - drop(state); - let complete = mux.try_terminal_adoption(&task); + let mut state = coordinator.state.lock().unwrap(); + let now = Instant::now(); + if state.rescan_due(now) { + state.rescan_required = false; + state.next_rescan = None; + drop(state); + if !mux.rescan_terminal_adoptions() { let mut state = coordinator.state.lock().unwrap(); - state.in_flight = state - .in_flight - .checked_sub(1) - .expect("adoption worker tracks in-flight tasks"); - if complete || mux.shutting_down.load(Ordering::Acquire) { - mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); - } else { - task.delay = (task.delay * 2).min(Duration::from_secs(5)); - task.next_attempt = Instant::now() + task.delay; - state.requeue_retry(task); - } - state.promote_deferred(); - drop(state); - coordinator.wake.notify_one(); - continue; + state.rescan_required = true; + state.next_rescan = Some(Instant::now() + state.rescan_delay); + state.rescan_delay = (state.rescan_delay * 2).min(Duration::from_secs(5)); + } else { + coordinator.state.lock().unwrap().rescan_delay = Duration::from_millis(100); } + continue; + } - let wait = state - .tasks - .iter() - .map(|task| task.next_attempt.saturating_duration_since(now)) - .min() - .unwrap_or(Duration::from_secs(1)) - .min(Duration::from_secs(1)); - drop(mux); - let _ = coordinator.wake.wait_timeout(state, wait).unwrap(); + if let Some(index) = state.tasks.iter().position(|task| task.next_attempt <= now) { + let mut task = state.tasks.swap_remove(index); + state.in_flight += 1; + drop(state); + let complete = mux.try_terminal_adoption(&task); + let mut state = coordinator.state.lock().unwrap(); + state.in_flight = + state.in_flight.checked_sub(1).expect("adoption worker tracks in-flight tasks"); + if complete || mux.shutting_down.load(Ordering::Acquire) { + mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); + } else { + task.delay = (task.delay * 2).min(Duration::from_secs(5)); + task.next_attempt = Instant::now() + task.delay; + state.requeue_retry(task); + } + state.promote_deferred(); + drop(state); + coordinator.wake.notify_one(); + continue; } - } - #[cfg(unix)] - fn terminal_adoption_worker_stopped(coordinator: &TerminalAdoptionCoordinator) { - let mut state = coordinator.state.lock().unwrap(); - state.workers_running = state.workers_running.saturating_sub(1); - coordinator.wake.notify_all(); + let wait = state + .tasks + .iter() + .map(|task| task.next_attempt.saturating_duration_since(now)) + .min() + .unwrap_or(Duration::from_secs(1)) + .min(Duration::from_secs(1)); + drop(mux); + let _ = coordinator.wake.wait_timeout(state, wait).unwrap(); } + } - #[cfg(unix)] - fn request_terminal_adoption_stop(&self) { - self.terminal_adoptions.lock().unwrap().clear(); - let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); - state.stopping = true; - state.tasks.clear(); - state.deferred.clear(); - self.terminal_adoption_coordinator.wake.notify_all(); - } + #[cfg(unix)] + fn terminal_adoption_worker_stopped(coordinator: &TerminalAdoptionCoordinator) { + let mut state = coordinator.state.lock().unwrap(); + state.workers_running = state.workers_running.saturating_sub(1); + coordinator.wake.notify_all(); + } - #[cfg(unix)] - fn wait_for_terminal_adoption_workers_until(&self, deadline: Instant) -> bool { - let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); - while state.workers_running != 0 { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return false; - }; - let (next, timeout) = - self.terminal_adoption_coordinator.wake.wait_timeout(state, remaining).unwrap(); - state = next; - if timeout.timed_out() && state.workers_running != 0 { - return false; - } - } - true - } + #[cfg(unix)] + fn request_terminal_adoption_stop(&self) { + self.terminal_adoptions.lock().unwrap().clear(); + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + state.stopping = true; + state.tasks.clear(); + state.deferred.clear(); + self.terminal_adoption_coordinator.wake.notify_all(); + } - #[cfg(unix)] - fn try_terminal_adoption(self: &Arc, task: &TerminalAdoptionTask) -> bool { - let terminal_id = &task.record.terminal_id; - let terminal = - match self.workspace_registry.lock().unwrap().terminal_record(terminal_id) { - Ok(terminal) => terminal, - Err(_) => return false, - }; - let Some(terminal) = terminal else { - return cleanup_terminal_host_record(&task.record, &task.record_path); + #[cfg(unix)] + fn wait_for_terminal_adoption_workers_until(&self, deadline: Instant) -> bool { + let mut state = self.terminal_adoption_coordinator.state.lock().unwrap(); + while state.workers_running != 0 { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; }; - if terminal.lifecycle == TerminalLifecycle::Exited { - if self.detach_exited_terminal_topology(terminal_id).is_err() { - return false; - } - return cleanup_terminal_host_record(&task.record, &task.record_path); + let (next, timeout) = + self.terminal_adoption_coordinator.wake.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && state.workers_running != 0 { + return false; } - if terminal.lifecycle == TerminalLifecycle::Tombstoned { - return cleanup_terminal_host_record(&task.record, &task.record_path); + } + true + } + + #[cfg(unix)] + fn try_terminal_adoption(self: &Arc, task: &TerminalAdoptionTask) -> bool { + let terminal_id = &task.record.terminal_id; + let terminal = match self.workspace_registry.lock().unwrap().terminal_record(terminal_id) { + Ok(terminal) => terminal, + Err(_) => return false, + }; + let Some(terminal) = terminal else { + return cleanup_terminal_host_record(&task.record, &task.record_path); + }; + if terminal.lifecycle == TerminalLifecycle::Exited { + if self.detach_exited_terminal_topology(terminal_id).is_err() { + return false; } - if terminal - .incarnation - .as_deref() - .is_some_and(|incarnation| incarnation != task.record.incarnation) + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if terminal.lifecycle == TerminalLifecycle::Tombstoned { + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if terminal + .incarnation + .as_deref() + .is_some_and(|incarnation| incarnation != task.record.incarnation) + { + if self + .mark_terminal_exited_and_detach( + terminal_id, + "terminal-incarnation-mismatch", + "host-incarnation-mismatch", + &task.options, + ) + .is_err() { - if self - .mark_terminal_exited_and_detach( - terminal_id, - "terminal-incarnation-mismatch", - "host-incarnation-mismatch", - &task.options, - ) - .is_err() - { - return false; - } - return cleanup_terminal_host_record(&task.record, &task.record_path); + return false; } - if terminal.lifecycle == TerminalLifecycle::Running { - let already_live = match self.resolve_terminal(terminal_id) { - Ok(resolution) => { - resolution.is_some_and(|resolution| resolution.surface.is_some()) - } - Err(_) => return false, - }; - if already_live { - return true; - } - if self - .transition_terminal_lifecycle( - "terminal-adopting", - "retry-terminal-adoption", - terminal_id, - TerminalLifecycle::Adopting, - Some(&task.record.incarnation), - None, - ) - .is_err() - { - let current = match self - .workspace_registry - .lock() - .unwrap() - .terminal_record(terminal_id) - { + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if terminal.lifecycle == TerminalLifecycle::Running { + let already_live = match self.resolve_terminal(terminal_id) { + Ok(resolution) => resolution.is_some_and(|resolution| resolution.surface.is_some()), + Err(_) => return false, + }; + if already_live { + return true; + } + if self + .transition_terminal_lifecycle( + "terminal-adopting", + "retry-terminal-adoption", + terminal_id, + TerminalLifecycle::Adopting, + Some(&task.record.incarnation), + None, + ) + .is_err() + { + let current = + match self.workspace_registry.lock().unwrap().terminal_record(terminal_id) { Ok(Some(current)) => current, Ok(None) | Err(_) => return false, }; - if current.lifecycle == TerminalLifecycle::Exited { - if self.detach_exited_terminal_topology(terminal_id).is_err() { - return false; - } - return cleanup_terminal_host_record(&task.record, &task.record_path); - } - if current.lifecycle == TerminalLifecycle::Tombstoned { - return cleanup_terminal_host_record(&task.record, &task.record_path); - } - if current.incarnation.as_deref() != Some(task.record.incarnation.as_str()) { + if current.lifecycle == TerminalLifecycle::Exited { + if self.detach_exited_terminal_topology(terminal_id).is_err() { return false; } - match current.lifecycle { - TerminalLifecycle::Adopting => {} - TerminalLifecycle::Running => { - return match self.resolve_terminal(terminal_id) { - Ok(resolution) => resolution - .is_some_and(|resolution| resolution.surface.is_some()), - Err(_) => false, - }; - } - TerminalLifecycle::Launching - | TerminalLifecycle::Exited - | TerminalLifecycle::Tombstoned => return false, + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if current.lifecycle == TerminalLifecycle::Tombstoned { + return cleanup_terminal_host_record(&task.record, &task.record_path); + } + if current.incarnation.as_deref() != Some(task.record.incarnation.as_str()) { + return false; + } + match current.lifecycle { + TerminalLifecycle::Adopting => {} + TerminalLifecycle::Running => { + return match self.resolve_terminal(terminal_id) { + Ok(resolution) => { + resolution.is_some_and(|resolution| resolution.surface.is_some()) + } + Err(_) => false, + }; } + TerminalLifecycle::Launching + | TerminalLifecycle::Exited + | TerminalLifecycle::Tombstoned => return false, } } - if terminal_host_record_liveness(&task.record_path, &task.record) - == TerminalHostLiveness::Dead - { - let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( - &task.record_path, - &task.record, - ); - return self - .mark_terminal_exited_and_detach( - terminal_id, - "terminal-host-proven-dead", - "host-process-ended-before-adoption", - &task.options, - ) - .is_ok(); - } - let Ok(_creation) = self.begin_surface_creation() else { - return true; - }; - let mut owner_reservation = match self.reserve_surface_owner() { - Ok(reservation) => reservation, - Err(_) => return false, - }; - let restored_binding = match self.restored_terminal_binding(terminal_id) { - Ok(binding) => binding, - Err(_) => return false, - }; - #[cfg(test)] - let id = restored_binding - .as_ref() - .and_then(|binding| binding.placements.first().map(|(slot, _)| *slot)) - .unwrap_or_else(|| self.next_id()); - #[cfg(test)] - let adoption_surface_factory = - self.terminal_adoption_surface_factory.lock().unwrap().clone(); - #[cfg(test)] - let adopted = if let Some(factory) = adoption_surface_factory { - factory(id) - } else { - self.adopt_restored_terminal( - restored_binding, + } + if terminal_host_record_liveness(&task.record_path, &task.record) + == TerminalHostLiveness::Dead + { + let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( + &task.record_path, + &task.record, + ); + return self + .mark_terminal_exited_and_detach( + terminal_id, + "terminal-host-proven-dead", + "host-process-ended-before-adoption", &task.options, - &task.record, - &task.record_path, ) - }; - #[cfg(not(test))] - let adopted = self.adopt_restored_terminal( + .is_ok(); + } + let Ok(_creation) = self.begin_surface_creation() else { + return true; + }; + let mut owner_reservation = match self.reserve_surface_owner() { + Ok(reservation) => reservation, + Err(_) => return false, + }; + let restored_binding = match self.restored_terminal_binding(terminal_id) { + Ok(binding) => binding, + Err(_) => return false, + }; + #[cfg(test)] + let id = restored_binding + .as_ref() + .and_then(|binding| binding.placements.first().map(|(slot, _)| *slot)) + .unwrap_or_else(|| self.next_id()); + #[cfg(test)] + let adoption_surface_factory = + self.terminal_adoption_surface_factory.lock().unwrap().clone(); + #[cfg(test)] + let adopted = if let Some(factory) = adoption_surface_factory { + factory(id) + } else { + self.adopt_restored_terminal( restored_binding, &task.options, &task.record, &task.record_path, - ); - let Ok(surface) = adopted else { return false }; - #[cfg(test)] - let after_attach = self.terminal_adoption_after_attach.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = after_attach { - hook(); - } - if self - .finish_terminal_adoption( - terminal_id, - &task.record.incarnation, - surface.clone(), - &mut owner_reservation, - ) - .is_ok() - { - self.reap_if_dead(&surface); - return true; - } - let host_is_dead = surface.is_dead() - || terminal_host_record_liveness(&task.record_path, &task.record) - == TerminalHostLiveness::Dead; - surface.disconnect_for_daemon_shutdown(); - owner_reservation.release(); - if !host_is_dead { - return false; - } - let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( - &task.record_path, - &task.record, - ); - self.mark_terminal_exited_and_detach( + ) + }; + #[cfg(not(test))] + let adopted = self.adopt_restored_terminal( + restored_binding, + &task.options, + &task.record, + &task.record_path, + ); + let Ok(surface) = adopted else { return false }; + #[cfg(test)] + let after_attach = self.terminal_adoption_after_attach.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = after_attach { + hook(); + } + if self + .finish_terminal_adoption( terminal_id, - "terminal-adoption-failed", - "host-adoption-topology-failed", - &task.options, + &task.record.incarnation, + surface.clone(), + &mut owner_reservation, ) .is_ok() + { + self.reap_if_dead(&surface); + return true; } - - #[cfg(test)] - pub(crate) fn new_for_test( - session: impl Into, - surface_options: SurfaceOptions, - ) -> Arc { - Self::new_with_test_surface_runtime( - session, - surface_options, - ProviderWorkspaceState::default(), - true, - ) + let host_is_dead = surface.is_dead() + || terminal_host_record_liveness(&task.record_path, &task.record) + == TerminalHostLiveness::Dead; + surface.disconnect_for_daemon_shutdown(); + owner_reservation.release(); + if !host_is_dead { + return false; } + let _ = crate::terminal_host_runtime::remove_stale_terminal_host_record( + &task.record_path, + &task.record, + ); + self.mark_terminal_exited_and_detach( + terminal_id, + "terminal-adoption-failed", + "host-adoption-topology-failed", + &task.options, + ) + .is_ok() + } - #[cfg(test)] - pub(crate) fn new_provider_managed_for_test( - session: impl Into, - surface_options: SurfaceOptions, - authority: ProviderWorkspaceAuthority, - ) -> Arc { - Self::new_with_test_surface_runtime( - session, - surface_options, - ProviderWorkspaceState { - managed: true, - mux_generation: None, - authority_generation: 1, - authority: Some(authority), - }, - true, - ) - } + #[cfg(test)] + pub(crate) fn new_for_test( + session: impl Into, + surface_options: SurfaceOptions, + ) -> Arc { + Self::new_with_test_surface_runtime( + session, + surface_options, + ProviderWorkspaceState::default(), + true, + ) + } - #[cfg(test)] - pub(crate) fn new_provider_managed_pending_for_test( - session: impl Into, - surface_options: SurfaceOptions, - mux_generation: &str, - ) -> Arc { - let mux = Self::new_with_test_surface_runtime( - session, - surface_options, - ProviderWorkspaceState { - managed: true, - mux_generation: Some(mux_generation.into()), - authority_generation: 0, - authority: None, - }, - true, - ); - validate_mux_generation(mux_generation).unwrap(); - mux - } + #[cfg(test)] + pub(crate) fn new_provider_managed_for_test( + session: impl Into, + surface_options: SurfaceOptions, + authority: ProviderWorkspaceAuthority, + ) -> Arc { + Self::new_with_test_surface_runtime( + session, + surface_options, + ProviderWorkspaceState { + managed: true, + mux_generation: None, + authority_generation: 1, + authority: Some(authority), + }, + true, + ) + } - fn next_id(&self) -> u64 { - self.next_id.fetch_add(1, Ordering::Relaxed) - } + #[cfg(test)] + pub(crate) fn new_provider_managed_pending_for_test( + session: impl Into, + surface_options: SurfaceOptions, + mux_generation: &str, + ) -> Arc { + let mux = Self::new_with_test_surface_runtime( + session, + surface_options, + ProviderWorkspaceState { + managed: true, + mux_generation: Some(mux_generation.into()), + authority_generation: 0, + authority: None, + }, + true, + ); + validate_mux_generation(mux_generation).unwrap(); + mux + } - fn begin_surface_creation(&self) -> anyhow::Result> { - self.surface_creations.begin() - } + fn next_id(&self) -> u64 { + self.next_id.fetch_add(1, Ordering::Relaxed) + } - fn ensure_surface_owner_capacity(&self) -> anyhow::Result<()> { - let state = self.state.lock().unwrap(); - self.ensure_surface_owner_capacity_with_state(&state) - } + fn begin_surface_creation(&self) -> anyhow::Result> { + self.surface_creations.begin() + } - fn ensure_surface_owner_capacity_with_state(&self, state: &State) -> anyhow::Result<()> { - let used = state - .surfaces - .len() - .saturating_add(self.shutdown_owners.len()) - .saturating_add(self.surface_owner_reservations.load(Ordering::Acquire)); - if used >= self.shutdown_owner_capacity() { - anyhow::bail!("surface_owner_capacity_exhausted"); - } - Ok(()) - } + fn ensure_surface_owner_capacity(&self) -> anyhow::Result<()> { + let state = self.state.lock().unwrap(); + self.ensure_surface_owner_capacity_with_state(&state) + } - fn reserve_surface_owner(&self) -> anyhow::Result> { - // Every topology removal transfers ownership to the shutdown ledger - // before releasing this same state lock. Counting reservations here - // therefore closes the only interval in which a new runtime could be - // spawned without an owner slot. - let state = self.state.lock().unwrap(); - self.ensure_surface_owner_capacity_with_state(&state)?; - self.surface_owner_reservations.fetch_add(1, Ordering::AcqRel); - drop(state); - Ok(SurfaceOwnerReservation { mux: self, active: true }) + fn ensure_surface_owner_capacity_with_state(&self, state: &State) -> anyhow::Result<()> { + let used = state + .surfaces + .len() + .saturating_add(self.shutdown_owners.len()) + .saturating_add(self.surface_owner_reservations.load(Ordering::Acquire)); + if used >= self.shutdown_owner_capacity() { + anyhow::bail!("surface_owner_capacity_exhausted"); } + Ok(()) + } - #[cfg(not(test))] - fn shutdown_owner_capacity(&self) -> usize { - SHUTDOWN_OWNER_CAPACITY - } + fn reserve_surface_owner(&self) -> anyhow::Result> { + // Every topology removal transfers ownership to the shutdown ledger + // before releasing this same state lock. Counting reservations here + // therefore closes the only interval in which a new runtime could be + // spawned without an owner slot. + let state = self.state.lock().unwrap(); + self.ensure_surface_owner_capacity_with_state(&state)?; + self.surface_owner_reservations.fetch_add(1, Ordering::AcqRel); + drop(state); + Ok(SurfaceOwnerReservation { mux: self, active: true }) + } - #[cfg(test)] - fn shutdown_owner_capacity(&self) -> usize { - self.shutdown_owner_capacity.load(Ordering::Acquire).min(SHUTDOWN_OWNER_CAPACITY) - } + #[cfg(not(test))] + fn shutdown_owner_capacity(&self) -> usize { + SHUTDOWN_OWNER_CAPACITY + } - fn next_active_at(&self) -> u64 { - self.next_active_at.fetch_add(1, Ordering::Relaxed) - } + #[cfg(test)] + fn shutdown_owner_capacity(&self) -> usize { + self.shutdown_owner_capacity.load(Ordering::Acquire).min(SHUTDOWN_OWNER_CAPACITY) + } - fn next_notification_id(&self) -> u64 { - self.next_notification_id.fetch_add(1, Ordering::Relaxed) - } + fn next_active_at(&self) -> u64 { + self.next_active_at.fetch_add(1, Ordering::Relaxed) + } - /// Allocate an undo-coalescing owner for one in-process frontend. - /// - /// Layout undo is in-memory state, so this namespace intentionally follows - /// the mux lifecycle rather than durable workspace identity. - pub fn allocate_in_process_resize_owner(&self) -> u64 { - self.next_in_process_resize_owner - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |owner| { - Some(owner.wrapping_add(1).max(1)) - }) - .expect("in-process resize owner allocation cannot fail") - } + fn next_notification_id(&self) -> u64 { + self.next_notification_id.fetch_add(1, Ordering::Relaxed) + } + + /// Allocate an undo-coalescing owner for one in-process frontend. + /// + /// Layout undo is in-memory state, so this namespace intentionally follows + /// the mux lifecycle rather than durable workspace identity. + pub fn allocate_in_process_resize_owner(&self) -> u64 { + self.next_in_process_resize_owner + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |owner| { + Some(owner.wrapping_add(1).max(1)) + }) + .expect("in-process resize owner allocation cannot fail") + } - fn new_workspace_key() -> anyhow::Result { - let mut bytes = [0u8; 16]; - getrandom::fill(&mut bytes).map_err(|_| { + fn new_workspace_key() -> anyhow::Result { + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).map_err(|_| { anyhow::anyhow!( "could not create workspace identity; retry, then restart cmux if the problem continues" ) })?; - // RFC 9562 UUIDv4 version and variant bits. Keeping the formatter - // local avoids making stable workspace identity depend on a UUID - // library at the protocol boundary. - bytes[6] = (bytes[6] & 0x0f) | 0x40; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - Ok(format!( - "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", - bytes[0], - bytes[1], - bytes[2], - bytes[3], - bytes[4], - bytes[5], - bytes[6], - bytes[7], - bytes[8], - bytes[9], - bytes[10], - bytes[11], - bytes[12], - bytes[13], - bytes[14], - bytes[15] - )) - } + // RFC 9562 UUIDv4 version and variant bits. Keeping the formatter + // local avoids making stable workspace identity depend on a UUID + // library at the protocol boundary. + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(format!( + "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + bytes[0], + bytes[1], + bytes[2], + bytes[3], + bytes[4], + bytes[5], + bytes[6], + bytes[7], + bytes[8], + bytes[9], + bytes[10], + bytes[11], + bytes[12], + bytes[13], + bytes[14], + bytes[15] + )) + } - fn validate_workspace_key(key: &str) -> anyhow::Result<()> { - if key.trim().is_empty() { - anyhow::bail!("workspace key cannot be empty"); - } - if key.len() > WORKSPACE_KEY_MAX_BYTES { - anyhow::bail!("workspace key exceeds {WORKSPACE_KEY_MAX_BYTES} bytes"); - } - Ok(()) + fn validate_workspace_key(key: &str) -> anyhow::Result<()> { + if key.trim().is_empty() { + anyhow::bail!("workspace key cannot be empty"); } - - fn validate_workspace_name(name: &str) -> anyhow::Result<()> { - if name.len() > WORKSPACE_NAME_MAX_BYTES { - anyhow::bail!("workspace name exceeds {WORKSPACE_NAME_MAX_BYTES} bytes"); - } - Ok(()) + if key.len() > WORKSPACE_KEY_MAX_BYTES { + anyhow::bail!("workspace key exceeds {WORKSPACE_KEY_MAX_BYTES} bytes"); } + Ok(()) + } - fn workspace_lifecycle(&self, workspace: WorkspaceId) -> Arc> { - let mut lifecycles = self.workspace_lifecycles.lock().unwrap(); - lifecycles.retain(|_, lifecycle| lifecycle.strong_count() > 0); - if let Some(lifecycle) = lifecycles.get(&workspace).and_then(Weak::upgrade) { - return lifecycle; - } - let lifecycle = Arc::new(Mutex::new(())); - lifecycles.insert(workspace, Arc::downgrade(&lifecycle)); - lifecycle + fn validate_workspace_name(name: &str) -> anyhow::Result<()> { + if name.len() > WORKSPACE_NAME_MAX_BYTES { + anyhow::bail!("workspace name exceeds {WORKSPACE_NAME_MAX_BYTES} bytes"); } + Ok(()) + } - /// Permanently assigns workspace rename/delete ownership to the external - /// provider for this mux generation. The transition is intentionally - /// one-way so a stale frontend cannot reopen ordinary mutation paths. - pub fn mark_workspaces_provider_managed_internal(&self) { - self.provider_workspace.lock().unwrap().managed = true; + fn workspace_lifecycle(&self, workspace: WorkspaceId) -> Arc> { + let mut lifecycles = self.workspace_lifecycles.lock().unwrap(); + lifecycles.retain(|_, lifecycle| lifecycle.strong_count() > 0); + if let Some(lifecycle) = lifecycles.get(&workspace).and_then(Weak::upgrade) { + return lifecycle; } + let lifecycle = Arc::new(Mutex::new(())); + lifecycles.insert(workspace, Arc::downgrade(&lifecycle)); + lifecycle + } - pub fn workspaces_are_provider_managed(&self) -> bool { - self.provider_workspace.lock().unwrap().managed - } + /// Permanently assigns workspace rename/delete ownership to the external + /// provider for this mux generation. The transition is intentionally + /// one-way so a stale frontend cannot reopen ordinary mutation paths. + pub fn mark_workspaces_provider_managed_internal(&self) { + self.provider_workspace.lock().unwrap().managed = true; + } - pub fn provider_workspace_authority_status(&self) -> ProviderWorkspaceAuthorityStatus { - self.provider_workspace.lock().unwrap().status() - } + pub fn workspaces_are_provider_managed(&self) -> bool { + self.provider_workspace.lock().unwrap().managed + } - pub fn install_or_rotate_provider_workspace_authority( - &self, - mux_generation: &str, - expected_authority_generation: u64, - authority_generation: u64, - authority: ProviderWorkspaceAuthority, - ) -> Result - { - let mut state = self.provider_workspace.lock().unwrap(); - if !state.managed || state.mux_generation.is_none() { - return Err(ProviderWorkspaceAuthorityUpdateError::Unmanaged); - } - if state.mux_generation.as_deref() != Some(mux_generation) { - return Err(ProviderWorkspaceAuthorityUpdateError::MuxGenerationMismatch); - } + pub fn provider_workspace_authority_status(&self) -> ProviderWorkspaceAuthorityStatus { + self.provider_workspace.lock().unwrap().status() + } - if authority_generation == state.authority_generation { - let identical = state.authority.as_ref().is_some_and(|installed| { - constant_time_eq(authority.expose(), installed.expose()) - }); - return if identical { - Ok(state.status()) - } else { - Err(ProviderWorkspaceAuthorityUpdateError::GenerationConflict) - }; - } - - if expected_authority_generation != state.authority_generation { - return Err(ProviderWorkspaceAuthorityUpdateError::ExpectedGenerationMismatch); - } - let valid_initial_install = state.authority_generation == 0 - && state.authority.is_none() - && authority_generation > 0; - let valid_rotation = state.authority.is_some() - && authority_generation == state.authority_generation.saturating_add(1); - if !valid_initial_install && !valid_rotation { - return Err(ProviderWorkspaceAuthorityUpdateError::InvalidGeneration); - } - state.authority_generation = authority_generation; - state.authority = Some(authority); - Ok(state.status()) + pub fn install_or_rotate_provider_workspace_authority( + &self, + mux_generation: &str, + expected_authority_generation: u64, + authority_generation: u64, + authority: ProviderWorkspaceAuthority, + ) -> Result { + let mut state = self.provider_workspace.lock().unwrap(); + if !state.managed || state.mux_generation.is_none() { + return Err(ProviderWorkspaceAuthorityUpdateError::Unmanaged); } - - /// Validates the secret provisioned for this provider-owned mux - /// generation. The same rejection covers missing and incorrect secrets so - /// the control socket cannot be used to probe whether a value was set. - pub fn authorize_provider_workspace_authority(&self, provided: &str) -> anyhow::Result<()> { - let authorized = - self.provider_workspace.lock().unwrap().authority.as_ref().is_some_and( - |expected| constant_time_eq(provided.as_bytes(), expected.expose()), - ); - if !authorized { - anyhow::bail!("invalid provider workspace authority"); - } - Ok(()) + if state.mux_generation.as_deref() != Some(mux_generation) { + return Err(ProviderWorkspaceAuthorityUpdateError::MuxGenerationMismatch); } - fn authorize_workspace_lifecycle_mutation( - &self, - authorization: WorkspaceMutationAuthority<'_>, - operation: &str, - ) -> anyhow::Result> { - let authority = self.provider_workspace.lock().unwrap(); - if authority.managed && matches!(authorization, WorkspaceMutationAuthority::Ordinary) { - anyhow::bail!( - "cannot {operation} a provider-managed workspace directly; use the managed workspace lifecycle controls" - ); - } - if !authority.managed && !matches!(authorization, WorkspaceMutationAuthority::Ordinary) - { - anyhow::bail!( - "cannot apply provider workspace {operation}; this session is not provider-managed" - ); - } - if let WorkspaceMutationAuthority::ProviderCredential(provided) = authorization { - let authorized = authority.authority.as_ref().is_some_and(|expected| { - constant_time_eq(provided.as_bytes(), expected.expose()) - }); - if !authorized { - anyhow::bail!("invalid provider workspace authority"); - } - } - Ok(authority) + if authority_generation == state.authority_generation { + let identical = state + .authority + .as_ref() + .is_some_and(|installed| constant_time_eq(authority.expose(), installed.expose())); + return if identical { + Ok(state.status()) + } else { + Err(ProviderWorkspaceAuthorityUpdateError::GenerationConflict) + }; } - fn pending_workspace_surface(&self, surface: SurfaceId) -> PendingWorkspaceSurface<'_> { - PendingWorkspaceSurface { pending: &self.pending_workspace_surfaces, surface } + if expected_authority_generation != state.authority_generation { + return Err(ProviderWorkspaceAuthorityUpdateError::ExpectedGenerationMismatch); } - - fn workspace_for_surface_in_state( - state: &State, - surface: SurfaceId, - ) -> Option { - let pane = state.pane_of(surface)?; - let (workspace, _) = state.screen_of(pane)?; - Some(state.workspaces[workspace].id) + let valid_initial_install = state.authority_generation == 0 + && state.authority.is_none() + && authority_generation > 0; + let valid_rotation = state.authority.is_some() + && authority_generation == state.authority_generation.saturating_add(1); + if !valid_initial_install && !valid_rotation { + return Err(ProviderWorkspaceAuthorityUpdateError::InvalidGeneration); } + state.authority_generation = authority_generation; + state.authority = Some(authority); + Ok(state.status()) + } - fn workspace_for_tree_target_in_state( - state: &State, - target: TreeCloseTarget, - ) -> Option { - match target { - TreeCloseTarget::Pane(pane) => { - let (workspace, _) = state.screen_of(pane)?; - Some(state.workspaces[workspace].id) - } - TreeCloseTarget::Screen(screen) => state - .workspaces - .iter() - .find(|workspace| { - workspace.screens.iter().any(|candidate| candidate.id == screen) - }) - .map(|workspace| workspace.id), - } + /// Validates the secret provisioned for this provider-owned mux + /// generation. The same rejection covers missing and incorrect secrets so + /// the control socket cannot be used to probe whether a value was set. + pub fn authorize_provider_workspace_authority(&self, provided: &str) -> anyhow::Result<()> { + let authorized = self + .provider_workspace + .lock() + .unwrap() + .authority + .as_ref() + .is_some_and(|expected| constant_time_eq(provided.as_bytes(), expected.expose())); + if !authorized { + anyhow::bail!("invalid provider workspace authority"); } + Ok(()) + } - fn surface_workspace(&self, surface: SurfaceId) -> Option { - self.pending_workspace_surfaces.lock().unwrap().get(&surface).copied().or_else(|| { - let state = self.state.lock().unwrap(); - Self::workspace_for_surface_in_state(&state, surface) - }) + fn authorize_workspace_lifecycle_mutation( + &self, + authorization: WorkspaceMutationAuthority<'_>, + operation: &str, + ) -> anyhow::Result> { + let authority = self.provider_workspace.lock().unwrap(); + if authority.managed && matches!(authorization, WorkspaceMutationAuthority::Ordinary) { + anyhow::bail!( + "cannot {operation} a provider-managed workspace directly; use the managed workspace lifecycle controls" + ); } - - fn require_workspace_revision(state: &State, expected: Option) -> anyhow::Result<()> { - if let Some(expected) = expected - && expected != state.workspace_revision - { - anyhow::bail!( - "workspace revision conflict: expected {expected}, current {}", - state.workspace_revision - ); + if !authority.managed && !matches!(authorization, WorkspaceMutationAuthority::Ordinary) { + anyhow::bail!( + "cannot apply provider workspace {operation}; this session is not provider-managed" + ); + } + if let WorkspaceMutationAuthority::ProviderCredential(provided) = authorization { + let authorized = authority + .authority + .as_ref() + .is_some_and(|expected| constant_time_eq(provided.as_bytes(), expected.expose())); + if !authorized { + anyhow::bail!("invalid provider workspace authority"); } - Ok(()) } + Ok(authority) + } - fn registry_projection(&self, state: &State) -> Vec { - state + fn pending_workspace_surface(&self, surface: SurfaceId) -> PendingWorkspaceSurface<'_> { + PendingWorkspaceSurface { pending: &self.pending_workspace_surfaces, surface } + } + + fn workspace_for_surface_in_state(state: &State, surface: SurfaceId) -> Option { + let pane = state.pane_of(surface)?; + let (workspace, _) = state.screen_of(pane)?; + Some(state.workspaces[workspace].id) + } + + fn workspace_for_tree_target_in_state( + state: &State, + target: TreeCloseTarget, + ) -> Option { + match target { + TreeCloseTarget::Pane(pane) => { + let (workspace, _) = state.screen_of(pane)?; + Some(state.workspaces[workspace].id) + } + TreeCloseTarget::Screen(screen) => state .workspaces .iter() - .map(|workspace| RegistryWorkspace { - id: workspace.id, - public_id: workspace.public_id.clone(), - key: workspace.key.clone(), - name: workspace.name.clone(), - group_key: self.session.clone(), - }) - .collect() + .find(|workspace| workspace.screens.iter().any(|candidate| candidate.id == screen)) + .map(|workspace| workspace.id), } + } - fn ordinary_resource_selectors() -> crate::ResourceSelectors { - crate::ResourceSelectors { - machine: Some("current".into()), - session: Some("current".into()), - ..crate::ResourceSelectors::default() - } - } + fn surface_workspace(&self, surface: SurfaceId) -> Option { + self.pending_workspace_surfaces.lock().unwrap().get(&surface).copied().or_else(|| { + let state = self.state.lock().unwrap(); + Self::workspace_for_surface_in_state(&state, surface) + }) + } - fn ordinary_workspace_selectors( - &self, - workspace: WorkspaceId, - ) -> Option { - let public_id = self.with_state(|state| { - state.resource_indexes.workspace_ids.get(&workspace).cloned() - })?; - Some(crate::ResourceSelectors { - workspace: Some(public_id.to_string()), - ..Self::ordinary_resource_selectors() - }) + fn require_workspace_revision(state: &State, expected: Option) -> anyhow::Result<()> { + if let Some(expected) = expected + && expected != state.workspace_revision + { + anyhow::bail!( + "workspace revision conflict: expected {expected}, current {}", + state.workspace_revision + ); } + Ok(()) + } - fn ordinary_screen_selectors(&self, screen: ScreenId) -> Option { - let public_id = - self.with_state(|state| state.resource_indexes.screen_ids.get(&screen).cloned())?; - Some(crate::ResourceSelectors { - screen: Some(public_id.to_string()), - ..Self::ordinary_resource_selectors() + fn registry_projection(&self, state: &State) -> Vec { + state + .workspaces + .iter() + .map(|workspace| RegistryWorkspace { + id: workspace.id, + public_id: workspace.public_id.clone(), + key: workspace.key.clone(), + name: workspace.name.clone(), + group_key: self.session.clone(), }) - } + .collect() + } - fn ordinary_pane_selectors(&self, pane: PaneId) -> Option { - let public_id = - self.with_state(|state| state.resource_indexes.pane_ids.get(&pane).cloned())?; - Some(crate::ResourceSelectors { - pane: Some(public_id.to_string()), - ..Self::ordinary_resource_selectors() - }) + fn ordinary_resource_selectors() -> crate::ResourceSelectors { + crate::ResourceSelectors { + machine: Some("current".into()), + session: Some("current".into()), + ..crate::ResourceSelectors::default() } + } - fn ordinary_tab_selectors(&self, surface: SurfaceId) -> Option { - let public_id = - self.with_state(|state| state.resource_indexes.tab_ids.get(&surface).cloned())?; - Some(crate::ResourceSelectors { - tab: Some(public_id.to_string()), - ..Self::ordinary_resource_selectors() - }) - } + fn ordinary_workspace_selectors( + &self, + workspace: WorkspaceId, + ) -> Option { + let public_id = + self.with_state(|state| state.resource_indexes.workspace_ids.get(&workspace).cloned())?; + Some(crate::ResourceSelectors { + workspace: Some(public_id.to_string()), + ..Self::ordinary_resource_selectors() + }) + } - fn commit_ordinary_topology_operation( - self: &Arc, - operation: ResourceOperation, - selectors: crate::ResourceSelectors, - fields: Map, - ) -> anyhow::Result { - self.commit_resource_topology_operation( - operation, - selectors, - fields, - None, - &WorkspaceMutation::local("cmux-tui"), - ) - } + fn ordinary_screen_selectors(&self, screen: ScreenId) -> Option { + let public_id = + self.with_state(|state| state.resource_indexes.screen_ids.get(&screen).cloned())?; + Some(crate::ResourceSelectors { + screen: Some(public_id.to_string()), + ..Self::ordinary_resource_selectors() + }) + } - fn nullable_name_fields(name: String) -> Map { - Map::from_iter([( - "name".into(), - if name.is_empty() { Value::Null } else { Value::String(name) }, - )]) - } + fn ordinary_pane_selectors(&self, pane: PaneId) -> Option { + let public_id = + self.with_state(|state| state.resource_indexes.pane_ids.get(&pane).cloned())?; + Some(crate::ResourceSelectors { + pane: Some(public_id.to_string()), + ..Self::ordinary_resource_selectors() + }) + } - fn insert_cell_size(fields: &mut Map, size: Option<(u16, u16)>) { - if let Some((cols, rows)) = size { - fields.insert("cols".into(), Value::from(cols)); - fields.insert("rows".into(), Value::from(rows)); - } + fn ordinary_tab_selectors(&self, surface: SurfaceId) -> Option { + let public_id = + self.with_state(|state| state.resource_indexes.tab_ids.get(&surface).cloned())?; + Some(crate::ResourceSelectors { + tab: Some(public_id.to_string()), + ..Self::ordinary_resource_selectors() + }) + } + + fn commit_ordinary_topology_operation( + self: &Arc, + operation: ResourceOperation, + selectors: crate::ResourceSelectors, + fields: Map, + ) -> anyhow::Result { + self.commit_resource_topology_operation( + operation, + selectors, + fields, + None, + &WorkspaceMutation::local("cmux-tui"), + ) + } + + fn nullable_name_fields(name: String) -> Map { + Map::from_iter([( + "name".into(), + if name.is_empty() { Value::Null } else { Value::String(name) }, + )]) + } + + fn insert_cell_size(fields: &mut Map, size: Option<(u16, u16)>) { + if let Some((cols, rows)) = size { + fields.insert("cols".into(), Value::from(cols)); + fields.insert("rows".into(), Value::from(rows)); } + } - fn insert_optional_string( - fields: &mut Map, - name: &'static str, - value: Option, - ) { - if let Some(value) = value { - fields.insert(name.into(), Value::String(value)); - } + fn insert_optional_string( + fields: &mut Map, + name: &'static str, + value: Option, + ) { + if let Some(value) = value { + fields.insert(name.into(), Value::String(value)); } + } - fn ordinary_created_surface( - &self, - commit: &ResourcePatchCommit, - ) -> anyhow::Result> { - let tab_id = TabPublicId::parse( - commit.result["tab_id"] - .as_str() - .context("created resource result omitted its tab id")? - .to_string(), - )?; - let surface = self - .with_state(|state| state.resource_indexes.tabs.get(&tab_id).copied()) - .context("created tab disappeared")?; - self.surface(surface).context("created surface disappeared") - } - - #[allow(clippy::too_many_arguments)] - fn commit_resource_mutation_plan( - &self, - mutation: &WorkspaceMutation, - operation: &str, - fingerprint: &Value, - expected_generation: Option<&str>, - expected_revision: Option, - prepare: impl FnOnce(&mut State, &WorkspaceRegistry) -> anyhow::Result, - ) -> anyhow::Result { - let mut registry = self.workspace_registry.lock().unwrap(); - if let Some(replay) = - registry.replay_resource_patch(mutation, operation, fingerprint)? - { - return Ok(replay); - } + fn ordinary_created_surface( + &self, + commit: &ResourcePatchCommit, + ) -> anyhow::Result> { + let tab_id = TabPublicId::parse( + commit.result["tab_id"] + .as_str() + .context("created resource result omitted its tab id")? + .to_string(), + )?; + let surface = self + .with_state(|state| state.resource_indexes.tabs.get(&tab_id).copied()) + .context("created tab disappeared")?; + self.surface(surface).context("created surface disappeared") + } - let mut state = self.state.lock().unwrap(); - let mut plan = prepare(&mut state, ®istry)?; - persist_public_topology_result(operation, &mut plan.result, &plan.deltas)?; - #[cfg(test)] - { - *self.resource_mutation_metrics.lock().unwrap() = Some(plan.metrics); - } - let commit = registry.commit_resource_patch( - mutation, - operation, - fingerprint, - expected_generation, - expected_revision, - &plan.patch, - &plan.result, - &plan.deltas, - )?; - plan.apply(&mut state, &commit); - drop(state); - drop(registry); - if !commit.replayed { - self.publish_resource_event(); - } - Ok(commit) + #[allow(clippy::too_many_arguments)] + fn commit_resource_mutation_plan( + &self, + mutation: &WorkspaceMutation, + operation: &str, + fingerprint: &Value, + expected_generation: Option<&str>, + expected_revision: Option, + prepare: impl FnOnce(&mut State, &WorkspaceRegistry) -> anyhow::Result, + ) -> anyhow::Result { + let mut registry = self.workspace_registry.lock().unwrap(); + if let Some(replay) = registry.replay_resource_patch(mutation, operation, fingerprint)? { + return Ok(replay); } + let mut state = self.state.lock().unwrap(); + let mut plan = prepare(&mut state, ®istry)?; + persist_public_topology_result(operation, &mut plan.result, &plan.deltas)?; #[cfg(test)] - pub(crate) fn resource_create_empty_workspace( - &self, - name: Option, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - if let Some(name) = name.as_deref() { - Self::validate_workspace_name(name)?; - } - let workspace_slot = self.next_id(); - let public_id = WorkspacePublicId::random()?; - let generated_key = Self::new_workspace_key()?; - let fingerprint = serde_json::json!({ - "operation": "workspace.create", - "name": name, - "initial_content": "empty", - }); - self.commit_resource_mutation_plan( - mutation, - "workspace.create", - &fingerprint, - expected_generation, - expected_revision, - move |state, registry| { - anyhow::ensure!( - state.workspaces.len() < WORKSPACE_REGISTRY_LIMIT, - "workspace limit reached ({WORKSPACE_REGISTRY_LIMIT})" - ); - let key = generated_key.clone(); - let name = name.clone().unwrap_or_else(|| Self::default_workspace_name(state)); - let index = state.workspaces.len(); - let workspace = Workspace { - id: workspace_slot, - public_id: public_id.clone(), - key: key.clone(), - name: name.clone(), - screens: Vec::new(), - active_screen: 0, - }; - let mut order = Vec::with_capacity(index + 1); - order.extend( - state.workspaces.iter().map(|workspace| workspace.public_id.clone()), - ); - order.push(public_id.clone()); - state.workspaces.reserve(1); - state.workspace_index_by_id.reserve(1); - state.workspace_id_by_key.reserve(1); - state.resource_indexes.workspaces.reserve(1); - state.resource_indexes.workspace_ids.reserve(1); - let result = serde_json::json!({ - "workspace": public_id.as_str(), - "name": name, - "index": index, - }); - let mut deltas = Vec::with_capacity(2); - if let Some(previous) = state.workspaces.get(state.active_workspace) { - deltas.push(workspace_resource_upsert( - 0, - registry.session_id().as_str(), - &previous.public_id, - &previous.name, - state.active_workspace, - false, - )); - } - deltas.push(workspace_resource_upsert( - deltas.len(), - registry.session_id().as_str(), - &public_id, - &name, - index, - true, - )); - Ok(ResourceMutationPlan::new( - ResourcePatch { - changes: vec![ - ResourceChange::UpsertWorkspace { - workspace: RegistryWorkspace { - id: workspace.id, - public_id: public_id.clone(), - key, - name, - group_key: self.session.clone(), - }, - position: index, - active_screen: None, - }, - ResourceChange::SetWorkspaceOrder { workspace_ids: order }, - ResourceChange::SetActiveWorkspace { - workspace_id: Some(public_id), - }, - ], - }, - result, - Value::Array(deltas), - move |state| { - state.push_workspace(workspace); - state.active_workspace = index; - state.workspace_revision = state.workspace_revision.saturating_add(1); - }, - ) - .with_metrics(ResourceMutationMetrics { - touched_resources: 1, - order_entries: index + 1, - terminal_queries: 0, - changed_rows: index + 3, - })) - }, - ) + { + *self.resource_mutation_metrics.lock().unwrap() = Some(plan.metrics); + } + let commit = registry.commit_resource_patch( + mutation, + operation, + fingerprint, + expected_generation, + expected_revision, + &plan.patch, + &plan.result, + &plan.deltas, + )?; + plan.apply(&mut state, &commit); + drop(state); + drop(registry); + if !commit.replayed { + self.publish_resource_event(); } + Ok(commit) + } - #[cfg(test)] - pub(crate) fn resource_rename_workspace( - &self, - workspace_id: &WorkspacePublicId, - name: String, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - Self::validate_workspace_name(&name)?; - let fingerprint = serde_json::json!({ - "operation": "workspace.rename", - "workspace": workspace_id.as_str(), - "name": name, - }); - let target = workspace_id.clone(); - self.commit_resource_mutation_plan( - mutation, - "workspace.rename", - &fingerprint, - expected_generation, - expected_revision, - move |state, registry| { - let slot = *state - .resource_indexes - .workspaces - .get(&target) - .with_context(|| format!("unknown workspace {target}"))?; - let index = state - .workspace_index(slot) - .with_context(|| format!("workspace {target} has no live slot"))?; - let workspace = &state.workspaces[index]; - let changed = workspace.name != name; - let active_screen = workspace - .screens - .get(workspace.active_screen) - .map(|screen| screen.public_id.clone()); - let durable = RegistryWorkspace { - id: workspace.id, - public_id: workspace.public_id.clone(), - key: workspace.key.clone(), - name: name.clone(), - group_key: self.session.clone(), - }; - let result = serde_json::json!({ - "workspace": target.as_str(), - "name": name, - "changed": changed, - }); - let deltas = Value::Array(vec![workspace_resource_upsert( + #[cfg(test)] + pub(crate) fn resource_create_empty_workspace( + &self, + name: Option, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + if let Some(name) = name.as_deref() { + Self::validate_workspace_name(name)?; + } + let workspace_slot = self.next_id(); + let public_id = WorkspacePublicId::random()?; + let generated_key = Self::new_workspace_key()?; + let fingerprint = serde_json::json!({ + "operation": "workspace.create", + "name": name, + "initial_content": "empty", + }); + self.commit_resource_mutation_plan( + mutation, + "workspace.create", + &fingerprint, + expected_generation, + expected_revision, + move |state, registry| { + anyhow::ensure!( + state.workspaces.len() < WORKSPACE_REGISTRY_LIMIT, + "workspace limit reached ({WORKSPACE_REGISTRY_LIMIT})" + ); + let key = generated_key.clone(); + let name = name.clone().unwrap_or_else(|| Self::default_workspace_name(state)); + let index = state.workspaces.len(); + let workspace = Workspace { + id: workspace_slot, + public_id: public_id.clone(), + key: key.clone(), + name: name.clone(), + screens: Vec::new(), + active_screen: 0, + }; + let mut order = Vec::with_capacity(index + 1); + order.extend(state.workspaces.iter().map(|workspace| workspace.public_id.clone())); + order.push(public_id.clone()); + state.workspaces.reserve(1); + state.workspace_index_by_id.reserve(1); + state.workspace_id_by_key.reserve(1); + state.resource_indexes.workspaces.reserve(1); + state.resource_indexes.workspace_ids.reserve(1); + let result = serde_json::json!({ + "workspace": public_id.as_str(), + "name": name, + "index": index, + }); + let mut deltas = Vec::with_capacity(2); + if let Some(previous) = state.workspaces.get(state.active_workspace) { + deltas.push(workspace_resource_upsert( 0, registry.session_id().as_str(), - &target, - &name, - index, - index == state.active_workspace, - )]); - Ok(ResourceMutationPlan::new( - ResourcePatch { - changes: vec![ResourceChange::UpsertWorkspace { - workspace: durable, + &previous.public_id, + &previous.name, + state.active_workspace, + false, + )); + } + deltas.push(workspace_resource_upsert( + deltas.len(), + registry.session_id().as_str(), + &public_id, + &name, + index, + true, + )); + Ok(ResourceMutationPlan::new( + ResourcePatch { + changes: vec![ + ResourceChange::UpsertWorkspace { + workspace: RegistryWorkspace { + id: workspace.id, + public_id: public_id.clone(), + key, + name, + group_key: self.session.clone(), + }, position: index, - active_screen, - }], - }, - result, - deltas, - move |state| { - state.workspaces[index].name = name; - state.workspace_revision = state.workspace_revision.saturating_add(1); - }, - ) - .with_metrics(ResourceMutationMetrics { - touched_resources: 1, - order_entries: 0, - terminal_queries: 0, - changed_rows: 1, - })) - }, - ) - } + active_screen: None, + }, + ResourceChange::SetWorkspaceOrder { workspace_ids: order }, + ResourceChange::SetActiveWorkspace { workspace_id: Some(public_id) }, + ], + }, + result, + Value::Array(deltas), + move |state| { + state.push_workspace(workspace); + state.active_workspace = index; + state.workspace_revision = state.workspace_revision.saturating_add(1); + }, + ) + .with_metrics(ResourceMutationMetrics { + touched_resources: 1, + order_entries: index + 1, + terminal_queries: 0, + changed_rows: index + 3, + })) + }, + ) + } - pub(crate) fn resource_rename_workspace_selected( - &self, - selectors: crate::ResourceSelectors, - name: String, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - Self::validate_workspace_name(&name)?; - let fingerprint = serde_json::json!({ - "operation": "workspace.rename", - "selectors": selectors, - "name": name, - }); - self.commit_resource_mutation_plan( - mutation, - "workspace.rename", - &fingerprint, - expected_generation, - expected_revision, - move |state, registry| { - let resolved = self - .resolve_resource_path_in_state( - state, - registry, - crate::ResourceTarget::Workspace, - &selectors, - ) - .map_err(anyhow::Error::new)?; - let target = resolved - .path - .workspace - .expect("workspace target resolution returns a workspace"); - let slot = resolved - .workspace - .expect("workspace target resolution returns a live slot"); - let index = state - .workspace_index(slot) - .with_context(|| format!("workspace {target} has no live slot"))?; - #[cfg(test)] - if let Some(hook) = - self.resource_rename_after_selector_resolution.lock().unwrap().clone() - { - hook(&target); - } - let workspace = &state.workspaces[index]; - let changed = workspace.name != name; - let active_screen = workspace - .screens - .get(workspace.active_screen) - .map(|screen| screen.public_id.clone()); - let durable = RegistryWorkspace { - id: workspace.id, - public_id: workspace.public_id.clone(), - key: workspace.key.clone(), - name: name.clone(), - group_key: self.session.clone(), - }; - let result = serde_json::json!({ - "workspace": target.as_str(), - "name": name, - "changed": changed, - }); - let deltas = Value::Array(vec![workspace_resource_upsert( - 0, - registry.session_id().as_str(), - &target, - &name, - index, - index == state.active_workspace, - )]); - Ok(ResourceMutationPlan::new( - ResourcePatch { - changes: vec![ResourceChange::UpsertWorkspace { - workspace: durable, - position: index, - active_screen, - }], - }, - result, - deltas, - move |state| { - state.workspaces[index].name = name; - state.workspace_revision = state.workspace_revision.saturating_add(1); - }, + #[cfg(test)] + pub(crate) fn resource_rename_workspace( + &self, + workspace_id: &WorkspacePublicId, + name: String, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + Self::validate_workspace_name(&name)?; + let fingerprint = serde_json::json!({ + "operation": "workspace.rename", + "workspace": workspace_id.as_str(), + "name": name, + }); + let target = workspace_id.clone(); + self.commit_resource_mutation_plan( + mutation, + "workspace.rename", + &fingerprint, + expected_generation, + expected_revision, + move |state, registry| { + let slot = *state + .resource_indexes + .workspaces + .get(&target) + .with_context(|| format!("unknown workspace {target}"))?; + let index = state + .workspace_index(slot) + .with_context(|| format!("workspace {target} has no live slot"))?; + let workspace = &state.workspaces[index]; + let changed = workspace.name != name; + let active_screen = workspace + .screens + .get(workspace.active_screen) + .map(|screen| screen.public_id.clone()); + let durable = RegistryWorkspace { + id: workspace.id, + public_id: workspace.public_id.clone(), + key: workspace.key.clone(), + name: name.clone(), + group_key: self.session.clone(), + }; + let result = serde_json::json!({ + "workspace": target.as_str(), + "name": name, + "changed": changed, + }); + let deltas = Value::Array(vec![workspace_resource_upsert( + 0, + registry.session_id().as_str(), + &target, + &name, + index, + index == state.active_workspace, + )]); + Ok(ResourceMutationPlan::new( + ResourcePatch { + changes: vec![ResourceChange::UpsertWorkspace { + workspace: durable, + position: index, + active_screen, + }], + }, + result, + deltas, + move |state| { + state.workspaces[index].name = name; + state.workspace_revision = state.workspace_revision.saturating_add(1); + }, + ) + .with_metrics(ResourceMutationMetrics { + touched_resources: 1, + order_entries: 0, + terminal_queries: 0, + changed_rows: 1, + })) + }, + ) + } + + pub(crate) fn resource_rename_workspace_selected( + &self, + selectors: crate::ResourceSelectors, + name: String, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + Self::validate_workspace_name(&name)?; + let fingerprint = serde_json::json!({ + "operation": "workspace.rename", + "selectors": selectors, + "name": name, + }); + self.commit_resource_mutation_plan( + mutation, + "workspace.rename", + &fingerprint, + expected_generation, + expected_revision, + move |state, registry| { + let resolved = self + .resolve_resource_path_in_state( + state, + registry, + crate::ResourceTarget::Workspace, + &selectors, ) - .with_metrics(ResourceMutationMetrics { - touched_resources: 1, - order_entries: 0, - terminal_queries: 0, - changed_rows: 1, - })) - }, - ) - } + .map_err(anyhow::Error::new)?; + let target = resolved + .path + .workspace + .expect("workspace target resolution returns a workspace"); + let slot = + resolved.workspace.expect("workspace target resolution returns a live slot"); + let index = state + .workspace_index(slot) + .with_context(|| format!("workspace {target} has no live slot"))?; + #[cfg(test)] + if let Some(hook) = + self.resource_rename_after_selector_resolution.lock().unwrap().clone() + { + hook(&target); + } + let workspace = &state.workspaces[index]; + let changed = workspace.name != name; + let active_screen = workspace + .screens + .get(workspace.active_screen) + .map(|screen| screen.public_id.clone()); + let durable = RegistryWorkspace { + id: workspace.id, + public_id: workspace.public_id.clone(), + key: workspace.key.clone(), + name: name.clone(), + group_key: self.session.clone(), + }; + let result = serde_json::json!({ + "workspace": target.as_str(), + "name": name, + "changed": changed, + }); + let deltas = Value::Array(vec![workspace_resource_upsert( + 0, + registry.session_id().as_str(), + &target, + &name, + index, + index == state.active_workspace, + )]); + Ok(ResourceMutationPlan::new( + ResourcePatch { + changes: vec![ResourceChange::UpsertWorkspace { + workspace: durable, + position: index, + active_screen, + }], + }, + result, + deltas, + move |state| { + state.workspaces[index].name = name; + state.workspace_revision = state.workspace_revision.saturating_add(1); + }, + ) + .with_metrics(ResourceMutationMetrics { + touched_resources: 1, + order_entries: 0, + terminal_queries: 0, + changed_rows: 1, + })) + }, + ) + } - #[cfg(test)] - pub(crate) fn resource_move_workspace( - &self, - workspace_id: &WorkspacePublicId, - index: usize, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "operation": "workspace.move", - "workspace": workspace_id.as_str(), - "index": index, - }); - let target = workspace_id.clone(); - self.commit_resource_mutation_plan( - mutation, - "workspace.move", - &fingerprint, - expected_generation, - expected_revision, - move |state, registry| { - let slot = *state - .resource_indexes - .workspaces - .get(&target) - .with_context(|| format!("unknown workspace {target}"))?; - let old_index = state - .workspace_index(slot) - .with_context(|| format!("workspace {target} has no live slot"))?; - let new_index = index.min(state.workspaces.len().saturating_sub(1)); - let changed = new_index != old_index; - let active_slot = - state.workspaces.get(state.active_workspace).map(|workspace| workspace.id); - let mut order = state - .workspaces + #[cfg(test)] + pub(crate) fn resource_move_workspace( + &self, + workspace_id: &WorkspacePublicId, + index: usize, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + let fingerprint = serde_json::json!({ + "operation": "workspace.move", + "workspace": workspace_id.as_str(), + "index": index, + }); + let target = workspace_id.clone(); + self.commit_resource_mutation_plan( + mutation, + "workspace.move", + &fingerprint, + expected_generation, + expected_revision, + move |state, registry| { + let slot = *state + .resource_indexes + .workspaces + .get(&target) + .with_context(|| format!("unknown workspace {target}"))?; + let old_index = state + .workspace_index(slot) + .with_context(|| format!("workspace {target} has no live slot"))?; + let new_index = index.min(state.workspaces.len().saturating_sub(1)); + let changed = new_index != old_index; + let active_slot = + state.workspaces.get(state.active_workspace).map(|workspace| workspace.id); + let mut order = state + .workspaces + .iter() + .map(|workspace| workspace.public_id.clone()) + .collect::>(); + if changed { + let moved = order.remove(old_index); + order.insert(new_index, moved); + } + let changes = if changed { + vec![ResourceChange::SetWorkspaceOrder { workspace_ids: order.clone() }] + } else { + let workspace = &state.workspaces[old_index]; + vec![ResourceChange::UpsertWorkspace { + workspace: RegistryWorkspace { + id: workspace.id, + public_id: workspace.public_id.clone(), + key: workspace.key.clone(), + name: workspace.name.clone(), + group_key: self.session.clone(), + }, + position: old_index, + active_screen: workspace + .screens + .get(workspace.active_screen) + .map(|screen| screen.public_id.clone()), + }] + }; + let result = serde_json::json!({ + "workspace": target.as_str(), + "index": new_index, + "changed": changed, + }); + let deltas = Value::Array( + order .iter() - .map(|workspace| workspace.public_id.clone()) - .collect::>(); - if changed { - let moved = order.remove(old_index); - order.insert(new_index, moved); - } - let changes = if changed { - vec![ResourceChange::SetWorkspaceOrder { workspace_ids: order.clone() }] - } else { - let workspace = &state.workspaces[old_index]; - vec![ResourceChange::UpsertWorkspace { - workspace: RegistryWorkspace { - id: workspace.id, - public_id: workspace.public_id.clone(), - key: workspace.key.clone(), - name: workspace.name.clone(), - group_key: self.session.clone(), - }, - position: old_index, - active_screen: workspace - .screens - .get(workspace.active_screen) - .map(|screen| screen.public_id.clone()), - }] - }; - let result = serde_json::json!({ - "workspace": target.as_str(), - "index": new_index, - "changed": changed, - }); - let deltas = Value::Array( - order - .iter() - .enumerate() - .map(|(position, workspace_id)| { - let workspace = state - .workspaces - .iter() - .find(|workspace| &workspace.public_id == workspace_id) - .expect("workspace order was built from live workspaces"); - workspace_resource_upsert( - position, - registry.session_id().as_str(), - workspace_id, - &workspace.name, - position, - active_slot == Some(workspace.id), - ) - }) - .collect(), - ); - let order_entries = usize::from(changed) * order.len(); - Ok(ResourceMutationPlan::new( - ResourcePatch { changes }, - result, - deltas, - move |state| { - if changed { - state.move_workspace(old_index, new_index); - for (workspace_index, _, _) in state.split_screens.values_mut() { - *workspace_index = if *workspace_index == old_index { - new_index - } else if old_index < new_index - && (old_index + 1..=new_index).contains(workspace_index) - { - workspace_index.saturating_sub(1) - } else if new_index < old_index - && (new_index..old_index).contains(workspace_index) - { - workspace_index.saturating_add(1) - } else { - *workspace_index - }; - } - state.active_workspace = active_slot - .and_then(|slot| state.workspace_index(slot)) - .unwrap_or_else(|| state.workspaces.len().saturating_sub(1)); + .enumerate() + .map(|(position, workspace_id)| { + let workspace = state + .workspaces + .iter() + .find(|workspace| &workspace.public_id == workspace_id) + .expect("workspace order was built from live workspaces"); + workspace_resource_upsert( + position, + registry.session_id().as_str(), + workspace_id, + &workspace.name, + position, + active_slot == Some(workspace.id), + ) + }) + .collect(), + ); + let order_entries = usize::from(changed) * order.len(); + Ok(ResourceMutationPlan::new( + ResourcePatch { changes }, + result, + deltas, + move |state| { + if changed { + state.move_workspace(old_index, new_index); + for (workspace_index, _, _) in state.split_screens.values_mut() { + *workspace_index = if *workspace_index == old_index { + new_index + } else if old_index < new_index + && (old_index + 1..=new_index).contains(workspace_index) + { + workspace_index.saturating_sub(1) + } else if new_index < old_index + && (new_index..old_index).contains(workspace_index) + { + workspace_index.saturating_add(1) + } else { + *workspace_index + }; } - state.workspace_revision = state.workspace_revision.saturating_add(1); - }, - ) - .with_metrics(ResourceMutationMetrics { - touched_resources: 1, - order_entries, - terminal_queries: 0, - changed_rows: if changed { order.len() } else { 1 }, - })) - }, - ) - } + state.active_workspace = active_slot + .and_then(|slot| state.workspace_index(slot)) + .unwrap_or_else(|| state.workspaces.len().saturating_sub(1)); + } + state.workspace_revision = state.workspace_revision.saturating_add(1); + }, + ) + .with_metrics(ResourceMutationMetrics { + touched_resources: 1, + order_entries, + terminal_queries: 0, + changed_rows: if changed { order.len() } else { 1 }, + })) + }, + ) + } - pub(crate) fn resource_move_workspace_selected( - &self, - selectors: crate::ResourceSelectors, - index: usize, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "operation": "workspace.move", - "selectors": selectors, - "index": index, - }); - self.commit_resource_mutation_plan( - mutation, - "workspace.move", - &fingerprint, - expected_generation, - expected_revision, - move |state, registry| { - let resolved = self - .resolve_resource_path_in_state( - state, - registry, - crate::ResourceTarget::Workspace, - &selectors, - ) - .map_err(anyhow::Error::new)?; - let target = resolved - .path - .workspace - .expect("workspace target resolution returns a workspace"); - let slot = resolved - .workspace - .expect("workspace target resolution returns a live slot"); - let old_index = state - .workspace_index(slot) - .with_context(|| format!("workspace {target} has no live slot"))?; - let new_index = index.min(state.workspaces.len().saturating_sub(1)); - let changed = new_index != old_index; - let active_slot = - state.workspaces.get(state.active_workspace).map(|workspace| workspace.id); - let mut order = state - .workspaces + pub(crate) fn resource_move_workspace_selected( + &self, + selectors: crate::ResourceSelectors, + index: usize, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + let fingerprint = serde_json::json!({ + "operation": "workspace.move", + "selectors": selectors, + "index": index, + }); + self.commit_resource_mutation_plan( + mutation, + "workspace.move", + &fingerprint, + expected_generation, + expected_revision, + move |state, registry| { + let resolved = self + .resolve_resource_path_in_state( + state, + registry, + crate::ResourceTarget::Workspace, + &selectors, + ) + .map_err(anyhow::Error::new)?; + let target = resolved + .path + .workspace + .expect("workspace target resolution returns a workspace"); + let slot = + resolved.workspace.expect("workspace target resolution returns a live slot"); + let old_index = state + .workspace_index(slot) + .with_context(|| format!("workspace {target} has no live slot"))?; + let new_index = index.min(state.workspaces.len().saturating_sub(1)); + let changed = new_index != old_index; + let active_slot = + state.workspaces.get(state.active_workspace).map(|workspace| workspace.id); + let mut order = state + .workspaces + .iter() + .map(|workspace| workspace.public_id.clone()) + .collect::>(); + if changed { + let moved = order.remove(old_index); + order.insert(new_index, moved); + } + let changes = if changed { + vec![ResourceChange::SetWorkspaceOrder { workspace_ids: order.clone() }] + } else { + let workspace = &state.workspaces[old_index]; + vec![ResourceChange::UpsertWorkspace { + workspace: RegistryWorkspace { + id: workspace.id, + public_id: workspace.public_id.clone(), + key: workspace.key.clone(), + name: workspace.name.clone(), + group_key: self.session.clone(), + }, + position: old_index, + active_screen: workspace + .screens + .get(workspace.active_screen) + .map(|screen| screen.public_id.clone()), + }] + }; + let result = serde_json::json!({ + "workspace": target.as_str(), + "index": new_index, + "changed": changed, + }); + let deltas = Value::Array( + order .iter() - .map(|workspace| workspace.public_id.clone()) - .collect::>(); - if changed { - let moved = order.remove(old_index); - order.insert(new_index, moved); - } - let changes = if changed { - vec![ResourceChange::SetWorkspaceOrder { workspace_ids: order.clone() }] - } else { - let workspace = &state.workspaces[old_index]; - vec![ResourceChange::UpsertWorkspace { - workspace: RegistryWorkspace { - id: workspace.id, - public_id: workspace.public_id.clone(), - key: workspace.key.clone(), - name: workspace.name.clone(), - group_key: self.session.clone(), - }, - position: old_index, - active_screen: workspace - .screens - .get(workspace.active_screen) - .map(|screen| screen.public_id.clone()), - }] - }; - let result = serde_json::json!({ - "workspace": target.as_str(), - "index": new_index, - "changed": changed, - }); - let deltas = Value::Array( - order - .iter() - .enumerate() - .map(|(position, workspace_id)| { - let workspace = state - .workspaces - .iter() - .find(|workspace| &workspace.public_id == workspace_id) - .expect("workspace order was built from live workspaces"); - workspace_resource_upsert( - position, - registry.session_id().as_str(), - workspace_id, - &workspace.name, - position, - active_slot == Some(workspace.id), - ) - }) - .collect(), - ); - let order_entries = usize::from(changed) * order.len(); - Ok(ResourceMutationPlan::new( - ResourcePatch { changes }, - result, - deltas, - move |state| { - if changed { - state.move_workspace(old_index, new_index); - for (workspace_index, _, _) in state.split_screens.values_mut() { - *workspace_index = if *workspace_index == old_index { - new_index - } else if old_index < new_index - && (old_index + 1..=new_index).contains(workspace_index) - { - workspace_index.saturating_sub(1) - } else if new_index < old_index - && (new_index..old_index).contains(workspace_index) - { - workspace_index.saturating_add(1) - } else { - *workspace_index - }; - } - state.active_workspace = active_slot - .and_then(|slot| state.workspace_index(slot)) - .unwrap_or_else(|| state.workspaces.len().saturating_sub(1)); + .enumerate() + .map(|(position, workspace_id)| { + let workspace = state + .workspaces + .iter() + .find(|workspace| &workspace.public_id == workspace_id) + .expect("workspace order was built from live workspaces"); + workspace_resource_upsert( + position, + registry.session_id().as_str(), + workspace_id, + &workspace.name, + position, + active_slot == Some(workspace.id), + ) + }) + .collect(), + ); + let order_entries = usize::from(changed) * order.len(); + Ok(ResourceMutationPlan::new( + ResourcePatch { changes }, + result, + deltas, + move |state| { + if changed { + state.move_workspace(old_index, new_index); + for (workspace_index, _, _) in state.split_screens.values_mut() { + *workspace_index = if *workspace_index == old_index { + new_index + } else if old_index < new_index + && (old_index + 1..=new_index).contains(workspace_index) + { + workspace_index.saturating_sub(1) + } else if new_index < old_index + && (new_index..old_index).contains(workspace_index) + { + workspace_index.saturating_add(1) + } else { + *workspace_index + }; } - state.workspace_revision = state.workspace_revision.saturating_add(1); - }, - ) - .with_metrics(ResourceMutationMetrics { - touched_resources: 1, - order_entries, - terminal_queries: 0, - changed_rows: if changed { order.len() } else { 1 }, - })) - }, - ) - } + state.active_workspace = active_slot + .and_then(|slot| state.workspace_index(slot)) + .unwrap_or_else(|| state.workspaces.len().saturating_sub(1)); + } + state.workspace_revision = state.workspace_revision.saturating_add(1); + }, + ) + .with_metrics(ResourceMutationMetrics { + touched_resources: 1, + order_entries, + terminal_queries: 0, + changed_rows: if changed { order.len() } else { 1 }, + })) + }, + ) + } - pub fn registry_identity(&self) -> (String, String) { - let registry = self.workspace_registry.lock().unwrap(); - (registry.registry_id().to_string(), registry.generation().to_string()) - } - - pub fn install_resource_machine_service( - &self, - service: Arc, - ) -> anyhow::Result<()> { - self.resource_machine_service - .set(service) - .map_err(|_| anyhow::anyhow!("resource machine service is already installed")) - } - - pub(crate) fn resource_machine_service( - self: &Arc, - ) -> Arc { - self.resource_machine_service - .get_or_init(|| { - Arc::new(crate::resource_api::LocalResourceMachineService::new(Arc::downgrade( - self, - ))) - }) - .clone() - } + pub fn registry_identity(&self) -> (String, String) { + let registry = self.workspace_registry.lock().unwrap(); + (registry.registry_id().to_string(), registry.generation().to_string()) + } + + pub fn install_resource_machine_service( + &self, + service: Arc, + ) -> anyhow::Result<()> { + self.resource_machine_service + .set(service) + .map_err(|_| anyhow::anyhow!("resource machine service is already installed")) + } - pub(crate) fn local_resource_context( - &self, - ) -> anyhow::Result { - let registry = self.workspace_registry.lock().unwrap(); - let topology = registry.resource_topology_snapshot()?; - Ok(crate::resource_api::LocalResourceContext { - machine_id: registry.machine_id().clone(), - session_id: registry.session_id().clone(), - session_name: self.session.clone(), - generation: topology.generation, - revision: topology.revision, + pub(crate) fn resource_machine_service( + self: &Arc, + ) -> Arc { + self.resource_machine_service + .get_or_init(|| { + Arc::new(crate::resource_api::LocalResourceMachineService::new(Arc::downgrade( + self, + ))) }) - } + .clone() + } - pub(crate) fn with_resource_projection( - &self, - project: impl FnOnce(&WorkspaceRegistry, &State) -> anyhow::Result, - ) -> anyhow::Result { - let registry = self.workspace_registry.lock().unwrap(); - let state = self.state.lock().unwrap(); - project(®istry, &state) - } + pub(crate) fn local_resource_context( + &self, + ) -> anyhow::Result { + let registry = self.workspace_registry.lock().unwrap(); + let topology = registry.resource_topology_snapshot()?; + Ok(crate::resource_api::LocalResourceContext { + machine_id: registry.machine_id().clone(), + session_id: registry.session_id().clone(), + session_name: self.session.clone(), + generation: topology.generation, + revision: topology.revision, + }) + } - pub(crate) fn lookup_resource_effect( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().lookup_resource_effect( - idempotency_key, - operation, - fingerprint, - ) - } + pub(crate) fn with_resource_projection( + &self, + project: impl FnOnce(&WorkspaceRegistry, &State) -> anyhow::Result, + ) -> anyhow::Result { + let registry = self.workspace_registry.lock().unwrap(); + let state = self.state.lock().unwrap(); + project(®istry, &state) + } - pub(crate) fn resource_input_receipt_hmac( - &self, - idempotency_key: &str, - operation: &str, - canonical_fields: &[u8], - ) -> [u8; 32] { - self.workspace_registry.lock().unwrap().resource_input_receipt_hmac( - idempotency_key, - operation, - canonical_fields, - ) - } + pub(crate) fn lookup_resource_effect( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().lookup_resource_effect( + idempotency_key, + operation, + fingerprint, + ) + } - #[allow(clippy::too_many_arguments)] - pub(crate) fn prepare_resource_effect( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - intent: &Value, - expected_generation: Option<&str>, - expected_revision: Option, - ) -> anyhow::Result { - self.workspace_registry.lock().unwrap().prepare_resource_effect( - idempotency_key, - operation, - fingerprint, - intent, - expected_generation, - expected_revision, - ) - } + pub(crate) fn resource_input_receipt_hmac( + &self, + idempotency_key: &str, + operation: &str, + canonical_fields: &[u8], + ) -> [u8; 32] { + self.workspace_registry.lock().unwrap().resource_input_receipt_hmac( + idempotency_key, + operation, + canonical_fields, + ) + } - pub(crate) fn mark_resource_effect_executing( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - ) -> anyhow::Result { - self.workspace_registry.lock().unwrap().mark_resource_effect_executing( - idempotency_key, - operation, - fingerprint, - ) - } + #[allow(clippy::too_many_arguments)] + pub(crate) fn prepare_resource_effect( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + intent: &Value, + expected_generation: Option<&str>, + expected_revision: Option, + ) -> anyhow::Result { + self.workspace_registry.lock().unwrap().prepare_resource_effect( + idempotency_key, + operation, + fingerprint, + intent, + expected_generation, + expected_revision, + ) + } - pub(crate) fn commit_resource_effect( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - outcome: &ResourceEffectOutcome, - deltas: Option<&Value>, - ) -> anyhow::Result { - let mut registry = self.workspace_registry.lock().unwrap(); - let revision = registry.commit_resource_effect( - idempotency_key, - operation, - fingerprint, - outcome, - deltas, - )?; - if deltas.is_some() { - self.state.lock().unwrap().resource_revision = revision; - drop(registry); - self.publish_resource_event(); - } else { - drop(registry); - self.publish_journal_event(); - } - Ok(revision) - } - - /// Capture a post-effect live projection and commit its topology, public - /// deltas, and effect receipt while holding one registry -> state writer - /// fence. This prevents another topology writer from landing between the - /// captured tree and its durable revision. - pub(crate) fn commit_resource_effect_projection( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - project: impl FnOnce( - &WorkspaceRegistry, - &mut State, - ) -> anyhow::Result, - ) -> anyhow::Result { - let mut registry = self.workspace_registry.lock().unwrap(); - let mut state = self.state.lock().unwrap(); - let mut projection = project(®istry, &mut state)?; - persist_public_topology_result(operation, &mut projection.result, &projection.changes)?; - #[cfg(test)] - if let Some(hook) = self.resource_projection_before_commit.lock().unwrap().clone() { - hook(); - } - let commit = registry.commit_resource_effect_patch( - idempotency_key, - operation, - fingerprint, - &projection.patch, - &projection.result, - &projection.changes, - )?; - state.resource_revision = commit.revision; - drop(state); + pub(crate) fn mark_resource_effect_executing( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + ) -> anyhow::Result { + self.workspace_registry.lock().unwrap().mark_resource_effect_executing( + idempotency_key, + operation, + fingerprint, + ) + } + + pub(crate) fn commit_resource_effect( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + outcome: &ResourceEffectOutcome, + deltas: Option<&Value>, + ) -> anyhow::Result { + let mut registry = self.workspace_registry.lock().unwrap(); + let revision = registry.commit_resource_effect( + idempotency_key, + operation, + fingerprint, + outcome, + deltas, + )?; + if deltas.is_some() { + self.state.lock().unwrap().resource_revision = revision; drop(registry); self.publish_resource_event(); - Ok(commit) + } else { + drop(registry); + self.publish_journal_event(); } + Ok(revision) + } - pub(crate) fn commit_full_resource_effect_projection( - &self, - idempotency_key: &str, - operation: &str, - fingerprint: &Value, - result: Value, - ) -> anyhow::Result { - self.commit_resource_effect_projection( - idempotency_key, - operation, - fingerprint, - |registry, state| self.resource_effect_projection_locked(registry, state, result), - ) + /// Capture a post-effect live projection and commit its topology, public + /// deltas, and effect receipt while holding one registry -> state writer + /// fence. This prevents another topology writer from landing between the + /// captured tree and its durable revision. + pub(crate) fn commit_resource_effect_projection( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + project: impl FnOnce(&WorkspaceRegistry, &mut State) -> anyhow::Result, + ) -> anyhow::Result { + let mut registry = self.workspace_registry.lock().unwrap(); + let mut state = self.state.lock().unwrap(); + let mut projection = project(®istry, &mut state)?; + persist_public_topology_result(operation, &mut projection.result, &projection.changes)?; + #[cfg(test)] + if let Some(hook) = self.resource_projection_before_commit.lock().unwrap().clone() { + hook(); } + let commit = registry.commit_resource_effect_patch( + idempotency_key, + operation, + fingerprint, + &projection.patch, + &projection.result, + &projection.changes, + )?; + state.resource_revision = commit.revision; + drop(state); + drop(registry); + self.publish_resource_event(); + Ok(commit) + } - /// Reconcile an already-committed local mutation into one public topology - /// revision. This is reserved for legacy/internal paths whose durable - /// side effect predates the resource coordinator. - fn commit_ordinary_full_resource_projection( - &self, - operation: &'static str, - result: Value, - ) -> anyhow::Result { - let mutation = WorkspaceMutation::local("cmux-tui"); - let fingerprint = serde_json::json!({"operation":operation,"result":result}); - self.commit_full_resource_projection_with_mutation( - &mutation, - operation, - &fingerprint, - result, - ) - } + pub(crate) fn commit_full_resource_effect_projection( + &self, + idempotency_key: &str, + operation: &str, + fingerprint: &Value, + result: Value, + ) -> anyhow::Result { + self.commit_resource_effect_projection( + idempotency_key, + operation, + fingerprint, + |registry, state| self.resource_effect_projection_locked(registry, state, result), + ) + } - pub(crate) fn commit_full_resource_projection_with_mutation( - &self, - mutation: &WorkspaceMutation, - operation: &str, - fingerprint: &Value, - result: Value, - ) -> anyhow::Result { - self.commit_resource_mutation_plan( - mutation, - operation, - fingerprint, - None, - None, - |state, registry| { - let projection = - self.resource_effect_projection_locked(registry, state, result)?; - Ok(ResourceMutationPlan::new( - projection.patch, - projection.result, - projection.changes, - |_| {}, - )) - }, - ) - } + /// Reconcile an already-committed local mutation into one public topology + /// revision. This is reserved for legacy/internal paths whose durable + /// side effect predates the resource coordinator. + fn commit_ordinary_full_resource_projection( + &self, + operation: &'static str, + result: Value, + ) -> anyhow::Result { + let mutation = WorkspaceMutation::local("cmux-tui"); + let fingerprint = serde_json::json!({"operation":operation,"result":result}); + self.commit_full_resource_projection_with_mutation( + &mutation, + operation, + &fingerprint, + result, + ) + } - pub(crate) fn mark_resource_effect_indeterminate( - &self, - idempotency_key: &str, - ) -> anyhow::Result<()> { - self.workspace_registry - .lock() - .unwrap() - .mark_resource_effect_indeterminate(idempotency_key) + pub(crate) fn commit_full_resource_projection_with_mutation( + &self, + mutation: &WorkspaceMutation, + operation: &str, + fingerprint: &Value, + result: Value, + ) -> anyhow::Result { + self.commit_resource_mutation_plan( + mutation, + operation, + fingerprint, + None, + None, + |state, registry| { + let projection = self.resource_effect_projection_locked(registry, state, result)?; + Ok(ResourceMutationPlan::new( + projection.patch, + projection.result, + projection.changes, + |_| {}, + )) + }, + ) + } + + pub(crate) fn mark_resource_effect_indeterminate( + &self, + idempotency_key: &str, + ) -> anyhow::Result<()> { + self.workspace_registry.lock().unwrap().mark_resource_effect_indeterminate(idempotency_key) + } + + pub(crate) fn resource_surface_for_terminal( + &self, + terminal_id: &TerminalPublicId, + ) -> Option { + self.state.lock().unwrap().terminal_catalog.get(terminal_id).map(|surface| surface.id) + } + + pub(crate) fn resource_selectors_for_pane( + &self, + pane: Option, + ) -> anyhow::Result { + let state = self.state.lock().unwrap(); + let pane = pane.or_else(|| state.active_pane()).context("session has no active pane")?; + let (workspace_index, screen_index) = + state.screen_of(pane).context("pane has no containing screen")?; + let workspace = &state.workspaces[workspace_index]; + let screen = &workspace.screens[screen_index]; + let pane = + state.resource_indexes.pane_ids.get(&pane).context("pane has no public identity")?; + Ok(crate::ResourceSelectors { + machine: Some("current".to_string()), + session: Some("current".to_string()), + workspace: Some(workspace.public_id.to_string()), + screen: Some(screen.public_id.to_string()), + pane: Some(pane.to_string()), + ..crate::ResourceSelectors::default() + }) + } + + pub(crate) fn resource_selectors_for_workspace( + &self, + workspace: Option, + ) -> anyhow::Result { + let state = self.state.lock().unwrap(); + let workspace = match workspace { + Some(workspace) => state + .workspaces + .iter() + .find(|candidate| candidate.id == workspace) + .context("workspace does not exist")?, + None => state + .workspaces + .get(state.active_workspace) + .context("session has no active workspace")?, + }; + Ok(crate::ResourceSelectors { + machine: Some("current".to_string()), + session: Some("current".to_string()), + workspace: Some(workspace.public_id.to_string()), + ..crate::ResourceSelectors::default() + }) + } + + pub(crate) fn resource_surface_for_created_path( + &self, + result: &Value, + ) -> anyhow::Result { + let tab = TabPublicId::parse( + result["tab_id"] + .as_str() + .context("creation receipt omitted its tab identity")? + .to_string(), + ) + .map_err(anyhow::Error::new)?; + self.state + .lock() + .unwrap() + .resource_indexes + .tabs + .get(&tab) + .copied() + .context("created tab no longer has a live view") + } + + pub(crate) fn has_durable_terminal_receipt( + &self, + terminal_id: &TerminalPublicId, + ) -> anyhow::Result { + let registry = self.workspace_registry.lock().unwrap(); + let Some(host_id) = registry.terminal_host_id(terminal_id)? else { + return Ok(false); + }; + Ok(registry + .terminal_record(&host_id)? + .is_some_and(|terminal| terminal.lifecycle != TerminalLifecycle::Tombstoned)) + } + + /// Read only public terminal completion state. The host UUID and + /// incarnation remain an internal fencing mechanism and never enter the + /// resource API result. + pub(crate) fn terminal_exit_state( + &self, + terminal_id: &TerminalPublicId, + ) -> anyhow::Result { + #[cfg(test)] + let _query = TerminalExitStateQueryGuard(&self.terminal_exit_state_queries); + let registry = self.workspace_registry.lock().unwrap(); + let host_id = registry + .terminal_host_id(terminal_id)? + .ok_or_else(|| anyhow::anyhow!("terminal {terminal_id} is not live"))?; + let terminal = registry + .terminal_record(&host_id)? + .ok_or_else(|| anyhow::anyhow!("terminal {terminal_id} has no durable placement"))?; + if terminal.lifecycle == TerminalLifecycle::Exited { + let exit = terminal.exit.as_ref().and_then(Value::as_object).ok_or_else(|| { + anyhow::anyhow!("exited terminal {terminal_id} omitted exit metadata") + })?; + let outcome = exit.get("outcome").cloned().ok_or_else(|| { + anyhow::anyhow!("exited terminal {terminal_id} omitted its outcome") + })?; + let exited_at = exit.get("exited_at").and_then(Value::as_str).ok_or_else(|| { + anyhow::anyhow!("exited terminal {terminal_id} omitted exited_at") + })?; + let exit_revision = exit.get("revision").and_then(Value::as_str).ok_or_else(|| { + anyhow::anyhow!("exited terminal {terminal_id} omitted its revision") + })?; + return Ok(serde_json::json!({ + "state": "exited", + "terminal_id": terminal_id, + "lifecycle": "exited", + "outcome": outcome, + "exited_at": exited_at, + "revision": exit_revision, + })); } + let revision = registry.resource_revision()?; + let lifecycle = match terminal.lifecycle { + TerminalLifecycle::Launching | TerminalLifecycle::Adopting => "launching", + TerminalLifecycle::Running => "running", + TerminalLifecycle::Exited => "exited", + TerminalLifecycle::Tombstoned => { + return Err(ResourceError::terminal_closed(terminal_id).into()); + } + }; + Ok(serde_json::json!({ + "state": "pending", + "terminal_id": terminal_id, + "lifecycle": lifecycle, + "revision": revision.to_string(), + })) + } + + pub(crate) fn subscribe_terminal_exit( + &self, + terminal_id: &TerminalPublicId, + ) -> TerminalExitSubscription<'_> { + self.terminal_exit_waiters.subscribe(terminal_id) + } - pub(crate) fn resource_surface_for_terminal( - &self, - terminal_id: &TerminalPublicId, - ) -> Option { - self.state.lock().unwrap().terminal_catalog.get(terminal_id).map(|surface| surface.id) + fn terminal_public_ids_for_hosted( + registry: &WorkspaceRegistry, + hosted: &[(String, Option)], + ) -> anyhow::Result> { + let mut public_ids = Vec::with_capacity(hosted.len()); + let mut unique = HashSet::with_capacity(hosted.len()); + for (terminal_id, _) in hosted { + let is_tombstoned = registry + .terminal_record(terminal_id)? + .is_none_or(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned); + if is_tombstoned { + continue; + } + if let Some(public_id) = registry.terminal_resource_id(terminal_id)? + && unique.insert(public_id.clone()) + { + public_ids.push(public_id); + } } + Ok(public_ids) + } - pub(crate) fn resource_selectors_for_pane( - &self, - pane: Option, - ) -> anyhow::Result { - let state = self.state.lock().unwrap(); - let pane = - pane.or_else(|| state.active_pane()).context("session has no active pane")?; - let (workspace_index, screen_index) = - state.screen_of(pane).context("pane has no containing screen")?; - let workspace = &state.workspaces[workspace_index]; - let screen = &workspace.screens[screen_index]; - let pane = state - .resource_indexes - .pane_ids - .get(&pane) - .context("pane has no public identity")?; - Ok(crate::ResourceSelectors { - machine: Some("current".to_string()), - session: Some("current".to_string()), - workspace: Some(workspace.public_id.to_string()), - screen: Some(screen.public_id.to_string()), - pane: Some(pane.to_string()), - ..crate::ResourceSelectors::default() - }) + fn notify_terminal_exit_waiters( + &self, + terminal_ids: impl IntoIterator, + ) { + for terminal_id in terminal_ids { + self.terminal_exit_waiters.notify(&terminal_id); } + } - pub(crate) fn resource_selectors_for_workspace( - &self, - workspace: Option, - ) -> anyhow::Result { - let state = self.state.lock().unwrap(); - let workspace = match workspace { - Some(workspace) => state - .workspaces - .iter() - .find(|candidate| candidate.id == workspace) - .context("workspace does not exist")?, - None => state - .workspaces - .get(state.active_workspace) - .context("session has no active workspace")?, - }; - Ok(crate::ResourceSelectors { - machine: Some("current".to_string()), - session: Some("current".to_string()), - workspace: Some(workspace.public_id.to_string()), - ..crate::ResourceSelectors::default() + pub(crate) fn wait_for_terminal_exit( + &self, + terminal_id: &TerminalPublicId, + timeout: Option, + ) -> anyhow::Result { + let deadline = timeout + .map(|timeout| { + Instant::now() + .checked_add(timeout) + .ok_or_else(|| anyhow::anyhow!("terminal exit timeout exceeds deadline range")) }) + .transpose()?; + // Register before the initial query. A concurrent durable exit either + // appears in that query or wakes this exact terminal subscription. + let subscription = self.subscribe_terminal_exit(terminal_id); + let state = self.terminal_exit_state(terminal_id)?; + if state["state"] == "exited" || timeout == Some(Duration::ZERO) { + return Ok(state); } + let _explicit_wake = subscription.wait_until(deadline); + // One targeted read closes either the exit-notification or deadline + // race. Idle waits perform no periodic registry work. + self.terminal_exit_state(terminal_id) + } - pub(crate) fn resource_surface_for_created_path( - &self, - result: &Value, - ) -> anyhow::Result { - let tab = TabPublicId::parse( - result["tab_id"] - .as_str() - .context("creation receipt omitted its tab identity")? - .to_string(), - ) - .map_err(anyhow::Error::new)?; - self.state - .lock() - .unwrap() - .resource_indexes - .tabs - .get(&tab) - .copied() - .context("created tab no longer has a live view") - } + #[cfg(test)] + pub(crate) fn terminal_exit_waiter_count_for_test( + &self, + terminal_id: &TerminalPublicId, + ) -> usize { + self.terminal_exit_waiters.waiter_count(terminal_id) + } + + #[cfg(test)] + pub(crate) fn reset_terminal_exit_state_query_count_for_test(&self) { + self.terminal_exit_state_queries.store(0, Ordering::Release); + } + + #[cfg(test)] + pub(crate) fn terminal_exit_state_query_count_for_test(&self) -> u64 { + self.terminal_exit_state_queries.load(Ordering::Acquire) + } - pub(crate) fn has_durable_terminal_receipt( - &self, - terminal_id: &TerminalPublicId, - ) -> anyhow::Result { - let registry = self.workspace_registry.lock().unwrap(); - let Some(host_id) = registry.terminal_host_id(terminal_id)? else { - return Ok(false); - }; - Ok(registry - .terminal_record(&host_id)? - .is_some_and(|terminal| terminal.lifecycle != TerminalLifecycle::Tombstoned)) - } - - /// Read only public terminal completion state. The host UUID and - /// incarnation remain an internal fencing mechanism and never enter the - /// resource API result. - pub(crate) fn terminal_exit_state( - &self, - terminal_id: &TerminalPublicId, - ) -> anyhow::Result { - #[cfg(test)] - let _query = TerminalExitStateQueryGuard(&self.terminal_exit_state_queries); + #[cfg(test)] + pub(crate) fn persist_terminal_exit_for_test( + &self, + terminal_id: &TerminalPublicId, + exit: &TerminalExit, + ) -> anyhow::Result { + let (host_id, incarnation) = { let registry = self.workspace_registry.lock().unwrap(); let host_id = registry - .terminal_host_id(terminal_id)? - .ok_or_else(|| anyhow::anyhow!("terminal {terminal_id} is not live"))?; - let terminal = registry.terminal_record(&host_id)?.ok_or_else(|| { - anyhow::anyhow!("terminal {terminal_id} has no durable placement") - })?; - if terminal.lifecycle == TerminalLifecycle::Exited { - let exit = terminal.exit.as_ref().and_then(Value::as_object).ok_or_else(|| { - anyhow::anyhow!("exited terminal {terminal_id} omitted exit metadata") - })?; - let outcome = exit.get("outcome").cloned().ok_or_else(|| { - anyhow::anyhow!("exited terminal {terminal_id} omitted its outcome") - })?; - let exited_at = exit.get("exited_at").and_then(Value::as_str).ok_or_else(|| { - anyhow::anyhow!("exited terminal {terminal_id} omitted exited_at") - })?; - let exit_revision = - exit.get("revision").and_then(Value::as_str).ok_or_else(|| { - anyhow::anyhow!("exited terminal {terminal_id} omitted its revision") - })?; - return Ok(serde_json::json!({ - "state": "exited", - "terminal_id": terminal_id, - "lifecycle": "exited", - "outcome": outcome, - "exited_at": exited_at, - "revision": exit_revision, - })); - } - let revision = registry.resource_revision()?; - let lifecycle = match terminal.lifecycle { - TerminalLifecycle::Launching | TerminalLifecycle::Adopting => "launching", - TerminalLifecycle::Running => "running", - TerminalLifecycle::Exited => "exited", - TerminalLifecycle::Tombstoned => { - return Err(ResourceError::terminal_closed(terminal_id).into()); - } - }; - Ok(serde_json::json!({ - "state": "pending", - "terminal_id": terminal_id, - "lifecycle": lifecycle, - "revision": revision.to_string(), - })) - } + .live_terminal_host_id(terminal_id)? + .ok_or_else(|| anyhow::anyhow!("unknown terminal {terminal_id}"))?; + let incarnation = + registry.terminal_record(&host_id)?.and_then(|terminal| terminal.incarnation); + (host_id, incarnation) + }; + self.persist_terminal_exit(&host_id, incarnation.as_deref(), exit) + } - pub(crate) fn subscribe_terminal_exit( - &self, - terminal_id: &TerminalPublicId, - ) -> TerminalExitSubscription<'_> { - self.terminal_exit_waiters.subscribe(terminal_id) - } + fn publish_resource_event(&self) { + self.publish_journal_event(); + } - fn terminal_public_ids_for_hosted( - registry: &WorkspaceRegistry, - hosted: &[(String, Option)], - ) -> anyhow::Result> { - let mut public_ids = Vec::with_capacity(hosted.len()); - let mut unique = HashSet::with_capacity(hosted.len()); - for (terminal_id, _) in hosted { - let is_tombstoned = registry - .terminal_record(terminal_id)? - .is_none_or(|terminal| terminal.lifecycle == TerminalLifecycle::Tombstoned); - if is_tombstoned { - continue; - } - if let Some(public_id) = registry.terminal_resource_id(terminal_id)? - && unique.insert(public_id.clone()) - { - public_ids.push(public_id); - } - } - Ok(public_ids) + fn publish_journal_event(&self) { + self.journal_kernel.notify_commit(); + let mut epoch = self.journal_event_epoch.lock().unwrap(); + *epoch = epoch.wrapping_add(1); + self.journal_event_changed.notify_all(); + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn journal_terminal_output( + &self, + terminal_id: Arc, + generation: Arc, + bytes: Vec, + ) { + if bytes.is_empty() { + return; } + self.journal_ingress.send(crate::journal_ingress::JournalIngressEvent::TerminalOutput { + terminal_id, + generation, + occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), + bytes, + }); + } - fn notify_terminal_exit_waiters( - &self, - terminal_ids: impl IntoIterator, + pub(crate) fn try_journal_terminal_output( + &self, + terminal_id: Arc, + generation: Arc, + occurred_at_ms: u64, + bytes: Vec, + ) -> Result, u64)>, String> { + if bytes.is_empty() { + return Ok(None); + } + match self.journal_ingress.try_send( + crate::journal_ingress::JournalIngressEvent::TerminalOutput { + terminal_id, + generation, + occurred_at_ms, + bytes, + }, ) { - for terminal_id in terminal_ids { - self.terminal_exit_waiters.notify(&terminal_id); + Ok(()) => Ok(None), + Err(crate::journal_ingress::JournalIngressTrySendError::Full { + event, + space_epoch, + }) => Ok(match *event { + crate::journal_ingress::JournalIngressEvent::TerminalOutput { bytes, .. } => { + Some((bytes, space_epoch)) + } + _ => None, + }), + Err(crate::journal_ingress::JournalIngressTrySendError::Failed { event, error }) => { + debug_assert!(matches!( + *event, + crate::journal_ingress::JournalIngressEvent::TerminalOutput { .. } + )); + Err(error) } } + } - pub(crate) fn wait_for_terminal_exit( - &self, - terminal_id: &TerminalPublicId, - timeout: Option, - ) -> anyhow::Result { - let deadline = timeout - .map(|timeout| { - Instant::now().checked_add(timeout).ok_or_else(|| { - anyhow::anyhow!("terminal exit timeout exceeds deadline range") - }) - }) - .transpose()?; - // Register before the initial query. A concurrent durable exit either - // appears in that query or wakes this exact terminal subscription. - let subscription = self.subscribe_terminal_exit(terminal_id); - let state = self.terminal_exit_state(terminal_id)?; - if state["state"] == "exited" || timeout == Some(Duration::ZERO) { - return Ok(state); - } - let _explicit_wake = subscription.wait_until(deadline); - // One targeted read closes either the exit-notification or deadline - // race. Idle waits perform no periodic registry work. - self.terminal_exit_state(terminal_id) - } + pub(crate) fn wait_for_terminal_journal_space(&self, observed: u64) -> Result<(), String> { + self.journal_ingress.wait_for_queue_space(observed) + } - #[cfg(test)] - pub(crate) fn terminal_exit_waiter_count_for_test( - &self, - terminal_id: &TerminalPublicId, - ) -> usize { - self.terminal_exit_waiters.waiter_count(terminal_id) - } + pub(crate) fn flush_terminal_journal(&self) -> anyhow::Result<()> { + self.journal_ingress.flush_terminal() + } - #[cfg(test)] - pub(crate) fn reset_terminal_exit_state_query_count_for_test(&self) { - self.terminal_exit_state_queries.store(0, Ordering::Release); - } + pub(crate) fn install_journal_writer( + &self, + writer: crate::journal_ingress::JournalWriter, + ) -> anyhow::Result<()> { + self.journal_ingress.install_writer(writer) + } - #[cfg(test)] - pub(crate) fn terminal_exit_state_query_count_for_test(&self) -> u64 { - self.terminal_exit_state_queries.load(Ordering::Acquire) - } + #[cfg(test)] + pub(crate) fn install_journal_failure_notifier_for_test(&self, notifier: SyncSender) { + self.journal_ingress.install_failure_notifier_for_test(notifier); + } - #[cfg(test)] - pub(crate) fn persist_terminal_exit_for_test( - &self, - terminal_id: &TerminalPublicId, - exit: &TerminalExit, - ) -> anyhow::Result { - let (host_id, incarnation) = { - let registry = self.workspace_registry.lock().unwrap(); - let host_id = registry - .live_terminal_host_id(terminal_id)? - .ok_or_else(|| anyhow::anyhow!("unknown terminal {terminal_id}"))?; - let incarnation = - registry.terminal_record(&host_id)?.and_then(|terminal| terminal.incarnation); - (host_id, incarnation) - }; - self.persist_terminal_exit(&host_id, incarnation.as_deref(), exit) - } + #[cfg(test)] + pub(crate) fn install_journal_nonretryable_failure_hook_for_test( + &self, + entered: SyncSender<()>, + release: Receiver<()>, + ) { + self.journal_ingress.install_nonretryable_failure_hook_for_test(entered, release); + } - fn publish_resource_event(&self) { - self.publish_journal_event(); - } + pub(crate) fn terminal_journal_enabled(&self) -> bool { + self.journal_ingress.enabled() + } - fn publish_journal_event(&self) { - self.journal_kernel.notify_commit(); - let mut epoch = self.journal_event_epoch.lock().unwrap(); - *epoch = epoch.wrapping_add(1); - self.journal_event_changed.notify_all(); - } + pub fn journal_local_frontend_event( + &self, + event: crate::FrontendJournalEvent, + ) -> anyhow::Result<()> { + let principal_id = crate::server::public_client_id(&self.session_public_id, 0)?.to_string(); + self.journal_frontend_event(principal_id, event) + } - #[cfg_attr(not(test), allow(dead_code))] - pub(crate) fn journal_terminal_output( - &self, - terminal_id: Arc, - generation: Arc, - bytes: Vec, - ) { - if bytes.is_empty() { - return; - } - self.journal_ingress.send( - crate::journal_ingress::JournalIngressEvent::TerminalOutput { - terminal_id, - generation, - occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), - bytes, - }, - ); - } + pub(crate) fn session_public_id(&self) -> SessionPublicId { + self.session_public_id.clone() + } - pub(crate) fn try_journal_terminal_output( - &self, - terminal_id: Arc, - generation: Arc, - occurred_at_ms: u64, - bytes: Vec, - ) -> Result, u64)>, String> { - if bytes.is_empty() { - return Ok(None); - } - match self.journal_ingress.try_send( - crate::journal_ingress::JournalIngressEvent::TerminalOutput { - terminal_id, - generation, - occurred_at_ms, - bytes, - }, - ) { - Ok(()) => Ok(None), - Err(crate::journal_ingress::JournalIngressTrySendError::Full { - event, - space_epoch, - }) => Ok(match *event { - crate::journal_ingress::JournalIngressEvent::TerminalOutput { - bytes, .. - } => Some((bytes, space_epoch)), - _ => None, - }), - Err(crate::journal_ingress::JournalIngressTrySendError::Failed { - event, - error, - }) => { - debug_assert!(matches!( - *event, - crate::journal_ingress::JournalIngressEvent::TerminalOutput { .. } - )); - Err(error) - } - } - } + pub(crate) fn journal_frontend_event( + &self, + principal_id: String, + event: crate::FrontendJournalEvent, + ) -> anyhow::Result<()> { + self.journal_ingress.send_durable(crate::journal_ingress::JournalIngressEvent::Frontend { + principal_id, + occurred_at_ms: crate::workspace_registry::unix_epoch_ms()?, + event, + }) + } - pub(crate) fn wait_for_terminal_journal_space(&self, observed: u64) -> Result<(), String> { - self.journal_ingress.wait_for_queue_space(observed) - } + pub(crate) fn journal_terminal_resize( + &self, + terminal_id: Arc, + generation: Arc, + cols: u16, + rows: u16, + cell_width: u16, + cell_height: u16, + ) { + self.journal_ingress.send(crate::journal_ingress::JournalIngressEvent::TerminalResize { + terminal_id, + generation, + occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), + cols, + rows, + cell_width, + cell_height, + }); + } - pub(crate) fn flush_terminal_journal(&self) -> anyhow::Result<()> { - self.journal_ingress.flush_terminal() - } + pub(crate) fn commit_session_journal_events( + &self, + events: &[&crate::journal_ingress::JournalIngressEvent], + deadline: Instant, + sqlite_wait_cap: Duration, + admit_commit: F, + ) -> anyhow::Result>> + where + F: FnOnce() -> anyhow::Result<()>, + { + let mut registry = self + .workspace_registry + .lock_until(deadline) + .context("waiting for the workspace registry journal writer")?; + let remaining = deadline.saturating_duration_since(Instant::now()); + anyhow::ensure!(!remaining.is_zero(), "session journal commit deadline expired"); + let commits = registry.append_journal_ingress_events_with_deadline( + events, + deadline, + remaining.min(sqlite_wait_cap), + admit_commit, + )?; + self.publish_journal_event(); + Ok(commits) + } - pub(crate) fn install_journal_writer( - &self, - writer: crate::journal_ingress::JournalWriter, - ) -> anyhow::Result<()> { - self.journal_ingress.install_writer(writer) - } + #[cfg(test)] + pub(crate) fn hold_workspace_registry_for_test( + &self, + entered: SyncSender<()>, + release: Receiver<()>, + ) { + let _registry = self.workspace_registry.lock().unwrap(); + entered.send(()).unwrap(); + release.recv().unwrap(); + } - #[cfg(test)] - pub(crate) fn install_journal_failure_notifier_for_test( - &self, - notifier: SyncSender, - ) { - self.journal_ingress.install_failure_notifier_for_test(notifier); - } + #[cfg(test)] + pub(crate) fn install_journal_before_commit_for_test( + &self, + entered: SyncSender<()>, + release: Receiver<()>, + ) { + self.workspace_registry + .lock() + .unwrap() + .set_journal_before_commit_for_test(entered, release); + } - #[cfg(test)] - pub(crate) fn install_journal_nonretryable_failure_hook_for_test( - &self, - entered: SyncSender<()>, - release: Receiver<()>, - ) { - self.journal_ingress.install_nonretryable_failure_hook_for_test(entered, release); - } + #[cfg(test)] + pub(crate) fn install_journal_after_commit_admission_for_test( + &self, + entered: SyncSender<()>, + release: Receiver<()>, + ) { + self.workspace_registry + .lock() + .unwrap() + .set_journal_after_commit_admission_for_test(entered, release); + } - pub(crate) fn terminal_journal_enabled(&self) -> bool { - self.journal_ingress.enabled() - } + pub(crate) fn journal_event_epoch(&self) -> u64 { + *self.journal_event_epoch.lock().unwrap() + } - pub fn journal_local_frontend_event( - &self, - event: crate::FrontendJournalEvent, - ) -> anyhow::Result<()> { - let principal_id = - crate::server::public_client_id(&self.session_public_id, 0)?.to_string(); - self.journal_frontend_event(principal_id, event) + pub(crate) fn wait_for_journal_event(&self, epoch: u64, timeout: Duration) -> u64 { + let current = self.journal_event_epoch.lock().unwrap(); + if *current != epoch { + return *current; } + let (current, _) = self.journal_event_changed.wait_timeout(current, timeout).unwrap(); + *current + } - pub(crate) fn session_public_id(&self) -> SessionPublicId { - self.session_public_id.clone() - } + pub(crate) fn resource_event_epoch(&self) -> u64 { + self.journal_event_epoch() + } - pub(crate) fn journal_frontend_event( - &self, - principal_id: String, - event: crate::FrontendJournalEvent, - ) -> anyhow::Result<()> { - self.journal_ingress.send_durable( - crate::journal_ingress::JournalIngressEvent::Frontend { - principal_id, - occurred_at_ms: crate::workspace_registry::unix_epoch_ms()?, - event, - }, - ) - } + pub(crate) fn wait_for_resource_event(&self, epoch: u64, timeout: Duration) -> u64 { + self.wait_for_journal_event(epoch, timeout) + } - pub(crate) fn journal_terminal_resize( - &self, - terminal_id: Arc, - generation: Arc, - cols: u16, - rows: u16, - cell_width: u16, - cell_height: u16, - ) { - self.journal_ingress.send( - crate::journal_ingress::JournalIngressEvent::TerminalResize { - terminal_id, - generation, - occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), - cols, - rows, - cell_width, - cell_height, - }, - ); - } + pub(crate) fn resource_events_after( + &self, + revision: u64, + ) -> anyhow::Result { + self.workspace_registry.lock().unwrap().resource_events_after(revision) + } - pub(crate) fn commit_session_journal_events( - &self, - events: &[&crate::journal_ingress::JournalIngressEvent], - deadline: Instant, - sqlite_wait_cap: Duration, - admit_commit: F, - ) -> anyhow::Result>> - where - F: FnOnce() -> anyhow::Result<()>, - { - let mut registry = self - .workspace_registry - .lock_until(deadline) - .context("waiting for the workspace registry journal writer")?; - let remaining = deadline.saturating_duration_since(Instant::now()); - anyhow::ensure!(!remaining.is_zero(), "session journal commit deadline expired"); - let commits = registry.append_journal_ingress_events_with_deadline( - events, - deadline, - remaining.min(sqlite_wait_cap), - admit_commit, - )?; - self.publish_journal_event(); - Ok(commits) - } + pub(crate) fn session_journal_after( + &self, + sequence: u64, + limit: usize, + ) -> anyhow::Result { + self.workspace_registry.lock().unwrap().session_journal_after(sequence, limit) + } - #[cfg(test)] - pub(crate) fn hold_workspace_registry_for_test( - &self, - entered: SyncSender<()>, - release: Receiver<()>, - ) { - let _registry = self.workspace_registry.lock().unwrap(); - entered.send(()).unwrap(); - release.recv().unwrap(); - } + pub(crate) fn session_journal_reader( + &self, + ) -> anyhow::Result> { + let database_path = self.workspace_registry.lock().unwrap().session_journal_database_path(); + database_path + .as_deref() + .map(crate::workspace_registry::SessionJournalReader::open) + .transpose() + } - #[cfg(test)] - pub(crate) fn install_journal_before_commit_for_test( - &self, - entered: SyncSender<()>, - release: Receiver<()>, - ) { - self.workspace_registry - .lock() - .unwrap() - .set_journal_before_commit_for_test(entered, release); - } + pub(crate) fn shared_journal_enabled(&self) -> bool { + self.journal_kernel.enabled() + } - #[cfg(test)] - pub(crate) fn install_journal_after_commit_admission_for_test( - &self, - entered: SyncSender<()>, - release: Receiver<()>, - ) { - self.workspace_registry - .lock() - .unwrap() - .set_journal_after_commit_admission_for_test(entered, release); - } + pub(crate) fn shared_journal_epoch(&self) -> u64 { + self.journal_kernel.epoch() + } - pub(crate) fn journal_event_epoch(&self) -> u64 { - *self.journal_event_epoch.lock().unwrap() - } + pub(crate) fn shared_journal_handle(&self) -> Arc { + self.journal_kernel.clone() + } - pub(crate) fn wait_for_journal_event(&self, epoch: u64, timeout: Duration) -> u64 { - let current = self.journal_event_epoch.lock().unwrap(); - if *current != epoch { - return *current; - } - let (current, _) = self.journal_event_changed.wait_timeout(current, timeout).unwrap(); - *current - } + pub(crate) fn wait_for_shared_journal(&self, epoch: u64, timeout: Duration) -> u64 { + self.journal_kernel.wait(epoch, timeout) + } - pub(crate) fn resource_event_epoch(&self) -> u64 { - self.journal_event_epoch() - } + pub(crate) fn shared_journal_after( + &self, + sequence: u64, + limit: usize, + ) -> crate::journal_kernel::SharedJournalRead { + self.journal_kernel.read_after(sequence, limit) + } - pub(crate) fn wait_for_resource_event(&self, epoch: u64, timeout: Duration) -> u64 { - self.wait_for_journal_event(epoch, timeout) - } + pub(crate) fn journal_producer_manifests( + &self, + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().journal_producer_manifests() + } - pub(crate) fn resource_events_after( - &self, - revision: u64, - ) -> anyhow::Result { - self.workspace_registry.lock().unwrap().resource_events_after(revision) + pub(crate) fn put_journal_producer( + &self, + manifest: &crate::JournalProducerManifest, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + let prepared = crate::journal_kernel::JournalKernel::prepare_producer(manifest)?; + let commit = self.workspace_registry.lock().unwrap().put_journal_producer( + manifest, + origin, + idempotency_key, + )?; + if !commit.replayed { + // Installation is version-monotonic, so concurrent successful + // updates cannot publish their compiled validators out of order. + self.journal_kernel.install_prepared_producer(prepared); + self.publish_journal_event(); } + Ok(commit) + } - pub(crate) fn session_journal_after( - &self, - sequence: u64, - limit: usize, - ) -> anyhow::Result { - self.workspace_registry.lock().unwrap().session_journal_after(sequence, limit) + pub(crate) fn append_journal_ingress( + &self, + ingress: &crate::JournalIngress, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + let validated = self.journal_kernel.validate_ingress(ingress)?; + if self.journal_ingress.enabled() { + return self.journal_ingress.send_producer( + ingress.clone(), + validated, + origin.into(), + idempotency_key.into(), + ); } - - pub(crate) fn session_journal_reader( - &self, - ) -> anyhow::Result> { - let database_path = - self.workspace_registry.lock().unwrap().session_journal_database_path(); - database_path - .as_deref() - .map(crate::workspace_registry::SessionJournalReader::open) - .transpose() + let commit = self.workspace_registry.lock().unwrap().append_journal_ingress( + ingress, + &validated, + origin, + idempotency_key, + )?; + if !commit.replayed { + self.publish_journal_event(); } + Ok(commit) + } - pub(crate) fn shared_journal_enabled(&self) -> bool { - self.journal_kernel.enabled() - } + pub(crate) fn journal_hook_states( + &self, + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().journal_hook_states() + } + + pub(crate) fn journal_events_caused_by_hooks( + &self, + hook_ids: &[String], + event_ids: &[String], + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().journal_events_caused_by_hooks(hook_ids, event_ids) + } - pub(crate) fn shared_journal_epoch(&self) -> u64 { - self.journal_kernel.epoch() + pub(crate) fn put_journal_hook( + self: &Arc, + manifest: &crate::JournalHookManifest, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + let commit = self.workspace_registry.lock().unwrap().put_journal_hook( + manifest, + origin, + idempotency_key, + )?; + if !commit.replayed { + self.publish_journal_event(); } + crate::journal_hooks::start(self)?; + Ok(commit) + } + + pub(crate) fn try_claim_journal_hook_dispatcher(&self) -> bool { + self.journal_hook_dispatcher_started + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + pub(crate) fn journal_hook_runtime(&self) -> Arc { + self.journal_hook_runtime.clone() + } + + pub(crate) fn release_journal_hook_dispatcher(&self) { + self.journal_hook_dispatcher_started.store(false, Ordering::Release); + } + + pub(crate) fn schedule_journal_hook_deliveries( + &self, + scans: &[crate::workspace_registry::JournalHookScan], + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().schedule_journal_hook_deliveries(scans) + } + + pub(crate) fn pending_journal_hook_deliveries( + &self, + limit: usize, + ) -> anyhow::Result> { + let now_ms = crate::workspace_registry::unix_epoch_ms()?; + self.workspace_registry.lock().unwrap().pending_journal_hook_deliveries(now_ms, limit) + } - pub(crate) fn shared_journal_handle(&self) -> Arc { - self.journal_kernel.clone() + pub(crate) fn start_journal_hook_deliveries( + &self, + deliveries: &[crate::workspace_registry::JournalHookDelivery], + ) -> anyhow::Result> { + let attempts = + self.workspace_registry.lock().unwrap().start_journal_hook_deliveries(deliveries)?; + if !attempts.is_empty() { + self.publish_journal_event(); } + Ok(attempts) + } - pub(crate) fn wait_for_shared_journal(&self, epoch: u64, timeout: Duration) -> u64 { - self.journal_kernel.wait(epoch, timeout) + pub(crate) fn finish_journal_hook_deliveries( + &self, + results: &[crate::workspace_registry::JournalHookDeliveryResult], + ) -> anyhow::Result<()> { + self.workspace_registry.lock().unwrap().finish_journal_hook_deliveries(results)?; + if !results.is_empty() { + self.publish_journal_event(); } + Ok(()) + } - pub(crate) fn shared_journal_after( - &self, - sequence: u64, - limit: usize, - ) -> crate::journal_kernel::SharedJournalRead { - self.journal_kernel.read_after(sequence, limit) + pub(crate) fn create_journal_checkpoint( + &self, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + if let Some(commit) = self + .workspace_registry + .lock() + .unwrap() + .journal_checkpoint_receipt(origin, idempotency_key)? + { + return Ok(commit); + } + let captured = crate::journal_checkpoint::capture(self)?; + let commit = self.workspace_registry.lock().unwrap().create_journal_checkpoint( + captured.source_sequence, + crate::journal_checkpoint::JOURNAL_REDUCER_VERSION, + &captured.state, + &captured.blobs, + origin, + idempotency_key, + )?; + if !commit.journal.replayed { + self.publish_journal_event(); } + Ok(commit) + } - pub(crate) fn journal_producer_manifests( - &self, - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().journal_producer_manifests() - } + pub(crate) fn journal_checkpoints( + &self, + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().journal_checkpoints() + } - pub(crate) fn put_journal_producer( - &self, - manifest: &crate::JournalProducerManifest, - origin: &str, - idempotency_key: &str, - ) -> anyhow::Result { - let prepared = crate::journal_kernel::JournalKernel::prepare_producer(manifest)?; - let commit = self.workspace_registry.lock().unwrap().put_journal_producer( - manifest, - origin, - idempotency_key, - )?; - if !commit.replayed { - // Installation is version-monotonic, so concurrent successful - // updates cannot publish their compiled validators out of order. - self.journal_kernel.install_prepared_producer(prepared); - self.publish_journal_event(); - } - Ok(commit) - } - - pub(crate) fn append_journal_ingress( - &self, - ingress: &crate::JournalIngress, - origin: &str, - idempotency_key: &str, - ) -> anyhow::Result { - let validated = self.journal_kernel.validate_ingress(ingress)?; - if self.journal_ingress.enabled() { - return self.journal_ingress.send_producer( - ingress.clone(), - validated, - origin.into(), - idempotency_key.into(), - ); + pub(crate) fn journal_restore_preview(&self, selector: &str) -> anyhow::Result { + let checkpoint = self + .workspace_registry + .lock() + .unwrap() + .journal_checkpoint(selector)? + .with_context(|| format!("journal checkpoint {selector:?} does not exist"))?; + let mut reducer = crate::journal_checkpoint::RestoreReducer::new(&checkpoint)?; + let mut sequence = checkpoint.source_sequence; + let mut target_head = None; + let head_sequence = loop { + let page = self.session_journal_after(sequence, 1024)?; + let head = *target_head.get_or_insert(page.head_sequence); + let empty = page.records.is_empty(); + for record in page.records { + if record.sequence > head { + break; + } + sequence = record.sequence; + reducer.apply(&record)?; } - let commit = self.workspace_registry.lock().unwrap().append_journal_ingress( - ingress, - &validated, + if empty || sequence >= head { + break head; + } + }; + reducer.finish(head_sequence) + } + + pub(crate) fn journal_segments(&self) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().journal_segments() + } + + pub(crate) fn seal_journal_segments( + &self, + through_sequence: u64, + origin: &str, + idempotency_key: &str, + ) -> anyhow::Result { + let database_path = self + .workspace_registry + .lock() + .unwrap() + .session_journal_database_path() + .context("journal segment sealing requires a persistent session")?; + let reader = crate::workspace_registry::SessionJournalReader::open(&database_path)?; + for _ in 0..4 { + let start = self.workspace_registry.lock().unwrap().begin_journal_segment_seal( + through_sequence, origin, idempotency_key, )?; - if !commit.replayed { - self.publish_journal_event(); + let plan = match start { + crate::workspace_registry::JournalSegmentSealStart::Replay(commit) => { + return Ok(commit); + } + crate::workspace_registry::JournalSegmentSealStart::Prepare(plan) => plan, + }; + #[cfg(test)] + if let Some(hook) = self.journal_segment_prepare_hook.lock().unwrap().take() { + hook(); } - Ok(commit) - } - - pub(crate) fn journal_hook_states( - &self, - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().journal_hook_states() - } - - pub(crate) fn journal_events_caused_by_hooks( - &self, - hook_ids: &[String], - event_ids: &[String], - ) -> anyhow::Result> { - self.workspace_registry - .lock() - .unwrap() - .journal_events_caused_by_hooks(hook_ids, event_ids) - } - - pub(crate) fn put_journal_hook( - self: &Arc, - manifest: &crate::JournalHookManifest, - origin: &str, - idempotency_key: &str, - ) -> anyhow::Result { - let commit = self.workspace_registry.lock().unwrap().put_journal_hook( - manifest, + let prepared = plan.prepare(&reader)?; + let commit = self.workspace_registry.lock().unwrap().commit_journal_segment_seal( + prepared, origin, idempotency_key, )?; - if !commit.replayed { - self.publish_journal_event(); + if let Some(commit) = commit { + if !commit.journal.replayed { + self.publish_journal_event(); + } + return Ok(commit); } - crate::journal_hooks::start(self)?; - Ok(commit) - } - - pub(crate) fn try_claim_journal_hook_dispatcher(&self) -> bool { - self.journal_hook_dispatcher_started - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - - pub(crate) fn journal_hook_runtime(&self) -> Arc { - self.journal_hook_runtime.clone() } + anyhow::bail!("journal segment boundary changed repeatedly during sealing") + } - pub(crate) fn release_journal_hook_dispatcher(&self) { - self.journal_hook_dispatcher_started.store(false, Ordering::Release); - } + #[cfg(test)] + pub(crate) fn set_journal_segment_prepare_hook_for_test( + &self, + hook: impl FnOnce() + Send + 'static, + ) { + *self.journal_segment_prepare_hook.lock().unwrap() = Some(Box::new(hook)); + } - pub(crate) fn schedule_journal_hook_deliveries( - &self, - scans: &[crate::workspace_registry::JournalHookScan], - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().schedule_journal_hook_deliveries(scans) - } + #[cfg(test)] + pub(crate) fn journal_database_reader_count_for_test(&self) -> u64 { + self.journal_kernel.database_reader_count() + } - pub(crate) fn pending_journal_hook_deliveries( - &self, - limit: usize, - ) -> anyhow::Result> { - let now_ms = crate::workspace_registry::unix_epoch_ms()?; - self.workspace_registry.lock().unwrap().pending_journal_hook_deliveries(now_ms, limit) - } + #[cfg(test)] + pub(crate) fn resource_mutation_count_for_test(&self) -> anyhow::Result { + self.workspace_registry.lock().unwrap().resource_mutation_count_for_test() + } - pub(crate) fn start_journal_hook_deliveries( - &self, - deliveries: &[crate::workspace_registry::JournalHookDelivery], - ) -> anyhow::Result> { - let attempts = self - .workspace_registry - .lock() - .unwrap() - .start_journal_hook_deliveries(deliveries)?; - if !attempts.is_empty() { - self.publish_journal_event(); - } - Ok(attempts) - } + #[cfg(test)] + pub(crate) fn resource_agent_projection_count_for_test(&self) -> anyhow::Result { + self.workspace_registry.lock().unwrap().resource_agent_projection_count_for_test() + } - pub(crate) fn finish_journal_hook_deliveries( - &self, - results: &[crate::workspace_registry::JournalHookDeliveryResult], - ) -> anyhow::Result<()> { - self.workspace_registry.lock().unwrap().finish_journal_hook_deliveries(results)?; - if !results.is_empty() { - self.publish_journal_event(); - } - Ok(()) - } + #[cfg(test)] + pub(crate) fn corrupt_agent_projection_for_test(&self, terminal_id: &TerminalPublicId) { + self.workspace_registry.lock().unwrap().corrupt_agent_projection_for_test(terminal_id); + } - pub(crate) fn create_journal_checkpoint( - &self, - origin: &str, - idempotency_key: &str, - ) -> anyhow::Result { - if let Some(commit) = self - .workspace_registry - .lock() - .unwrap() - .journal_checkpoint_receipt(origin, idempotency_key)? - { - return Ok(commit); - } - let captured = crate::journal_checkpoint::capture(self)?; - let commit = self.workspace_registry.lock().unwrap().create_journal_checkpoint( - captured.source_sequence, - crate::journal_checkpoint::JOURNAL_REDUCER_VERSION, - &captured.state, - &captured.blobs, - origin, - idempotency_key, - )?; - if !commit.journal.replayed { - self.publish_journal_event(); - } - Ok(commit) - } + pub fn terminal_registry_snapshot(&self) -> anyhow::Result { + self.workspace_registry.lock().unwrap().terminal_snapshot() + } - pub(crate) fn journal_checkpoints( - &self, - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().journal_checkpoints() - } + pub fn terminal_registry_revision(&self) -> anyhow::Result { + self.workspace_registry.lock().unwrap().terminal_revision() + } - pub(crate) fn journal_restore_preview(&self, selector: &str) -> anyhow::Result { - let checkpoint = self - .workspace_registry - .lock() - .unwrap() - .journal_checkpoint(selector)? - .with_context(|| format!("journal checkpoint {selector:?} does not exist"))?; - let mut reducer = crate::journal_checkpoint::RestoreReducer::new(&checkpoint)?; - let mut sequence = checkpoint.source_sequence; - let mut target_head = None; - let head_sequence = loop { - let page = self.session_journal_after(sequence, 1024)?; - let head = *target_head.get_or_insert(page.head_sequence); - let empty = page.records.is_empty(); - for record in page.records { - if record.sequence > head { - break; - } - sequence = record.sequence; - reducer.apply(&record)?; - } - if empty || sequence >= head { - break head; - } - }; - reducer.finish(head_sequence) - } + #[cfg(test)] + pub(crate) fn reset_terminal_snapshot_count_for_test(&self) { + self.workspace_registry.lock().unwrap().reset_terminal_snapshot_count_for_test(); + } - pub(crate) fn journal_segments(&self) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().journal_segments() - } + #[cfg(test)] + pub(crate) fn terminal_snapshot_count_for_test(&self) -> usize { + self.workspace_registry.lock().unwrap().terminal_snapshot_count_for_test() + } - pub(crate) fn seal_journal_segments( - &self, - through_sequence: u64, - origin: &str, - idempotency_key: &str, - ) -> anyhow::Result { - let database_path = self - .workspace_registry - .lock() - .unwrap() - .session_journal_database_path() - .context("journal segment sealing requires a persistent session")?; - let reader = crate::workspace_registry::SessionJournalReader::open(&database_path)?; - for _ in 0..4 { - let start = self.workspace_registry.lock().unwrap().begin_journal_segment_seal( - through_sequence, - origin, - idempotency_key, - )?; - let plan = match start { - crate::workspace_registry::JournalSegmentSealStart::Replay(commit) => { - return Ok(commit); - } - crate::workspace_registry::JournalSegmentSealStart::Prepare(plan) => plan, - }; - #[cfg(test)] - if let Some(hook) = self.journal_segment_prepare_hook.lock().unwrap().take() { - hook(); - } - let prepared = plan.prepare(&reader)?; - let commit = self.workspace_registry.lock().unwrap().commit_journal_segment_seal( - prepared, - origin, - idempotency_key, - )?; - if let Some(commit) = commit { - if !commit.journal.replayed { - self.publish_journal_event(); - } - return Ok(commit); - } - } - anyhow::bail!("journal segment boundary changed repeatedly during sealing") - } + pub fn terminal_registry_events_page( + &self, + revision: u64, + ) -> anyhow::Result<( + TerminalRegistrySnapshot, + Vec, + )> { + // One writer guard is the read transaction boundary exposed to a + // frontend. Otherwise a commit between snapshot and event queries can + // return an event whose revision is newer than terminal_revision. + let registry = self.workspace_registry.lock().unwrap(); + let snapshot = registry.terminal_snapshot()?; + let events = registry.terminal_events_after(revision)?; + Ok((snapshot, events)) + } - #[cfg(test)] - pub(crate) fn set_journal_segment_prepare_hook_for_test( - &self, - hook: impl FnOnce() + Send + 'static, - ) { - *self.journal_segment_prepare_hook.lock().unwrap() = Some(Box::new(hook)); + pub fn workspace_registry_event( + &self, + revision: u64, + ) -> anyhow::Result> { + if revision == 0 { + return Ok(None); } + Ok(self + .workspace_registry + .lock() + .unwrap() + .events_after(revision - 1)? + .into_iter() + .find(|event| event.revision == revision)) + } - #[cfg(test)] - pub(crate) fn journal_database_reader_count_for_test(&self) -> u64 { - self.journal_kernel.database_reader_count() - } + pub fn get_frontend_projection( + &self, + frontend: &str, + scope: &str, + subject_key: &str, + ) -> anyhow::Result> { + self.workspace_registry.lock().unwrap().get_frontend_projection( + frontend, + scope, + subject_key, + ) + } - #[cfg(test)] - pub(crate) fn resource_mutation_count_for_test(&self) -> anyhow::Result { - self.workspace_registry.lock().unwrap().resource_mutation_count_for_test() + #[allow(clippy::too_many_arguments)] + pub fn put_frontend_projection( + &self, + mutation: &WorkspaceMutation, + frontend: &str, + scope: &str, + subject_key: &str, + schema_version: u32, + expected_projection_revision: Option, + projection: &Value, + ) -> anyhow::Result { + let mut registry = self.workspace_registry.lock().unwrap(); + let commit = registry.put_frontend_projection( + mutation, + frontend, + scope, + subject_key, + schema_version, + expected_projection_revision, + projection, + )?; + if !commit.replayed { + self.emit(MuxEvent::FrontendProjectionChanged { + frontend: frontend.to_string(), + scope: scope.to_string(), + subject_key: subject_key.to_string(), + projection_revision: commit.projection.projection_revision, + origin: mutation.origin.clone(), + mutation_id: mutation.id.clone(), + }); } + Ok(commit) + } - #[cfg(test)] - pub(crate) fn resource_agent_projection_count_for_test(&self) -> anyhow::Result { - self.workspace_registry.lock().unwrap().resource_agent_projection_count_for_test() + pub(crate) fn resource_put_frontend_projection_selected( + &self, + selectors: crate::ResourceSelectors, + projection_id: &FrontendProjectionPublicId, + projection: &Value, + expected_projection_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + let fingerprint = serde_json::json!({ + "operation":"frontend_projection.put", + "selectors":selectors, + "projection":projection, + }); + let mut registry = self.workspace_registry.lock().unwrap(); + if let Some(replay) = + registry.replay_resource_patch(mutation, "frontend_projection.put", &fingerprint)? + { + return Ok(replay); } - - #[cfg(test)] - pub(crate) fn corrupt_agent_projection_for_test(&self, terminal_id: &TerminalPublicId) { - self.workspace_registry.lock().unwrap().corrupt_agent_projection_for_test(terminal_id); + let projection_revision = registry + .get_frontend_projection("resource-api", "session", projection_id.as_str())? + .map(|projection| projection.projection_revision) + .unwrap_or(0) + .checked_add(1) + .context("frontend projection revision exhausted")?; + let mut session_selectors = selectors; + session_selectors.frontend_projection = None; + let mut state = self.state.lock().unwrap(); + let resolved = self + .resolve_resource_path_in_state( + &state, + ®istry, + crate::ResourceTarget::Session, + &session_selectors, + ) + .map_err(anyhow::Error::new)?; + let session_id = + resolved.path.session.context("projection route omitted its session identity")?; + let value = serde_json::json!({ + "id":projection_id, + "session_id":session_id, + "frontend_id":projection["frontend_id"], + "window_id":projection["window_id"], + "generation":projection["generation"], + "projection":projection["projection"], + "projection_revision":projection_revision.to_string(), + }); + let deltas = serde_json::json!([{ + "kind":"upsert", + "sequence":0, + "resource":"frontend_projection", + "id":projection_id, + "value":value, + }]); + let commit = registry.commit_resource_projection( + mutation, + "frontend_projection.put", + &fingerprint, + None, + expected_projection_revision, + "resource-api", + "session", + projection_id.as_str(), + RESOURCE_API_FRONTEND_PROJECTION_SCHEMA_VERSION, + projection, + &value, + &deltas, + )?; + state.resource_revision = commit.revision; + drop(state); + drop(registry); + if !commit.replayed { + self.publish_resource_event(); + self.emit(MuxEvent::FrontendProjectionChanged { + frontend: "resource-api".to_string(), + scope: "session".to_string(), + subject_key: projection_id.to_string(), + projection_revision: commit.revision, + origin: mutation.origin.clone(), + mutation_id: mutation.id.clone(), + }); } + Ok(commit) + } - pub fn terminal_registry_snapshot(&self) -> anyhow::Result { - self.workspace_registry.lock().unwrap().terminal_snapshot() - } + fn resolve_workspace_selector( + state: &State, + id: Option, + key: Option<&str>, + ) -> anyhow::Result> { + let by_id = id.and_then(|id| state.workspace_by_id(id)); + let by_key = key.and_then(|key| state.workspace_by_key(key)); + let workspace = match (id, key, by_id, by_key) { + (None, None, _, _) => anyhow::bail!("workspace or key is required"), + (Some(id), None, Some(workspace), _) if workspace.id == id => Some(workspace), + (Some(_), None, None, _) => None, + (None, Some(key), _, Some(workspace)) if workspace.key == key => Some(workspace), + (None, Some(_), _, None) => None, + (Some(_), Some(_), Some(by_id), Some(by_key)) if by_id.id == by_key.id => Some(by_id), + (Some(_), Some(_), _, _) => { + anyhow::bail!("workspace id and key do not identify the same workspace") + } + _ => unreachable!("workspace selector cases are exhaustive"), + }; + Ok(workspace.map(|workspace| (workspace.id, workspace.key.clone()))) + } - pub fn terminal_registry_revision(&self) -> anyhow::Result { - self.workspace_registry.lock().unwrap().terminal_revision() - } + pub fn subscribe(&self) -> MuxEventReceiver { + self.subscribers.subscribe() + } - #[cfg(test)] - pub(crate) fn reset_terminal_snapshot_count_for_test(&self) { - self.workspace_registry.lock().unwrap().reset_terminal_snapshot_count_for_test(); - } + pub fn subscribe_attached_surface(&self, surface: SurfaceId) -> MuxEventReceiver { + self.subscribers.subscribe_attached_surface(surface) + } - #[cfg(test)] - pub(crate) fn terminal_snapshot_count_for_test(&self) -> usize { - self.workspace_registry.lock().unwrap().terminal_snapshot_count_for_test() - } + pub fn subscribe_surface_session(&self, surface: SurfaceId) -> Option { + let state = self.state.lock().unwrap(); + let pane = state.pane_of(surface)?; + let (workspace_index, screen_index) = state.screen_of(pane)?; + let workspace = state.workspaces.get(workspace_index)?; + let screen = workspace.screens.get(screen_index)?; + Some(self.subscribers.subscribe_surface_session(surface, workspace.id, screen.id, pane)) + } - pub fn terminal_registry_events_page( - &self, - revision: u64, - ) -> anyhow::Result<( - TerminalRegistrySnapshot, - Vec, - )> { - // One writer guard is the read transaction boundary exposed to a - // frontend. Otherwise a commit between snapshot and event queries can - // return an event whose revision is newer than terminal_revision. - let registry = self.workspace_registry.lock().unwrap(); - let snapshot = registry.terminal_snapshot()?; - let events = registry.terminal_events_after(revision)?; - Ok((snapshot, events)) - } + pub fn emit(&self, event: MuxEvent) { + self.subscribers.emit(event); + } - pub fn workspace_registry_event( - &self, - revision: u64, - ) -> anyhow::Result> { - if revision == 0 { - return Ok(None); - } - Ok(self - .workspace_registry - .lock() - .unwrap() - .events_after(revision - 1)? - .into_iter() - .find(|event| event.revision == revision)) - } - - pub fn get_frontend_projection( - &self, - frontend: &str, - scope: &str, - subject_key: &str, - ) -> anyhow::Result> { - self.workspace_registry.lock().unwrap().get_frontend_projection( - frontend, - scope, - subject_key, - ) + pub(crate) fn emit_terminal_output(&self, runtime_id: SurfaceId) { + for placement in self.terminal_event_placements(runtime_id) { + self.emit(MuxEvent::SurfaceOutput(placement)); } + } - #[allow(clippy::too_many_arguments)] - pub fn put_frontend_projection( - &self, - mutation: &WorkspaceMutation, - frontend: &str, - scope: &str, - subject_key: &str, - schema_version: u32, - expected_projection_revision: Option, - projection: &Value, - ) -> anyhow::Result { - let mut registry = self.workspace_registry.lock().unwrap(); - let commit = registry.put_frontend_projection( - mutation, - frontend, - scope, - subject_key, - schema_version, - expected_projection_revision, - projection, - )?; - if !commit.replayed { - self.emit(MuxEvent::FrontendProjectionChanged { - frontend: frontend.to_string(), - scope: scope.to_string(), - subject_key: subject_key.to_string(), - projection_revision: commit.projection.projection_revision, - origin: mutation.origin.clone(), - mutation_id: mutation.id.clone(), - }); - } - Ok(commit) - } - - pub(crate) fn resource_put_frontend_projection_selected( - &self, - selectors: crate::ResourceSelectors, - projection_id: &FrontendProjectionPublicId, - projection: &Value, - expected_projection_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "operation":"frontend_projection.put", - "selectors":selectors, - "projection":projection, - }); - let mut registry = self.workspace_registry.lock().unwrap(); - if let Some(replay) = - registry.replay_resource_patch(mutation, "frontend_projection.put", &fingerprint)? - { - return Ok(replay); - } - let projection_revision = registry - .get_frontend_projection("resource-api", "session", projection_id.as_str())? - .map(|projection| projection.projection_revision) - .unwrap_or(0) - .checked_add(1) - .context("frontend projection revision exhausted")?; - let mut session_selectors = selectors; - session_selectors.frontend_projection = None; - let mut state = self.state.lock().unwrap(); - let resolved = self - .resolve_resource_path_in_state( - &state, - ®istry, - crate::ResourceTarget::Session, - &session_selectors, - ) - .map_err(anyhow::Error::new)?; - let session_id = - resolved.path.session.context("projection route omitted its session identity")?; - let value = serde_json::json!({ - "id":projection_id, - "session_id":session_id, - "frontend_id":projection["frontend_id"], - "window_id":projection["window_id"], - "generation":projection["generation"], - "projection":projection["projection"], - "projection_revision":projection_revision.to_string(), - }); - let deltas = serde_json::json!([{ - "kind":"upsert", - "sequence":0, - "resource":"frontend_projection", - "id":projection_id, - "value":value, - }]); - let commit = registry.commit_resource_projection( - mutation, - "frontend_projection.put", - &fingerprint, - None, - expected_projection_revision, - "resource-api", - "session", - projection_id.as_str(), - RESOURCE_API_FRONTEND_PROJECTION_SCHEMA_VERSION, - projection, - &value, - &deltas, - )?; - state.resource_revision = commit.revision; - drop(state); - drop(registry); - if !commit.replayed { - self.publish_resource_event(); - self.emit(MuxEvent::FrontendProjectionChanged { - frontend: "resource-api".to_string(), - scope: "session".to_string(), - subject_key: projection_id.to_string(), - projection_revision: commit.revision, - origin: mutation.origin.clone(), - mutation_id: mutation.id.clone(), - }); - } - Ok(commit) - } - - fn resolve_workspace_selector( - state: &State, - id: Option, - key: Option<&str>, - ) -> anyhow::Result> { - let by_id = id.and_then(|id| state.workspace_by_id(id)); - let by_key = key.and_then(|key| state.workspace_by_key(key)); - let workspace = match (id, key, by_id, by_key) { - (None, None, _, _) => anyhow::bail!("workspace or key is required"), - (Some(id), None, Some(workspace), _) if workspace.id == id => Some(workspace), - (Some(_), None, None, _) => None, - (None, Some(key), _, Some(workspace)) if workspace.key == key => Some(workspace), - (None, Some(_), _, None) => None, - (Some(_), Some(_), Some(by_id), Some(by_key)) if by_id.id == by_key.id => { - Some(by_id) - } - (Some(_), Some(_), _, _) => { - anyhow::bail!("workspace id and key do not identify the same workspace") - } - _ => unreachable!("workspace selector cases are exhaustive"), - }; - Ok(workspace.map(|workspace| (workspace.id, workspace.key.clone()))) + fn terminal_event_placements(&self, surface_id: SurfaceId) -> Vec { + let state = self.state.lock().unwrap(); + let Some(surface) = + state.surfaces.get(&surface_id).or_else(|| state.terminal_runtime_by_id(surface_id)) + else { + return vec![surface_id]; + }; + let Some(runtime_id) = surface.terminal_runtime_id() else { return vec![surface_id] }; + let Some(terminal_id) = surface.terminal_public_id() else { return vec![surface_id] }; + let placements = state + .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) + .iter() + .copied() + .filter(|placement| { + state + .surfaces + .get(placement) + .is_some_and(|surface| surface.terminal_runtime_id() == Some(runtime_id)) + }) + .collect::>(); + if placements.is_empty() && state.surfaces.contains_key(&surface_id) { + vec![surface_id] + } else { + placements } + } - pub fn subscribe(&self) -> MuxEventReceiver { - self.subscribers.subscribe() + pub(crate) fn emit_terminal_title(&self, surface: SurfaceId, title: Arc) { + for placement in self.terminal_event_placements(surface) { + self.emit(MuxEvent::TitleChanged { surface: placement, title: title.clone() }); } + } - pub fn subscribe_attached_surface(&self, surface: SurfaceId) -> MuxEventReceiver { - self.subscribers.subscribe_attached_surface(surface) + pub(crate) fn emit_terminal_bell(&self, surface: SurfaceId) { + for placement in self.terminal_event_placements(surface) { + self.emit(MuxEvent::Bell(placement)); } + } - pub fn subscribe_surface_session(&self, surface: SurfaceId) -> Option { - let state = self.state.lock().unwrap(); - let pane = state.pane_of(surface)?; - let (workspace_index, screen_index) = state.screen_of(pane)?; - let workspace = state.workspaces.get(workspace_index)?; - let screen = workspace.screens.get(screen_index)?; - Some(self.subscribers.subscribe_surface_session(surface, workspace.id, screen.id, pane)) + pub(crate) fn emit_terminal_resized( + &self, + surface: SurfaceId, + cols: u16, + rows: u16, + reservation_id: Option, + ) { + for placement in self.terminal_event_placements(surface) { + self.emit(MuxEvent::SurfaceResized { surface: placement, cols, rows, reservation_id }); } + } - pub fn emit(&self, event: MuxEvent) { - self.subscribers.emit(event); + pub(crate) fn emit_terminal_scroll(&self, surface: SurfaceId, offset: u64, at_bottom: bool) { + for placement in self.terminal_event_placements(surface) { + self.emit(MuxEvent::ScrollChanged { surface: placement, offset, at_bottom }); } + } - pub(crate) fn emit_terminal_output(&self, runtime_id: SurfaceId) { - for placement in self.terminal_event_placements(runtime_id) { - self.emit(MuxEvent::SurfaceOutput(placement)); - } + #[cfg(test)] + fn emit_terminal_exited(&self, surface: SurfaceId) { + for placement in self.terminal_event_placements(surface) { + self.emit(MuxEvent::SurfaceExited(placement)); } + } - fn terminal_event_placements(&self, surface_id: SurfaceId) -> Vec { - let state = self.state.lock().unwrap(); - let Some(surface) = state - .surfaces - .get(&surface_id) - .or_else(|| state.terminal_runtime_by_id(surface_id)) - else { - return vec![surface_id]; - }; - let Some(runtime_id) = surface.terminal_runtime_id() else { return vec![surface_id] }; - let Some(terminal_id) = surface.terminal_public_id() else { return vec![surface_id] }; - let placements = state - .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) - .iter() - .copied() - .filter(|placement| { - state - .surfaces - .get(placement) - .is_some_and(|surface| surface.terminal_runtime_id() == Some(runtime_id)) - }) - .collect::>(); - if placements.is_empty() && state.surfaces.contains_key(&surface_id) { - vec![surface_id] - } else { - placements - } + fn emit_tree_delta(&self, delta: TreeDelta, selection_resync: bool) { + #[cfg(test)] + if let Some(revision) = delta.workspace_revision + && let Some(hook) = self.workspace_delta_before_emit.lock().unwrap().clone() + { + hook(revision); } - - pub(crate) fn emit_terminal_title(&self, surface: SurfaceId, title: Arc) { - for placement in self.terminal_event_placements(surface) { - self.emit(MuxEvent::TitleChanged { surface: placement, title: title.clone() }); - } + self.emit(MuxEvent::TreeDelta(delta)); + if selection_resync { + self.emit(MuxEvent::TreeSelectionChanged); } + } - pub(crate) fn emit_terminal_bell(&self, surface: SurfaceId) { - for placement in self.terminal_event_placements(surface) { - self.emit(MuxEvent::Bell(placement)); - } - } + fn emit_committed_workspace_delta( + &self, + _registry: &WorkspaceRegistry, + delta: TreeDelta, + selection_resync: bool, + ) { + debug_assert!(delta.workspace_revision.is_some()); + self.emit_tree_delta(delta, selection_resync); + } - pub(crate) fn emit_terminal_resized( - &self, - surface: SurfaceId, - cols: u16, - rows: u16, - reservation_id: Option, - ) { - for placement in self.terminal_event_placements(surface) { - self.emit(MuxEvent::SurfaceResized { - surface: placement, - cols, - rows, - reservation_id, - }); - } + fn emit_empty_if_current(&self, workspace_revision: Option) { + let Some(workspace_revision) = workspace_revision else { return }; + #[cfg(test)] + let before_empty_check = self.workspace_close_before_empty_check.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = before_empty_check { + hook(); + } + let state = self.state.lock().unwrap(); + if state.workspaces.is_empty() && state.workspace_revision == workspace_revision { + self.emit(MuxEvent::Empty); } + } - pub(crate) fn emit_terminal_scroll( - &self, - surface: SurfaceId, - offset: u64, - at_bottom: bool, + fn rebuild_split_screen_index(state: &mut State) { + fn index_node( + node: &Node, + workspace_index: usize, + screen_index: usize, + screen: ScreenId, + index: &mut HashMap, ) { - for placement in self.terminal_event_placements(surface) { - self.emit(MuxEvent::ScrollChanged { surface: placement, offset, at_bottom }); + if let Node::Split { id, a, b, .. } = node { + index.insert(*id, (workspace_index, screen_index, screen)); + index_node(a, workspace_index, screen_index, screen, index); + index_node(b, workspace_index, screen_index, screen, index); } } - #[cfg(test)] - fn emit_terminal_exited(&self, surface: SurfaceId) { - for placement in self.terminal_event_placements(surface) { - self.emit(MuxEvent::SurfaceExited(placement)); + let mut index = HashMap::new(); + for (workspace_index, workspace) in state.workspaces.iter().enumerate() { + for (screen_index, screen) in workspace.screens.iter().enumerate() { + debug_assert!( + screen.layout_column_projection_is_consistent(), + "screen {} has a stale layout column projection", + screen.id + ); + index_node(&screen.root, workspace_index, screen_index, screen.id, &mut index); } } + state.split_screens = index; + state.rebuild_resource_indexes(); + } - fn emit_tree_delta(&self, delta: TreeDelta, selection_resync: bool) { - #[cfg(test)] - if let Some(revision) = delta.workspace_revision - && let Some(hook) = self.workspace_delta_before_emit.lock().unwrap().clone() - { - hook(revision); - } - self.emit(MuxEvent::TreeDelta(delta)); - if selection_resync { - self.emit(MuxEvent::TreeSelectionChanged); - } - } + fn emit_terminal_registry_changed(&self, registry: &WorkspaceRegistry, terminal_revision: u64) { + self.emit(MuxEvent::TerminalRegistryChanged { + registry_id: registry.registry_id().to_string(), + generation: registry.generation().to_string(), + terminal_revision, + }); + } - fn emit_committed_workspace_delta( - &self, - _registry: &WorkspaceRegistry, - delta: TreeDelta, - selection_resync: bool, - ) { - debug_assert!(delta.workspace_revision.is_some()); - self.emit_tree_delta(delta, selection_resync); - } + fn transition_terminal_lifecycle( + &self, + event_kind: &str, + operation: &str, + terminal_id: &str, + lifecycle: TerminalLifecycle, + incarnation: Option<&str>, + exit: Option, + ) -> anyhow::Result<(RegistryTerminal, u64)> { + anyhow::ensure!( + lifecycle != TerminalLifecycle::Exited, + "terminal exits must use the durable public exit latch" + ); + let mut registry = self.workspace_registry.lock().unwrap(); + let result = commit_terminal_lifecycle( + &mut registry, + event_kind, + operation, + terminal_id, + lifecycle, + incarnation, + exit, + )?; + self.emit_terminal_registry_changed(®istry, result.1); + Ok(result) + } - fn emit_empty_if_current(&self, workspace_revision: Option) { - let Some(workspace_revision) = workspace_revision else { return }; - #[cfg(test)] - let before_empty_check = - self.workspace_close_before_empty_check.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = before_empty_check { - hook(); - } - let state = self.state.lock().unwrap(); - if state.workspaces.is_empty() && state.workspace_revision == workspace_revision { - self.emit(MuxEvent::Empty); - } + /// A broken admin stream is not evidence that the per-terminal process + /// died. The surface keeps its tab and reconnects the same incarnation; + /// this callback only exposes the transient lifecycle to frontends. + pub(crate) fn terminal_host_connection_lost( + &self, + surface_id: SurfaceId, + identity: &TerminalHostIdentity, + ) -> bool { + if self.shutting_down.load(Ordering::Acquire) { + return false; } - - fn rebuild_split_screen_index(state: &mut State) { - fn index_node( - node: &Node, - workspace_index: usize, - screen_index: usize, - screen: ScreenId, - index: &mut HashMap, - ) { - if let Node::Split { id, a, b, .. } = node { - index.insert(*id, (workspace_index, screen_index, screen)); - index_node(a, workspace_index, screen_index, screen, index); - index_node(b, workspace_index, screen_index, screen, index); - } + let mut registry = self.workspace_registry.lock().unwrap(); + let state = self.state.lock().unwrap(); + let identity_matches = state + .surfaces + .get(&surface_id) + .or_else(|| state.terminal_runtime_by_id(surface_id)) + .and_then(|surface| surface.terminal_host_identity()) + .is_some_and(|current| current == *identity); + drop(state); + if !identity_matches { + return false; + } + let Ok(Some(terminal)) = registry.terminal_record(&identity.terminal_id) else { + return false; + }; + if terminal.incarnation.as_deref() != Some(identity.incarnation.as_str()) + || matches!( + terminal.lifecycle, + TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned + ) + { + return false; + } + if terminal.lifecycle == TerminalLifecycle::Adopting { + return true; + } + match commit_terminal_lifecycle( + &mut registry, + "terminal-adopting", + "terminal-admin-stream-lost", + &identity.terminal_id, + TerminalLifecycle::Adopting, + Some(&identity.incarnation), + None, + ) { + Ok((_, revision)) => { + self.emit_terminal_registry_changed(®istry, revision); + true } - - let mut index = HashMap::new(); - for (workspace_index, workspace) in state.workspaces.iter().enumerate() { - for (screen_index, screen) in workspace.screens.iter().enumerate() { - debug_assert!( - screen.layout_column_projection_is_consistent(), - "screen {} has a stale layout column projection", - screen.id - ); - index_node(&screen.root, workspace_index, screen_index, screen.id, &mut index); - } + Err(error) => { + self.emit(MuxEvent::Status(format!( + "could not persist terminal {} reconnect state: {error}", + identity.terminal_id + ))); + false } - state.split_screens = index; - state.rebuild_resource_indexes(); } + } - fn emit_terminal_registry_changed( - &self, - registry: &WorkspaceRegistry, - terminal_revision: u64, - ) { - self.emit(MuxEvent::TerminalRegistryChanged { - registry_id: registry.registry_id().to_string(), - generation: registry.generation().to_string(), - terminal_revision, - }); + pub(crate) fn terminal_host_reconnected( + self: &Arc, + surface_id: SurfaceId, + identity: &TerminalHostIdentity, + applied_kitty_limits: KittyGraphicsLimits, + ) -> bool { + if self.shutting_down.load(Ordering::Acquire) { + return false; } - - fn transition_terminal_lifecycle( - &self, - event_kind: &str, - operation: &str, - terminal_id: &str, - lifecycle: TerminalLifecycle, - incarnation: Option<&str>, - exit: Option, - ) -> anyhow::Result<(RegistryTerminal, u64)> { - anyhow::ensure!( - lifecycle != TerminalLifecycle::Exited, - "terminal exits must use the durable public exit latch" - ); - let mut registry = self.workspace_registry.lock().unwrap(); - let result = commit_terminal_lifecycle( - &mut registry, - event_kind, - operation, - terminal_id, - lifecycle, - incarnation, - exit, - )?; - self.emit_terminal_registry_changed(®istry, result.1); - Ok(result) - } - - /// A broken admin stream is not evidence that the per-terminal process - /// died. The surface keeps its tab and reconnects the same incarnation; - /// this callback only exposes the transient lifecycle to frontends. - pub(crate) fn terminal_host_connection_lost( - &self, - surface_id: SurfaceId, - identity: &TerminalHostIdentity, - ) -> bool { - if self.shutting_down.load(Ordering::Acquire) { - return false; - } - let mut registry = self.workspace_registry.lock().unwrap(); - let state = self.state.lock().unwrap(); - let identity_matches = state - .surfaces - .get(&surface_id) - .or_else(|| state.terminal_runtime_by_id(surface_id)) - .and_then(|surface| surface.terminal_host_identity()) - .is_some_and(|current| current == *identity); - drop(state); - if !identity_matches { - return false; - } - let Ok(Some(terminal)) = registry.terminal_record(&identity.terminal_id) else { - return false; - }; - if terminal.incarnation.as_deref() != Some(identity.incarnation.as_str()) - || matches!( - terminal.lifecycle, - TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned - ) - { - return false; - } - if terminal.lifecycle == TerminalLifecycle::Adopting { - return true; - } + let mut registry = self.workspace_registry.lock().unwrap(); + let state = self.state.lock().unwrap(); + let surface = state + .surfaces + .get(&surface_id) + .or_else(|| state.terminal_runtime_by_id(surface_id)) + .cloned(); + let identity_matches = surface + .as_ref() + .and_then(|surface| surface.terminal_host_identity()) + .is_some_and(|current| current == *identity); + drop(state); + if !identity_matches { + return false; + } + let Ok(Some(terminal)) = registry.terminal_record(&identity.terminal_id) else { + return false; + }; + if terminal.incarnation.as_deref() != Some(identity.incarnation.as_str()) + || matches!( + terminal.lifecycle, + TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned + ) + { + return false; + } + let lifecycle_ready = if terminal.lifecycle == TerminalLifecycle::Running { + true + } else { match commit_terminal_lifecycle( &mut registry, - "terminal-adopting", - "terminal-admin-stream-lost", + "terminal-ready", + "terminal-admin-stream-reconnected", &identity.terminal_id, - TerminalLifecycle::Adopting, + TerminalLifecycle::Running, Some(&identity.incarnation), None, ) { @@ -6971,478 +6945,267 @@ impl Mux { } Err(error) => { self.emit(MuxEvent::Status(format!( - "could not persist terminal {} reconnect state: {error}", + "could not persist terminal {} reconnect completion: {error}", identity.terminal_id ))); false } } + }; + drop(registry); + if !lifecycle_ready { + return false; } - - pub(crate) fn terminal_host_reconnected( - self: &Arc, - surface_id: SurfaceId, - identity: &TerminalHostIdentity, - applied_kitty_limits: KittyGraphicsLimits, - ) -> bool { - if self.shutting_down.load(Ordering::Acquire) { - return false; - } - let mut registry = self.workspace_registry.lock().unwrap(); - let state = self.state.lock().unwrap(); - let surface = state - .surfaces - .get(&surface_id) - .or_else(|| state.terminal_runtime_by_id(surface_id)) - .cloned(); - let identity_matches = surface - .as_ref() - .and_then(|surface| surface.terminal_host_identity()) - .is_some_and(|current| current == *identity); - drop(state); - if !identity_matches { - return false; - } - let Ok(Some(terminal)) = registry.terminal_record(&identity.terminal_id) else { - return false; - }; - if terminal.incarnation.as_deref() != Some(identity.incarnation.as_str()) - || matches!( - terminal.lifecycle, - TerminalLifecycle::Exited | TerminalLifecycle::Tombstoned - ) - { - return false; - } - let lifecycle_ready = if terminal.lifecycle == TerminalLifecycle::Running { - true - } else { - match commit_terminal_lifecycle( - &mut registry, - "terminal-ready", - "terminal-admin-stream-reconnected", - &identity.terminal_id, - TerminalLifecycle::Running, - Some(&identity.incarnation), - None, - ) { - Ok((_, revision)) => { - self.emit_terminal_registry_changed(®istry, revision); - true - } - Err(error) => { - self.emit(MuxEvent::Status(format!( - "could not persist terminal {} reconnect completion: {error}", - identity.terminal_id - ))); - false - } - } - }; - drop(registry); - if !lifecycle_ready { - return false; - } - #[cfg(debug_assertions)] - if self - .terminal_host_reconnect_completion_failures - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { - (remaining > 0).then(|| remaining - 1) - }) - .is_ok() - { - return false; - } - surface.is_some_and(|surface| { - self.reconcile_reconnected_kitty_image_surface(&surface, applied_kitty_limits) + #[cfg(debug_assertions)] + if self + .terminal_host_reconnect_completion_failures + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + (remaining > 0).then(|| remaining - 1) }) + .is_ok() + { + return false; + } + surface.is_some_and(|surface| { + self.reconcile_reconnected_kitty_image_surface(&surface, applied_kitty_limits) + }) + } + + pub(crate) fn lock_client_sizing_lifecycle(&self) -> MutexGuard<'_, ()> { + self.client_sizing_lifecycle.lock().unwrap() + } + + #[cfg(debug_assertions)] + pub(crate) fn take_test_terminal_host_disconnect_after_spawn(&self) -> Option { + let delay_ms = self.terminal_host_test_disconnect_after_spawn_ms.swap(0, Ordering::AcqRel); + (delay_ms > 0).then(|| Duration::from_millis(delay_ms)) + } + + pub fn begin_pairing( + &self, + peer: std::net::IpAddr, + ) -> Result<(PairingChallenge, Receiver), PairingError> { + let result = self.pairing.begin(peer)?; + self.emit(MuxEvent::PairingRequested(result.0.clone())); + Ok(result) + } + + pub fn respond_pairing(&self, id: u64, approve: bool) -> bool { + let responded = self.pairing.respond(id, approve); + if responded { + self.emit(MuxEvent::PairingResolved { request: id }); } + responded + } - pub(crate) fn lock_client_sizing_lifecycle(&self) -> MutexGuard<'_, ()> { - self.client_sizing_lifecycle.lock().unwrap() + pub(crate) fn resource_resolve_pairing_selected( + &self, + selectors: crate::ResourceSelectors, + pairing_id: &PairingRequestPublicId, + decision: &str, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + let fingerprint = serde_json::json!({ + "operation":"pairing_request.resolve", + "selectors":selectors, + "pairing_request_id":pairing_id, + "decision":decision, + }); + let mut registry = self.workspace_registry.lock().unwrap(); + if let Some(replay) = + registry.replay_resource_patch(mutation, "pairing_request.resolve", &fingerprint)? + { + return Ok(replay); } - #[cfg(debug_assertions)] - pub(crate) fn take_test_terminal_host_disconnect_after_spawn(&self) -> Option { - let delay_ms = - self.terminal_host_test_disconnect_after_spawn_ms.swap(0, Ordering::AcqRel); - (delay_ms > 0).then(|| Duration::from_millis(delay_ms)) - } - - pub fn begin_pairing( - &self, - peer: std::net::IpAddr, - ) -> Result<(PairingChallenge, Receiver), PairingError> { - let result = self.pairing.begin(peer)?; - self.emit(MuxEvent::PairingRequested(result.0.clone())); - Ok(result) - } - - pub fn respond_pairing(&self, id: u64, approve: bool) -> bool { - let responded = self.pairing.respond(id, approve); - if responded { - self.emit(MuxEvent::PairingResolved { request: id }); - } - responded - } - - pub(crate) fn resource_resolve_pairing_selected( - &self, - selectors: crate::ResourceSelectors, - pairing_id: &PairingRequestPublicId, - decision: &str, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "operation":"pairing_request.resolve", - "selectors":selectors, - "pairing_request_id":pairing_id, - "decision":decision, - }); - let mut registry = self.workspace_registry.lock().unwrap(); - if let Some(replay) = - registry.replay_resource_patch(mutation, "pairing_request.resolve", &fingerprint)? - { - return Ok(replay); + let approve = match decision { + "accept" => true, + "reject" => false, + _ => { + return Err(anyhow::Error::new(ResourceError::validation_invalid( + Some("decision"), + "pairing decision must be accept or reject", + ))); } + }; + let payload = + pairing_id.as_str().strip_prefix("pairing_").expect("typed pairing id prefix"); + let numeric = u128::from_str_radix(payload, 16) + .ok() + .and_then(|value| u64::try_from(value).ok()) + .ok_or_else(|| { + anyhow::Error::new(ResourceError::not_found("pairing_request", pairing_id.as_str())) + })?; - let approve = match decision { - "accept" => true, - "reject" => false, - _ => { - return Err(anyhow::Error::new(ResourceError::validation_invalid( - Some("decision"), - "pairing decision must be accept or reject", - ))); - } - }; - let payload = - pairing_id.as_str().strip_prefix("pairing_").expect("typed pairing id prefix"); - let numeric = u128::from_str_radix(payload, 16) - .ok() - .and_then(|value| u64::try_from(value).ok()) - .ok_or_else(|| { - anyhow::Error::new(ResourceError::not_found( - "pairing_request", - pairing_id.as_str(), - )) - })?; + let mut session_selectors = selectors; + session_selectors.pairing_request = None; + let mut state = self.state.lock().unwrap(); + let resolved = self + .resolve_resource_path_in_state( + &state, + ®istry, + crate::ResourceTarget::Session, + &session_selectors, + ) + .map_err(anyhow::Error::new)?; + let session_id = + resolved.path.session.context("pairing route omitted its session identity")?; - let mut session_selectors = selectors; - session_selectors.pairing_request = None; - let mut state = self.state.lock().unwrap(); - let resolved = self - .resolve_resource_path_in_state( - &state, - ®istry, - crate::ResourceTarget::Session, - &session_selectors, - ) - .map_err(anyhow::Error::new)?; - let session_id = - resolved.path.session.context("pairing route omitted its session identity")?; - - let commit = self - .pairing - .respond_after(numeric, approve, |challenge| { - let status = if approve { "accepted" } else { "rejected" }; - let value = serde_json::json!({ - "pairing_request":{ - "id":pairing_id, - "session_id":session_id, - "peer":challenge.peer, - "code":challenge.code, - "expires_in_seconds":challenge.expires_in.to_string(), - "status":status, - }, - }); - let deltas = serde_json::json!([{ - "kind":"delete", - "sequence":0, - "resource":"pairing_request", + let commit = self + .pairing + .respond_after(numeric, approve, |challenge| { + let status = if approve { "accepted" } else { "rejected" }; + let value = serde_json::json!({ + "pairing_request":{ "id":pairing_id, - }]); - registry.commit_resource_patch( - mutation, - "pairing_request.resolve", - &fingerprint, - None, - expected_revision, - &ResourcePatch { changes: Vec::new() }, - &value, - &deltas, - ) - })? - .ok_or_else(|| { - anyhow::Error::new(ResourceError::not_found( - "pairing_request", - pairing_id.as_str(), - )) - })?; + "session_id":session_id, + "peer":challenge.peer, + "code":challenge.code, + "expires_in_seconds":challenge.expires_in.to_string(), + "status":status, + }, + }); + let deltas = serde_json::json!([{ + "kind":"delete", + "sequence":0, + "resource":"pairing_request", + "id":pairing_id, + }]); + registry.commit_resource_patch( + mutation, + "pairing_request.resolve", + &fingerprint, + None, + expected_revision, + &ResourcePatch { changes: Vec::new() }, + &value, + &deltas, + ) + })? + .ok_or_else(|| { + anyhow::Error::new(ResourceError::not_found("pairing_request", pairing_id.as_str())) + })?; - state.resource_revision = commit.revision; - drop(state); - drop(registry); - if !commit.replayed { - self.publish_resource_event(); - self.emit(MuxEvent::PairingResolved { request: numeric }); - } - Ok(commit) + state.resource_revision = commit.revision; + drop(state); + drop(registry); + if !commit.replayed { + self.publish_resource_event(); + self.emit(MuxEvent::PairingResolved { request: numeric }); } + Ok(commit) + } - pub fn cancel_pairing(&self, id: u64) { - if self.pairing.cancel(id) { - self.emit(MuxEvent::PairingResolved { request: id }); - } + pub fn cancel_pairing(&self, id: u64) { + if self.pairing.cancel(id) { + self.emit(MuxEvent::PairingResolved { request: id }); } + } - pub fn authenticate_pairing_credential(&self, credential: &str) -> bool { - self.pairing.authenticate(credential) - } + pub fn authenticate_pairing_credential(&self, credential: &str) -> bool { + self.pairing.authenticate(credential) + } - pub fn pending_pairings(&self) -> Vec { - self.pairing.pending() - } + pub fn pending_pairings(&self) -> Vec { + self.pairing.pending() + } - fn spawn_surface_in_workspace( - self: &Arc, - workspace_key: &str, - cwd: Option, - size: Option<(u16, u16)>, - command: Option>, - ) -> anyhow::Result> { - self.spawn_surface_with(cwd, command, size, Some(workspace_key), None) - } + fn spawn_surface_in_workspace( + self: &Arc, + workspace_key: &str, + cwd: Option, + size: Option<(u16, u16)>, + command: Option>, + ) -> anyhow::Result> { + self.spawn_surface_with(cwd, command, size, Some(workspace_key), None) + } - fn spawn_surface_in_workspace_reserved( - self: &Arc, - workspace_key: &str, - cwd: Option, - size: Option<(u16, u16)>, - command: Option>, - reservation: TerminalReservationRequest, - ) -> anyhow::Result> { - self.spawn_surface_with(cwd, command, size, Some(workspace_key), Some(reservation)) - } + fn spawn_surface_in_workspace_reserved( + self: &Arc, + workspace_key: &str, + cwd: Option, + size: Option<(u16, u16)>, + command: Option>, + reservation: TerminalReservationRequest, + ) -> anyhow::Result> { + self.spawn_surface_with(cwd, command, size, Some(workspace_key), Some(reservation)) + } - fn spawn_surface_with( - self: &Arc, - cwd: Option, - command: Option>, - size: Option<(u16, u16)>, - workspace_key: Option<&str>, - reservation: Option, - ) -> anyhow::Result> { - let mut owner_reservation = self.reserve_surface_owner()?; - let id = self.next_id(); - let mut opts = self.surface_options.lock().unwrap().clone(); - if cwd.is_some() { - opts.cwd = cwd; - } - if command.is_some() { - opts.command = command; - } - // Spawn at the latest client-owned size: starting at the default - // 80x24 and resizing a frame later makes shells emit artifacts - // (e.g. zsh's reverse-video %% partial-line marker). - let (cols, rows) = self.resolve_client_size(size, (opts.cols, opts.rows)); - opts.cols = cols; - opts.rows = rows; - let cell_pixels = { - let cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); - let cell_pixels = self.cell_pixel_creation_size(); - drop(cell_pixel_lifecycle); - cell_pixels - }; - #[cfg(test)] - if let Some(hook) = - self.terminal_spawn_after_cell_pixel_snapshot.lock().unwrap().clone() - { - let unlocked = match self.cell_pixel_lifecycle.try_lock() { - Ok(lifecycle) => { - drop(lifecycle); - true - } - Err(_) => false, - }; - hook(unlocked); - } - #[cfg(all(test, unix))] - let use_host_runtime = !self.test_surface_runtime; - #[cfg(all(not(test), unix))] - let use_host_runtime = true; - #[cfg(unix)] - if let (Some(host_root), Some(workspace_key), true) = - (opts.terminal_host_root.as_ref(), workspace_key, use_host_runtime) - { - let terminal_id = reservation - .as_ref() - .map(|reservation| reservation.terminal_id) - .map(Ok) - .unwrap_or_else(TerminalId::random)?; - let terminal_hex = terminal_id.to_hex(); - // Acquire before the durable Launching commit. A replacement can - // therefore finalize record absence only after this parent dies - // before spawn or the inherited child finishes publication. - let publication_guard = - crate::terminal_host_runtime::acquire_terminal_host_publication( - host_root, - &terminal_hex, - )?; - let launch_spec = terminal_launch_spec(&opts); - let terminal = RegistryTerminal { - terminal_id: terminal_hex.clone(), - workspace_key: workspace_key.to_string(), - incarnation: None, - lifecycle: TerminalLifecycle::Launching, - launch_spec, - exit: None, - }; - let reserve_replayed = { - let mut registry = self.workspace_registry.lock().unwrap(); - let (replayed, revision) = if let Some(reservation) = reservation.as_ref() { - let commit = registry.commit_terminal( - &reservation.mutation, - &reservation.fingerprint, - reservation.expected_generation.as_deref(), - reservation.expected_revision, - "terminal-reserved", - &terminal, - &serde_json::json!({ - "terminal_id":terminal_hex, - "workspace_key":workspace_key, - "state":"launching", - }), - )?; - (commit.replayed, commit.revision) - } else { - let revision = commit_terminal_transition( - &mut registry, - "terminal-reserved", - "reserve-terminal", - &terminal, - )?; - (false, revision) - }; - if !replayed { - self.emit_terminal_registry_changed(®istry, revision); - } - replayed - }; - if reserve_replayed { - anyhow::bail!("terminal_create_replayed"); - } - let surface = match Surface::spawn_with_terminal_id_at_cell_pixels( - id, - opts, - Arc::downgrade(self), - terminal_id, - publication_guard, - cell_pixels, - ) { - Ok(surface) => surface, - Err(error) => { - let _ = self.persist_terminal_exit( - &terminal_hex, - None, - &TerminalExit::unknown(format!("launch-failed: {error}")), - ); - return Err(error); - } - }; - let Some(identity) = surface.terminal_host_identity() else { - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - anyhow::bail!("reserved terminal did not return host identity"); - }; - if identity.terminal_id != terminal_hex { - let _ = self.persist_terminal_exit( - &terminal_hex, - None, - &TerminalExit::unknown("host-identity-mismatch"), - ); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - anyhow::bail!("terminal host changed registry-reserved identity"); - } - let ready_revision = { - let mut registry = self.workspace_registry.lock().unwrap(); - let ready = commit_terminal_lifecycle( - &mut registry, - "terminal-ready", - "terminal-ready", - &terminal_hex, - TerminalLifecycle::Running, - Some(&identity.incarnation), - None, - ); - let (_, ready_revision) = match ready { - Ok(ready) => ready, - Err(error) => { - drop(registry); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - return Err(error); - } - }; - self.emit_terminal_registry_changed(®istry, ready_revision); - ready_revision - }; - let cell_pixel_lifecycle = - match self.reconcile_surface_cell_pixels_for_publish(&surface) { - Ok(lifecycle) => lifecycle, - Err(error) => { - let _ = self.transition_terminal_lifecycle( - "terminal-exited", - "terminal-cell-pixel-reconcile-failed", - &terminal_hex, - TerminalLifecycle::Exited, - Some(&identity.incarnation), - Some(serde_json::json!({ - "reason":"cell-pixel-reconcile-failed", - "error":error.to_string(), - "ready_revision":ready_revision, - })), - ); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - return Err(error); - } - }; - let insert_result = { - let mut state = self.state.lock().unwrap(); - insert_reserved_surface_checked( - self, - &mut state, - surface.clone(), - &mut owner_reservation, - ) - }; - drop(cell_pixel_lifecycle); - if let Err(error) = insert_result { - let _ = self.persist_terminal_exit( - &terminal_hex, - Some(&identity.incarnation), - &TerminalExit::unknown("surface-insert-failed"), - ); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - return Err(error); + fn spawn_surface_with( + self: &Arc, + cwd: Option, + command: Option>, + size: Option<(u16, u16)>, + workspace_key: Option<&str>, + reservation: Option, + ) -> anyhow::Result> { + let mut owner_reservation = self.reserve_surface_owner()?; + let id = self.next_id(); + let mut opts = self.surface_options.lock().unwrap().clone(); + if cwd.is_some() { + opts.cwd = cwd; + } + if command.is_some() { + opts.command = command; + } + // Spawn at the latest client-owned size: starting at the default + // 80x24 and resizing a frame later makes shells emit artifacts + // (e.g. zsh's reverse-video %% partial-line marker). + let (cols, rows) = self.resolve_client_size(size, (opts.cols, opts.rows)); + opts.cols = cols; + opts.rows = rows; + let cell_pixels = { + let cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); + let cell_pixels = self.cell_pixel_creation_size(); + drop(cell_pixel_lifecycle); + cell_pixels + }; + #[cfg(test)] + if let Some(hook) = self.terminal_spawn_after_cell_pixel_snapshot.lock().unwrap().clone() { + let unlocked = match self.cell_pixel_lifecycle.try_lock() { + Ok(lifecycle) => { + drop(lifecycle); + true } - // Deprecated recovery mirror only; SQLite is placement authority. - let _ = surface.persist_host_workspace(workspace_key); - return Ok(surface); - } - if let (Some(workspace_key), Some(reservation)) = (workspace_key, reservation.as_ref()) - { - let terminal_hex = reservation.terminal_id.to_hex(); - let launch_spec = terminal_launch_spec(&opts); - let terminal = RegistryTerminal { - terminal_id: terminal_hex.clone(), - workspace_key: workspace_key.to_string(), - incarnation: None, - lifecycle: TerminalLifecycle::Launching, - launch_spec, - exit: None, - }; - { - let mut registry = self.workspace_registry.lock().unwrap(); + Err(_) => false, + }; + hook(unlocked); + } + #[cfg(all(test, unix))] + let use_host_runtime = !self.test_surface_runtime; + #[cfg(all(not(test), unix))] + let use_host_runtime = true; + #[cfg(unix)] + if let (Some(host_root), Some(workspace_key), true) = + (opts.terminal_host_root.as_ref(), workspace_key, use_host_runtime) + { + let terminal_id = reservation + .as_ref() + .map(|reservation| reservation.terminal_id) + .map(Ok) + .unwrap_or_else(TerminalId::random)?; + let terminal_hex = terminal_id.to_hex(); + // Acquire before the durable Launching commit. A replacement can + // therefore finalize record absence only after this parent dies + // before spawn or the inherited child finishes publication. + let publication_guard = + crate::terminal_host_runtime::acquire_terminal_host_publication( + host_root, + &terminal_hex, + )?; + let launch_spec = terminal_launch_spec(&opts); + let terminal = RegistryTerminal { + terminal_id: terminal_hex.clone(), + workspace_key: workspace_key.to_string(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec, + exit: None, + }; + let reserve_replayed = { + let mut registry = self.workspace_registry.lock().unwrap(); + let (replayed, revision) = if let Some(reservation) = reservation.as_ref() { let commit = registry.commit_terminal( &reservation.mutation, &reservation.fingerprint, @@ -7456,124 +7219,93 @@ impl Mux { "state":"launching", }), )?; - if commit.replayed { - anyhow::bail!("terminal_create_replayed"); - } - self.emit_terminal_registry_changed(®istry, commit.revision); - } - #[cfg(test)] - if let Some(hook) = - self.terminal_create_after_terminal_reservation.lock().unwrap().clone() - { - hook(&terminal_hex); - } - #[cfg(test)] - let surface_result = if self.test_surface_runtime { - Surface::spawn_for_test_at_cell_pixels( - id, - opts, - Arc::downgrade(self), - cell_pixels, - ) + (commit.replayed, commit.revision) } else { - Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) - }; - #[cfg(not(test))] - let surface_result = - Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels); - let surface = match surface_result { - Ok(surface) => surface, - Err(error) => { - let _ = self.persist_terminal_exit( - &terminal_hex, - None, - &TerminalExit::unknown(format!("launch-failed: {error}")), - ); - return Err(error); - } - }; - let incarnation = TerminalId::random()?.to_hex(); - let identity = TerminalHostIdentity { - terminal_id: terminal_hex.clone(), - incarnation: incarnation.clone(), - }; - { - let mut registry = self.workspace_registry.lock().unwrap(); - let (_, revision) = match commit_terminal_lifecycle( + let revision = commit_terminal_transition( &mut registry, - "terminal-ready", - "terminal-ready", - &terminal_hex, - TerminalLifecycle::Running, - Some(&incarnation), - None, - ) { - Ok(ready) => ready, - Err(error) => { - drop(registry); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - return Err(error); - } - }; + "terminal-reserved", + "reserve-terminal", + &terminal, + )?; + (false, revision) + }; + if !replayed { self.emit_terminal_registry_changed(®istry, revision); } - let cell_pixel_lifecycle = match self - .reconcile_surface_cell_pixels_for_publish(&surface) - { - Ok(lifecycle) => lifecycle, - Err(error) => { - let _ = self.persist_terminal_exit( - &terminal_hex, - Some(&incarnation), - &TerminalExit::unknown(format!("cell-pixel-reconcile-failed: {error}")), - ); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); - return Err(error); - } - }; - let insert_result = { - let mut state = self.state.lock().unwrap(); - insert_reserved_surface_checked( - self, - &mut state, - surface.clone(), - &mut owner_reservation, - ) - }; - drop(cell_pixel_lifecycle); - if let Err(error) = insert_result { + replayed + }; + if reserve_replayed { + anyhow::bail!("terminal_create_replayed"); + } + let surface = match Surface::spawn_with_terminal_id_at_cell_pixels( + id, + opts, + Arc::downgrade(self), + terminal_id, + publication_guard, + cell_pixels, + ) { + Ok(surface) => surface, + Err(error) => { let _ = self.persist_terminal_exit( &terminal_hex, - Some(&incarnation), - &TerminalExit::unknown("surface-insert-failed"), + None, + &TerminalExit::unknown(format!("launch-failed: {error}")), ); - self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } - self.reserved_in_process_terminals.lock().unwrap().insert(surface.id, identity); - return Ok(surface); - } - #[cfg(test)] - let surface_result = if self.test_surface_runtime { - Surface::spawn_for_test_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) - } else { - Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) }; - #[cfg(not(test))] - let surface_result = - Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels); - let surface = match surface_result { - Ok(surface) => surface, - Err(error) => { - self.pending_workspace_surfaces.lock().unwrap().remove(&id); - return Err(error); - } + let Some(identity) = surface.terminal_host_identity() else { + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + anyhow::bail!("reserved terminal did not return host identity"); + }; + if identity.terminal_id != terminal_hex { + let _ = self.persist_terminal_exit( + &terminal_hex, + None, + &TerminalExit::unknown("host-identity-mismatch"), + ); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + anyhow::bail!("terminal host changed registry-reserved identity"); + } + let ready_revision = { + let mut registry = self.workspace_registry.lock().unwrap(); + let ready = commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "terminal-ready", + &terminal_hex, + TerminalLifecycle::Running, + Some(&identity.incarnation), + None, + ); + let (_, ready_revision) = match ready { + Ok(ready) => ready, + Err(error) => { + drop(registry); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); + } + }; + self.emit_terminal_registry_changed(®istry, ready_revision); + ready_revision }; let cell_pixel_lifecycle = match self.reconcile_surface_cell_pixels_for_publish(&surface) { Ok(lifecycle) => lifecycle, Err(error) => { - self.pending_workspace_surfaces.lock().unwrap().remove(&id); + let _ = self.transition_terminal_lifecycle( + "terminal-exited", + "terminal-cell-pixel-reconcile-failed", + &terminal_hex, + TerminalLifecycle::Exited, + Some(&identity.incarnation), + Some(serde_json::json!({ + "reason":"cell-pixel-reconcile-failed", + "error":error.to_string(), + "ready_revision":ready_revision, + })), + ); self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } @@ -7589,62 +7321,109 @@ impl Mux { }; drop(cell_pixel_lifecycle); if let Err(error) = insert_result { - self.pending_workspace_surfaces.lock().unwrap().remove(&id); + let _ = self.persist_terminal_exit( + &terminal_hex, + Some(&identity.incarnation), + &TerminalExit::unknown("surface-insert-failed"), + ); self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } - Ok(surface) + // Deprecated recovery mirror only; SQLite is placement authority. + let _ = surface.persist_host_workspace(workspace_key); + return Ok(surface); } - - fn spawn_sidebar_plugin_surface( - self: &Arc, - options: &SidebarPluginOptions, - size: (u16, u16), - ) -> anyhow::Result> { - if options.command.is_empty() { - anyhow::bail!("sidebar plugin command is empty"); - } - let mut owner_reservation = self.reserve_surface_owner()?; - let id = self.next_id(); - let mut opts = self.surface_options.lock().unwrap().clone(); - opts.command = Some(options.command.clone()); - opts.cwd = options.cwd.clone(); - opts.cols = size.0.max(1); - opts.rows = size.1.max(1); - opts.extra_env.push(("CMUX_SIDEBAR".to_string(), "1".to_string())); - let cell_pixels = { - let cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); - let cell_pixels = self.cell_pixel_creation_size(); - drop(cell_pixel_lifecycle); - cell_pixels + if let (Some(workspace_key), Some(reservation)) = (workspace_key, reservation.as_ref()) { + let terminal_hex = reservation.terminal_id.to_hex(); + let launch_spec = terminal_launch_spec(&opts); + let terminal = RegistryTerminal { + terminal_id: terminal_hex.clone(), + workspace_key: workspace_key.to_string(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec, + exit: None, }; + { + let mut registry = self.workspace_registry.lock().unwrap(); + let commit = registry.commit_terminal( + &reservation.mutation, + &reservation.fingerprint, + reservation.expected_generation.as_deref(), + reservation.expected_revision, + "terminal-reserved", + &terminal, + &serde_json::json!({ + "terminal_id":terminal_hex, + "workspace_key":workspace_key, + "state":"launching", + }), + )?; + if commit.replayed { + anyhow::bail!("terminal_create_replayed"); + } + self.emit_terminal_registry_changed(®istry, commit.revision); + } #[cfg(test)] - let surface = if self.test_surface_runtime { - Surface::spawn_auxiliary_for_test_at_cell_pixels( - id, - opts, - Arc::downgrade(self), - cell_pixels, - )? + if let Some(hook) = + self.terminal_create_after_terminal_reservation.lock().unwrap().clone() + { + hook(&terminal_hex); + } + #[cfg(test)] + let surface_result = if self.test_surface_runtime { + Surface::spawn_for_test_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) } else { - Surface::spawn_auxiliary_at_cell_pixels( - id, - opts, - Arc::downgrade(self), - cell_pixels, - )? + Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) }; #[cfg(not(test))] - let surface = Surface::spawn_auxiliary_at_cell_pixels( - id, - opts, - Arc::downgrade(self), - cell_pixels, - )?; + let surface_result = + Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels); + let surface = match surface_result { + Ok(surface) => surface, + Err(error) => { + let _ = self.persist_terminal_exit( + &terminal_hex, + None, + &TerminalExit::unknown(format!("launch-failed: {error}")), + ); + return Err(error); + } + }; + let incarnation = TerminalId::random()?.to_hex(); + let identity = TerminalHostIdentity { + terminal_id: terminal_hex.clone(), + incarnation: incarnation.clone(), + }; + { + let mut registry = self.workspace_registry.lock().unwrap(); + let (_, revision) = match commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "terminal-ready", + &terminal_hex, + TerminalLifecycle::Running, + Some(&incarnation), + None, + ) { + Ok(ready) => ready, + Err(error) => { + drop(registry); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); + } + }; + self.emit_terminal_registry_changed(®istry, revision); + } let cell_pixel_lifecycle = match self.reconcile_surface_cell_pixels_for_publish(&surface) { Ok(lifecycle) => lifecycle, Err(error) => { + let _ = self.persist_terminal_exit( + &terminal_hex, + Some(&incarnation), + &TerminalExit::unknown(format!("cell-pixel-reconcile-failed: {error}")), + ); self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } @@ -7660,352 +7439,396 @@ impl Mux { }; drop(cell_pixel_lifecycle); if let Err(error) = insert_result { + let _ = self.persist_terminal_exit( + &terminal_hex, + Some(&incarnation), + &TerminalExit::unknown("surface-insert-failed"), + ); self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } - Ok(surface) + self.reserved_in_process_terminals.lock().unwrap().insert(surface.id, identity); + return Ok(surface); } - - fn spawn_browser_surface_with_resource_identity( - self: &Arc, - url: String, - size: Option<(u16, u16)>, - pending_workspace: Option, - resource_identity: Option, - ) -> anyhow::Result> { - let mut owner_reservation = self.reserve_surface_owner()?; - let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); - let id = self.next_id(); - if let Some(workspace) = pending_workspace { - self.pending_workspace_surfaces.lock().unwrap().insert(id, workspace); + #[cfg(test)] + let surface_result = if self.test_surface_runtime { + Surface::spawn_for_test_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) + } else { + Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels) + }; + #[cfg(not(test))] + let surface_result = + Surface::spawn_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels); + let surface = match surface_result { + Ok(surface) => surface, + Err(error) => { + self.pending_workspace_surfaces.lock().unwrap().remove(&id); + return Err(error); } - let opts = self.surface_options.lock().unwrap().clone(); - let size = self.resolve_client_size(size, (opts.cols, opts.rows)); - let cell_pixels = self.cell_pixel_creation_size(); - let surface = match resource_identity { - Some(identity) => browser::new_surface_with_resource_identity( - id, - url.clone(), - size, - cell_pixels, - &opts, - Arc::downgrade(self), - identity, - )?, - None => browser::new_surface( - id, - url.clone(), - size, - cell_pixels, - &opts, - Arc::downgrade(self), - )?, - }; - let insertion = { - let mut state = self.state.lock().unwrap(); - insert_reserved_surface_checked( - self, - &mut state, - surface.clone(), - &mut owner_reservation, - ) - }; - if let Err(error) = insertion { + }; + let cell_pixel_lifecycle = match self.reconcile_surface_cell_pixels_for_publish(&surface) { + Ok(lifecycle) => lifecycle, + Err(error) => { self.pending_workspace_surfaces.lock().unwrap().remove(&id); self.retire_reserved_surface_runtime(surface, &mut owner_reservation); return Err(error); } - let tab_id = surface - .resource_identity() - .context("browser surface omitted its public tab identity")? - .tab_id - .clone(); - self.start_browser_bootstrap( + }; + let insert_result = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, surface.clone(), - BrowserBootstrap::Provider { tab_id, url }, - None, - ); - Ok(surface) + &mut owner_reservation, + ) + }; + drop(cell_pixel_lifecycle); + if let Err(error) = insert_result { + self.pending_workspace_surfaces.lock().unwrap().remove(&id); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); } + Ok(surface) + } - fn resolve_client_size( - &self, - requested: Option<(u16, u16)>, - default: (u16, u16), - ) -> (u16, u16) { - let mut sizing = self.client_sizing.lock().unwrap(); - if let Some((cols, rows)) = requested { - let size = clamp_terminal_size(cols, rows); - sizing.record_explicit_size(size); - return size; + fn spawn_sidebar_plugin_surface( + self: &Arc, + options: &SidebarPluginOptions, + size: (u16, u16), + ) -> anyhow::Result> { + if options.command.is_empty() { + anyhow::bail!("sidebar plugin command is empty"); + } + let mut owner_reservation = self.reserve_surface_owner()?; + let id = self.next_id(); + let mut opts = self.surface_options.lock().unwrap().clone(); + opts.command = Some(options.command.clone()); + opts.cwd = options.cwd.clone(); + opts.cols = size.0.max(1); + opts.rows = size.1.max(1); + opts.extra_env.push(("CMUX_SIDEBAR".to_string(), "1".to_string())); + let cell_pixels = { + let cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); + let cell_pixels = self.cell_pixel_creation_size(); + drop(cell_pixel_lifecycle); + cell_pixels + }; + #[cfg(test)] + let surface = if self.test_surface_runtime { + Surface::spawn_auxiliary_for_test_at_cell_pixels( + id, + opts, + Arc::downgrade(self), + cell_pixels, + )? + } else { + Surface::spawn_auxiliary_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels)? + }; + #[cfg(not(test))] + let surface = + Surface::spawn_auxiliary_at_cell_pixels(id, opts, Arc::downgrade(self), cell_pixels)?; + let cell_pixel_lifecycle = match self.reconcile_surface_cell_pixels_for_publish(&surface) { + Ok(lifecycle) => lifecycle, + Err(error) => { + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); } - let attached_clients = self.control_clients.attached_client_ids_by_surface(); - sizing - .creation_size(&attached_clients) - .unwrap_or_else(|| clamp_terminal_size(default.0, default.1)) + }; + let insert_result = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, + surface.clone(), + &mut owner_reservation, + ) + }; + drop(cell_pixel_lifecycle); + if let Err(error) = insert_result { + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); + } + Ok(surface) + } + + fn spawn_browser_surface_with_resource_identity( + self: &Arc, + url: String, + size: Option<(u16, u16)>, + pending_workspace: Option, + resource_identity: Option, + ) -> anyhow::Result> { + let mut owner_reservation = self.reserve_surface_owner()?; + let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); + let id = self.next_id(); + if let Some(workspace) = pending_workspace { + self.pending_workspace_surfaces.lock().unwrap().insert(id, workspace); + } + let opts = self.surface_options.lock().unwrap().clone(); + let size = self.resolve_client_size(size, (opts.cols, opts.rows)); + let cell_pixels = self.cell_pixel_creation_size(); + let surface = match resource_identity { + Some(identity) => browser::new_surface_with_resource_identity( + id, + url.clone(), + size, + cell_pixels, + &opts, + Arc::downgrade(self), + identity, + )?, + None => browser::new_surface( + id, + url.clone(), + size, + cell_pixels, + &opts, + Arc::downgrade(self), + )?, + }; + let insertion = { + let mut state = self.state.lock().unwrap(); + insert_reserved_surface_checked( + self, + &mut state, + surface.clone(), + &mut owner_reservation, + ) + }; + if let Err(error) = insertion { + self.pending_workspace_surfaces.lock().unwrap().remove(&id); + self.retire_reserved_surface_runtime(surface, &mut owner_reservation); + return Err(error); + } + let tab_id = surface + .resource_identity() + .context("browser surface omitted its public tab identity")? + .tab_id + .clone(); + self.start_browser_bootstrap( + surface.clone(), + BrowserBootstrap::Provider { tab_id, url }, + None, + ); + Ok(surface) + } + + fn resolve_client_size( + &self, + requested: Option<(u16, u16)>, + default: (u16, u16), + ) -> (u16, u16) { + let mut sizing = self.client_sizing.lock().unwrap(); + if let Some((cols, rows)) = requested { + let size = clamp_terminal_size(cols, rows); + sizing.record_explicit_size(size); + return size; + } + let attached_clients = self.control_clients.attached_client_ids_by_surface(); + sizing + .creation_size(&attached_clients) + .unwrap_or_else(|| clamp_terminal_size(default.0, default.1)) + } + + /// Record a genuine client-chosen size (protocol resize-surface, sized + /// creation, or the local TUI sizing a pane) as the default for future + /// unsized surface creation. + pub fn record_client_size(&self, cols: u16, rows: u16) -> (u16, u16) { + let size = clamp_terminal_size(cols, rows); + self.client_sizing.lock().unwrap().record_explicit_size(size); + size + } + + /// Record one viewer's available grid. A terminal report changes its PTY + /// only when that client and placement hold explicit geometry authority; + /// browser surfaces retain their existing shared-size reducer. + pub fn resize_surface_for_client( + &self, + id: SurfaceId, + client: u64, + cols: u16, + rows: u16, + ) -> anyhow::Result { + self.resize_surface_for_client_with_reservation(id, client, cols, rows) + .map(|(accepted, _)| accepted) + } + + pub fn resize_surface_for_client_with_reservation( + &self, + id: SurfaceId, + client: u64, + cols: u16, + rows: u16, + ) -> anyhow::Result<(bool, Option)> { + let requested = clamp_terminal_size(cols, rows); + let terminal_runtime = self.surface(id).and_then(|surface| surface.terminal_runtime_id()); + // Serialize the report and its application. Otherwise an older + // effective size can reach the PTY after a newer shared minimum. + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(id); + let result = self.resize_surface_for_client_locked( + &mut sizing, + Some(&attached_clients), + ClientResizeRequest { + surface: id, + client, + requested, + completion: None, + terminal_runtime, + }, + )?; + sizing.note_applied_report( + id, + client, + &attached_clients, + result.1, + result.2.applied_report_order, + ); + drop(sizing); + Ok(result.0) + } + + pub(crate) fn resize_surface_for_control_client_with_reservation( + &self, + id: SurfaceId, + client: u64, + cols: u16, + rows: u16, + ) -> anyhow::Result { + self.resize_surface_for_control_client_with_completion(id, client, cols, rows, None) + } + + pub(crate) fn resize_surface_for_control_client_with_completion( + &self, + id: SurfaceId, + client: u64, + cols: u16, + rows: u16, + completion: Option, + ) -> anyhow::Result { + let _lifecycle = self.lock_client_sizing_lifecycle(); + anyhow::ensure!( + !self.control_clients.surface_attachment_is_retired_without_current(client, id), + "surface {id} attachment was superseded" + ); + let requested = clamp_terminal_size(cols, rows); + let terminal_runtime = self.surface(id).and_then(|surface| surface.terminal_runtime_id()); + // Keep registration, report insertion, and reducer insertion in one + // critical section. Disconnect and final stream detach remove their + // leases through this same sizing lock after dropping the registry lock. + let mut sizing = self.client_sizing.lock().unwrap(); + let attached = self.control_clients.record_size(client, id, requested.0, requested.1)?; + let result = self.resize_surface_for_prepared_control_client_locked( + &mut sizing, + PreparedControlClientResize { + request: ClientResizeRequest { + surface: id, + client, + requested, + completion, + terminal_runtime, + }, + attached: attached.clone(), + }, + ); + if result.is_err() + && let Some((_, _, _, previous)) = attached.as_ref() + { + self.control_clients.restore_size(client, id, *previous); } + drop(sizing); + result + } - /// Record a genuine client-chosen size (protocol resize-surface, sized - /// creation, or the local TUI sizing a pane) as the default for future - /// unsized surface creation. - pub fn record_client_size(&self, cols: u16, rows: u16) -> (u16, u16) { - let size = clamp_terminal_size(cols, rows); - self.client_sizing.lock().unwrap().record_explicit_size(size); - size - } - - /// Record one viewer's available grid. A terminal report changes its PTY - /// only when that client and placement hold explicit geometry authority; - /// browser surfaces retain their existing shared-size reducer. - pub fn resize_surface_for_client( - &self, - id: SurfaceId, - client: u64, - cols: u16, - rows: u16, - ) -> anyhow::Result { - self.resize_surface_for_client_with_reservation(id, client, cols, rows) - .map(|(accepted, _)| accepted) - } - - pub fn resize_surface_for_client_with_reservation( - &self, - id: SurfaceId, - client: u64, - cols: u16, - rows: u16, - ) -> anyhow::Result<(bool, Option)> { - let requested = clamp_terminal_size(cols, rows); - let terminal_runtime = - self.surface(id).and_then(|surface| surface.terminal_runtime_id()); - // Serialize the report and its application. Otherwise an older - // effective size can reach the PTY after a newer shared minimum. - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(id); - let result = self.resize_surface_for_client_locked( - &mut sizing, - Some(&attached_clients), - ClientResizeRequest { + pub(crate) fn resize_surface_for_prepared_control_client_with_completion( + &self, + id: SurfaceId, + client: u64, + requested: (u16, u16), + completion: Option, + attached: Option, + ) -> anyhow::Result { + let requested = clamp_terminal_size(requested.0, requested.1); + let terminal_runtime = self.surface(id).and_then(|surface| surface.terminal_runtime_id()); + let mut sizing = self.client_sizing.lock().unwrap(); + let result = self.resize_surface_for_prepared_control_client_locked( + &mut sizing, + PreparedControlClientResize { + request: ClientResizeRequest { surface: id, client, requested, - completion: None, + completion, terminal_runtime, }, - )?; - sizing.note_applied_report( - id, - client, - &attached_clients, - result.1, - result.2.applied_report_order, - ); - drop(sizing); - Ok(result.0) - } - - pub(crate) fn resize_surface_for_control_client_with_reservation( - &self, - id: SurfaceId, - client: u64, - cols: u16, - rows: u16, - ) -> anyhow::Result { - self.resize_surface_for_control_client_with_completion(id, client, cols, rows, None) - } - - pub(crate) fn resize_surface_for_control_client_with_completion( - &self, - id: SurfaceId, - client: u64, - cols: u16, - rows: u16, - completion: Option, - ) -> anyhow::Result { - let _lifecycle = self.lock_client_sizing_lifecycle(); - anyhow::ensure!( - !self.control_clients.surface_attachment_is_retired_without_current(client, id), - "surface {id} attachment was superseded" - ); - let requested = clamp_terminal_size(cols, rows); - let terminal_runtime = - self.surface(id).and_then(|surface| surface.terminal_runtime_id()); - // Keep registration, report insertion, and reducer insertion in one - // critical section. Disconnect and final stream detach remove their - // leases through this same sizing lock after dropping the registry lock. - let mut sizing = self.client_sizing.lock().unwrap(); - let attached = - self.control_clients.record_size(client, id, requested.0, requested.1)?; - let result = self.resize_surface_for_prepared_control_client_locked( - &mut sizing, - PreparedControlClientResize { - request: ClientResizeRequest { - surface: id, - client, - requested, - completion, - terminal_runtime, - }, - attached: attached.clone(), - }, - ); - if result.is_err() - && let Some((_, _, _, previous)) = attached.as_ref() - { - self.control_clients.restore_size(client, id, *previous); - } - drop(sizing); - result - } + attached, + }, + ); + drop(sizing); + result + } - pub(crate) fn resize_surface_for_prepared_control_client_with_completion( - &self, - id: SurfaceId, - client: u64, - requested: (u16, u16), - completion: Option, - attached: Option, - ) -> anyhow::Result { - let requested = clamp_terminal_size(requested.0, requested.1); - let terminal_runtime = - self.surface(id).and_then(|surface| surface.terminal_runtime_id()); - let mut sizing = self.client_sizing.lock().unwrap(); - let result = self.resize_surface_for_prepared_control_client_locked( - &mut sizing, - PreparedControlClientResize { - request: ClientResizeRequest { - surface: id, - client, - requested, - completion, - terminal_runtime, - }, - attached, - }, - ); - drop(sizing); - result - } + fn resize_surface_for_prepared_control_client_locked( + &self, + sizing: &mut ClientSizingState, + prepared: PreparedControlClientResize, + ) -> anyhow::Result { + let PreparedControlClientResize { request, attached } = prepared; + let id = request.surface; + let client = request.client; + let attached_clients = self.control_clients.attached_client_ids_for_surface(id); + let result = + self.resize_surface_for_client_locked(sizing, Some(&attached_clients), request); + let result = result?; + self.control_clients.set_report_order(client, id, result.2.applied_report_order); + sizing.note_applied_report( + id, + client, + &attached_clients, + result.1, + result.2.applied_report_order, + ); + Ok(ControlClientResize { + accepted: result.0.0, + reservation_id: result.0.1, + effective_size: result.1, + attached, + rollback: result.2, + }) + } - fn resize_surface_for_prepared_control_client_locked( - &self, - sizing: &mut ClientSizingState, - prepared: PreparedControlClientResize, - ) -> anyhow::Result { - let PreparedControlClientResize { request, attached } = prepared; - let id = request.surface; - let client = request.client; - let attached_clients = self.control_clients.attached_client_ids_for_surface(id); - let result = - self.resize_surface_for_client_locked(sizing, Some(&attached_clients), request); - let result = result?; - self.control_clients.set_report_order(client, id, result.2.applied_report_order); - sizing.note_applied_report( - id, - client, - &attached_clients, - result.1, - result.2.applied_report_order, - ); - Ok(ControlClientResize { - accepted: result.0.0, - reservation_id: result.0.1, - effective_size: result.1, - attached, - rollback: result.2, - }) + fn resize_surface_for_client_locked( + &self, + sizing: &mut ClientSizingState, + attached_clients: Option<&HashSet>, + request: ClientResizeRequest, + ) -> anyhow::Result { + let ClientResizeRequest { surface: id, client, requested, completion, terminal_runtime } = + request; + let previous_geometry = self.surface(id).map(|surface| surface.size()); + if terminal_runtime.is_none() + && sizing + .policies + .get(&id) + .and_then(|policy| policy.exclusive_client) + .is_some_and(|exclusive| exclusive != client) + { + sizing.policies.entry(id).or_default().excluded_clients.insert(client); } - - fn resize_surface_for_client_locked( - &self, - sizing: &mut ClientSizingState, - attached_clients: Option<&HashSet>, - request: ClientResizeRequest, - ) -> anyhow::Result { - let ClientResizeRequest { - surface: id, - client, - requested, - completion, - terminal_runtime, - } = request; - let previous_geometry = self.surface(id).map(|surface| surface.size()); - if terminal_runtime.is_none() - && sizing - .policies - .get(&id) - .and_then(|policy| policy.exclusive_client) - .is_some_and(|exclusive| exclusive != client) - { - sizing.policies.entry(id).or_default().excluded_clients.insert(client); - } - let report_order = sizing.next_size_order(); - let previous_order = sizing.report_order.insert((id, client), report_order); - let previous = { - let viewers = sizing.surfaces.entry(id).or_default(); - viewers.insert(client, requested) - }; - if let Some(runtime) = terminal_runtime { - sizing.terminal_runtime_by_placement.insert(id, runtime); - let authoritative = sizing.owns_terminal_geometry(runtime, id, client); - if !authoritative { - return Ok(( - (false, None), - previous_geometry, - ClientSizeRollback { - previous_size: previous, - previous_report_order: previous_order, - previous_geometry, - applied_report_order: report_order, - }, - )); - } - return match self.resize_surface_with_completion( - id, - requested.0, - requested.1, - completion, - ) { - Ok(changed) => Ok(( - changed, - Some(requested), - ClientSizeRollback { - previous_size: previous, - previous_report_order: previous_order, - previous_geometry, - applied_report_order: report_order, - }, - )), - Err(error) => { - if let Some(viewers) = sizing.surfaces.get_mut(&id) { - if let Some(previous) = previous { - viewers.insert(client, previous); - } else { - viewers.remove(&client); - } - } - if sizing.surfaces.get(&id).is_some_and(HashMap::is_empty) { - sizing.surfaces.remove(&id); - sizing.terminal_runtime_by_placement.remove(&id); - } - match previous_order { - Some(order) => { - sizing.report_order.insert((id, client), order); - } - None => { - sizing.report_order.remove(&(id, client)); - } - } - Err(error) - } - }; - } - let use_excluded = sizing.uses_excluded_fallback(id, attached_clients); - let effective = sizing.effective_size(id, use_excluded); - let Some(effective) = effective else { + let report_order = sizing.next_size_order(); + let previous_order = sizing.report_order.insert((id, client), report_order); + let previous = { + let viewers = sizing.surfaces.entry(id).or_default(); + viewers.insert(client, requested) + }; + if let Some(runtime) = terminal_runtime { + sizing.terminal_runtime_by_placement.insert(id, runtime); + let authoritative = sizing.owns_terminal_geometry(runtime, id, client); + if !authoritative { return Ok(( (false, None), - None, + previous_geometry, ClientSizeRollback { previous_size: previous, previous_report_order: previous_order, @@ -8013,17 +7836,16 @@ impl Mux { applied_report_order: report_order, }, )); - }; - #[cfg(test)] - let before_apply = self.client_resize_before_apply.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = before_apply { - hook(); } - match self.resize_surface_with_completion(id, effective.0, effective.1, completion) { + return match self.resize_surface_with_completion( + id, + requested.0, + requested.1, + completion, + ) { Ok(changed) => Ok(( changed, - Some(effective), + Some(requested), ClientSizeRollback { previous_size: previous, previous_report_order: previous_order, @@ -8038,1339 +7860,1337 @@ impl Mux { } else { viewers.remove(&client); } - if viewers.is_empty() { - sizing.surfaces.remove(&id); - } } - if let Some(previous_order) = previous_order { - sizing.report_order.insert((id, client), previous_order); - } else { - sizing.report_order.remove(&(id, client)); + if sizing.surfaces.get(&id).is_some_and(HashMap::is_empty) { + sizing.surfaces.remove(&id); + sizing.terminal_runtime_by_placement.remove(&id); + } + match previous_order { + Some(order) => { + sizing.report_order.insert((id, client), order); + } + None => { + sizing.report_order.remove(&(id, client)); + } } Err(error) } - } + }; } - - pub(crate) fn rollback_surface_size_client( - &self, - id: SurfaceId, - client: u64, - rollback: ClientSizeRollback, - ) { - let terminal_runtime = - self.surface(id).and_then(|surface| surface.terminal_runtime_id()); - let lifecycle = self.lock_client_sizing_lifecycle(); - if !self.control_clients.contains(client) { - return; - } - let mut sizing = self.client_sizing.lock().unwrap(); - let current_size = - sizing.surfaces.get(&id).and_then(|viewers| viewers.get(&client).copied()); - let current_report_order = sizing.report_order.get(&(id, client)).copied(); - if current_report_order != Some(rollback.applied_report_order) { - return; - } - self.control_clients.restore_size_and_report_order( - client, - id, - rollback.previous_size, - rollback.previous_report_order, - ); - match rollback.previous_size { - Some(size) => { - sizing.surfaces.entry(id).or_default().insert(client, size); - } - None => { - if let Some(viewers) = sizing.surfaces.get_mut(&id) { + let use_excluded = sizing.uses_excluded_fallback(id, attached_clients); + let effective = sizing.effective_size(id, use_excluded); + let Some(effective) = effective else { + return Ok(( + (false, None), + None, + ClientSizeRollback { + previous_size: previous, + previous_report_order: previous_order, + previous_geometry, + applied_report_order: report_order, + }, + )); + }; + #[cfg(test)] + let before_apply = self.client_resize_before_apply.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = before_apply { + hook(); + } + match self.resize_surface_with_completion(id, effective.0, effective.1, completion) { + Ok(changed) => Ok(( + changed, + Some(effective), + ClientSizeRollback { + previous_size: previous, + previous_report_order: previous_order, + previous_geometry, + applied_report_order: report_order, + }, + )), + Err(error) => { + if let Some(viewers) = sizing.surfaces.get_mut(&id) { + if let Some(previous) = previous { + viewers.insert(client, previous); + } else { viewers.remove(&client); - if viewers.is_empty() { - sizing.surfaces.remove(&id); - } + } + if viewers.is_empty() { + sizing.surfaces.remove(&id); } } - } - match rollback.previous_report_order { - Some(order) => { - sizing.report_order.insert((id, client), order); - } - None => { + if let Some(previous_order) = previous_order { + sizing.report_order.insert((id, client), previous_order); + } else { sizing.report_order.remove(&(id, client)); } + Err(error) } - if let Some(runtime) = terminal_runtime { - let owns_geometry = sizing.owns_terminal_geometry(runtime, id, client); - let desired_geometry = owns_geometry - .then_some(rollback.previous_size.or(rollback.previous_geometry)) - .flatten(); - drop(sizing); - drop(lifecycle); - if let Some((cols, rows)) = desired_geometry { - let _ = self.resize_surface(id, cols, rows); - } - return; - } - let attached_clients = self.control_clients.attached_client_ids_for_surface(id); - let use_excluded = sizing.uses_excluded_fallback(id, Some(&attached_clients)); - let desired_geometry = - sizing.effective_size(id, use_excluded).or(rollback.previous_geometry); - let restore = - desired_geometry.map_or(SurfaceResizeRestore::Complete(true), |(cols, rows)| { - let (completion, completed) = std::sync::mpsc::sync_channel(1); - match self.resize_surface_with_completion(id, cols, rows, Some(completion)) { - Ok((true, Some(_))) => SurfaceResizeRestore::Pending(completed), - Ok((_, _)) => match self.surface(id) { - Some(surface) if surface.size() == (cols, rows) => { - SurfaceResizeRestore::Complete(true) - } - Some(surface) => match surface.pending_resize_completion(cols, rows) { - Ok(Some(pending)) => { - SurfaceResizeRestore::Pending(pending.completion) - } - Ok(None) | Err(_) => SurfaceResizeRestore::Complete(false), - }, - None => SurfaceResizeRestore::Complete(false), - }, - Err(_) => SurfaceResizeRestore::Complete(false), - } - }); - let rollback_token = sizing.rollback_token(id, Some(&attached_clients)); - drop(sizing); - drop(lifecycle); + } + } - #[cfg(test)] - if let Some(hook) = self.client_rollback_before_wait.lock().unwrap().clone() { - hook(); + pub(crate) fn rollback_surface_size_client( + &self, + id: SurfaceId, + client: u64, + rollback: ClientSizeRollback, + ) { + let terminal_runtime = self.surface(id).and_then(|surface| surface.terminal_runtime_id()); + let lifecycle = self.lock_client_sizing_lifecycle(); + if !self.control_clients.contains(client) { + return; + } + let mut sizing = self.client_sizing.lock().unwrap(); + let current_size = + sizing.surfaces.get(&id).and_then(|viewers| viewers.get(&client).copied()); + let current_report_order = sizing.report_order.get(&(id, client)).copied(); + if current_report_order != Some(rollback.applied_report_order) { + return; + } + self.control_clients.restore_size_and_report_order( + client, + id, + rollback.previous_size, + rollback.previous_report_order, + ); + match rollback.previous_size { + Some(size) => { + sizing.surfaces.entry(id).or_default().insert(client, size); } - - let restoration_failed = match restore { - SurfaceResizeRestore::Complete(restored) => !restored, - SurfaceResizeRestore::Pending(completion) => { - match completion.recv_timeout(Duration::from_secs(10)) { - Ok(Ok(())) => false, - Ok(Err(_)) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => true, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - // The rolled-back registry remains authoritative while - // the compensating browser reservation stays queued. - // Do not reinstall the failed attach's claim before the - // browser worker reaches a terminal outcome, and do not - // retain the connection or another blocking waiter. - return; - } + None => { + if let Some(viewers) = sizing.surfaces.get_mut(&id) { + viewers.remove(&client); + if viewers.is_empty() { + sizing.surfaces.remove(&id); } } - }; - if restoration_failed { - self.reconcile_failed_surface_size_rollback( - id, - client, - current_size, - current_report_order, - rollback_token, - ); } } - - fn reconcile_failed_surface_size_rollback( - &self, - id: SurfaceId, - client: u64, - current_size: Option<(u16, u16)>, - current_report_order: Option, - rollback_token: ClientSizingRollbackToken, - ) { - let _lifecycle = self.lock_client_sizing_lifecycle(); - if !self.control_clients.contains(client) { - return; - } - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(id); - if sizing.rollback_token(id, Some(&attached_clients)) != rollback_token { - return; - } - // The failed attach already changed the real surface geometry. If - // restoration fails, retain the pre-rollback report only when no - // newer sizing mutation superseded this rollback while it was pending. - self.control_clients.restore_size_and_report_order( - client, - id, - current_size, - current_report_order, - ); - match current_size { - Some(size) => { - sizing.surfaces.entry(id).or_default().insert(client, size); - } - None => { - if let Some(viewers) = sizing.surfaces.get_mut(&id) { - viewers.remove(&client); - if viewers.is_empty() { - sizing.surfaces.remove(&id); - } - } - } + match rollback.previous_report_order { + Some(order) => { + sizing.report_order.insert((id, client), order); } - match current_report_order { - Some(order) => { - sizing.report_order.insert((id, client), order); - } - None => { - sizing.report_order.remove(&(id, client)); - } + None => { + sizing.report_order.remove(&(id, client)); } } - - fn apply_effective_client_size( - &self, - sizing: &ClientSizingState, - surface_id: SurfaceId, - attached_clients: Option<&HashSet>, - ) { - if let Some(runtime) = - self.surface(surface_id).and_then(|surface| surface.terminal_runtime_id()) - { - let Some(authority) = sizing.terminal_authorities.get(&runtime) else { - return; - }; - let Some((cols, rows)) = sizing - .surfaces - .get(&authority.placement) - .and_then(|viewers| viewers.get(&authority.client)) - .copied() - else { - return; - }; - let _ = self.resize_surface(authority.placement, cols, rows); - return; - } - let use_excluded = sizing.uses_excluded_fallback(surface_id, attached_clients); - if let Some((cols, rows)) = sizing.effective_size(surface_id, use_excluded) { - let _ = self.resize_surface(surface_id, cols, rows); - } else if let Some(surface) = self.surface(surface_id) { - let _ = surface.release_viewer_size(); + if let Some(runtime) = terminal_runtime { + let owns_geometry = sizing.owns_terminal_geometry(runtime, id, client); + let desired_geometry = owns_geometry + .then_some(rollback.previous_size.or(rollback.previous_geometry)) + .flatten(); + drop(sizing); + drop(lifecycle); + if let Some((cols, rows)) = desired_geometry { + let _ = self.resize_surface(id, cols, rows); } + return; } + let attached_clients = self.control_clients.attached_client_ids_for_surface(id); + let use_excluded = sizing.uses_excluded_fallback(id, Some(&attached_clients)); + let desired_geometry = + sizing.effective_size(id, use_excluded).or(rollback.previous_geometry); + let restore = + desired_geometry.map_or(SurfaceResizeRestore::Complete(true), |(cols, rows)| { + let (completion, completed) = std::sync::mpsc::sync_channel(1); + match self.resize_surface_with_completion(id, cols, rows, Some(completion)) { + Ok((true, Some(_))) => SurfaceResizeRestore::Pending(completed), + Ok((_, _)) => match self.surface(id) { + Some(surface) if surface.size() == (cols, rows) => { + SurfaceResizeRestore::Complete(true) + } + Some(surface) => match surface.pending_resize_completion(cols, rows) { + Ok(Some(pending)) => SurfaceResizeRestore::Pending(pending.completion), + Ok(None) | Err(_) => SurfaceResizeRestore::Complete(false), + }, + None => SurfaceResizeRestore::Complete(false), + }, + Err(_) => SurfaceResizeRestore::Complete(false), + } + }); + let rollback_token = sizing.rollback_token(id, Some(&attached_clients)); + drop(sizing); + drop(lifecycle); - fn apply_effective_client_sizes( - &self, - sizing: &ClientSizingState, - affected: impl IntoIterator, - attached_clients: &HashMap>, - ) { - let mut affected = affected.into_iter().collect::>(); - affected.sort_unstable(); - affected.dedup(); - for surface_id in affected { - self.apply_effective_client_size( - sizing, - surface_id, - attached_clients.get(&surface_id), - ); - } + #[cfg(test)] + if let Some(hook) = self.client_rollback_before_wait.lock().unwrap().clone() { + hook(); } - pub fn remove_surface_size_client(&self, id: SurfaceId, client: u64) { - // Removal participates in the same ordering as size reports. - let terminal_runtime = - self.surface(id).and_then(|surface| surface.terminal_runtime_id()); - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(id); - // Final-stream cleanup runs after the registry removes this - // attachment. Reconstruct the preceding attachment set so an - // unreported client only triggers geometry when its removal actually - // changes excluded-report fallback. - let mut attached_clients_before = attached_clients.clone(); - attached_clients_before.insert(client); - let fallback_before = sizing.uses_excluded_fallback(id, Some(&attached_clients_before)); - let removed = { - let removed = sizing - .surfaces - .get_mut(&id) - .is_some_and(|viewers| viewers.remove(&client).is_some()); - if sizing.surfaces.get(&id).is_some_and(HashMap::is_empty) { - sizing.surfaces.remove(&id); - sizing.terminal_runtime_by_placement.remove(&id); - } - removed - }; - sizing.report_order.remove(&(id, client)); - if let Some(runtime) = terminal_runtime { - let released = sizing.owns_terminal_geometry(runtime, id, client); - if released { - sizing.terminal_authorities.remove(&runtime); - } - drop(sizing); - if released { - self.emit_client_sizing_changes([client]); + let restoration_failed = match restore { + SurfaceResizeRestore::Complete(restored) => !restored, + SurfaceResizeRestore::Pending(completion) => { + match completion.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(())) => false, + Ok(Err(_)) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => true, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + // The rolled-back registry remains authoritative while + // the compensating browser reservation stays queued. + // Do not reinstall the failed attach's claim before the + // browser worker reaches a terminal outcome, and do not + // retain the connection or another blocking waiter. + return; + } } - return; - } - let fallback_after = sizing.uses_excluded_fallback(id, Some(&attached_clients)); - // A final unreported attachment can be the only thing suppressing - // this terminal's excluded-report fallback even though it had no - // visibility lease of its own to remove. - if !removed && fallback_before == fallback_after { - return; - } - #[cfg(test)] - let before_apply = self.client_resize_before_apply.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = before_apply { - hook(); } - self.apply_effective_client_size(&sizing, id, Some(&attached_clients)); - drop(sizing); + }; + if restoration_failed { + self.reconcile_failed_surface_size_rollback( + id, + client, + current_size, + current_report_order, + rollback_token, + ); } + } - #[cfg(test)] - pub fn remove_size_client(&self, client: u64) { - self.remove_size_client_from_attached_surfaces(client, []); + fn reconcile_failed_surface_size_rollback( + &self, + id: SurfaceId, + client: u64, + current_size: Option<(u16, u16)>, + current_report_order: Option, + rollback_token: ClientSizingRollbackToken, + ) { + let _lifecycle = self.lock_client_sizing_lifecycle(); + if !self.control_clients.contains(client) { + return; } - - pub(crate) fn remove_size_client_from_attached_surfaces( - &self, - client: u64, - attached_surfaces: impl IntoIterator, - ) { - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_by_surface(); - // The registry snapshot no longer contains this client. Reconstruct - // whether each old attachment suppressed excluded-report fallback so - // an unsized disconnect only reapplies geometry when that changed. - let detached_fallbacks = attached_surfaces - .into_iter() - .map(|surface| { - let used_fallback = if sizing.client_participates(surface, client) { - false - } else { - sizing.uses_excluded_fallback(surface, attached_clients.get(&surface)) - }; - (surface, used_fallback) - }) - .collect::>(); - let mut affected = HashSet::new(); - for (surface, viewers) in &mut sizing.surfaces { - if viewers.remove(&client).is_some() { - affected.insert(*surface); - } + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(id); + if sizing.rollback_token(id, Some(&attached_clients)) != rollback_token { + return; + } + // The failed attach already changed the real surface geometry. If + // restoration fails, retain the pre-rollback report only when no + // newer sizing mutation superseded this rollback while it was pending. + self.control_clients.restore_size_and_report_order( + client, + id, + current_size, + current_report_order, + ); + match current_size { + Some(size) => { + sizing.surfaces.entry(id).or_default().insert(client, size); } - sizing.surfaces.retain(|_, viewers| !viewers.is_empty()); - let reported_placements = sizing.surfaces.keys().copied().collect::>(); - sizing - .terminal_runtime_by_placement - .retain(|surface, _| reported_placements.contains(surface)); - sizing.report_order.retain(|(surface, reporter), _| { - if *reporter != client { - return true; - } - affected.insert(*surface); - false - }); - sizing.terminal_authorities.retain(|_, authority| authority.client != client); - let mut restored_surfaces = HashSet::new(); - for (surface, policy) in &mut sizing.policies { - let changed = if policy.exclusive_client == Some(client) { - policy.exclusive_client = None; - policy.excluded_clients.clear(); - restored_surfaces.insert(*surface); - true - } else { - policy.excluded_clients.remove(&client) - }; - if changed { - affected.insert(*surface); + None => { + if let Some(viewers) = sizing.surfaces.get_mut(&id) { + viewers.remove(&client); + if viewers.is_empty() { + sizing.surfaces.remove(&id); + } } } - sizing.policies.retain(|_, policy| { - policy.exclusive_client.is_some() || !policy.excluded_clients.is_empty() - }); - for (surface, fallback_before) in detached_fallbacks { - let fallback_after = - sizing.uses_excluded_fallback(surface, attached_clients.get(&surface)); - if fallback_before != fallback_after { - affected.insert(surface); - } + } + match current_report_order { + Some(order) => { + sizing.report_order.insert((id, client), order); } - let mut changed_clients = HashSet::new(); - for surface in restored_surfaces { - if let Some(clients) = attached_clients.get(&surface) { - changed_clients.extend(clients.iter().copied()); - } - if let Some(reporters) = sizing.surfaces.get(&surface) { - changed_clients.extend(reporters.keys().copied()); - } + None => { + sizing.report_order.remove(&(id, client)); } - changed_clients.remove(&client); - self.apply_effective_client_sizes(&sizing, affected, &attached_clients); - drop(sizing); - self.emit_client_sizing_changes(changed_clients); } + } - pub fn client_surface_size(&self, id: SurfaceId, client: u64) -> Option<(u16, u16)> { - self.client_sizing - .lock() - .unwrap() + fn apply_effective_client_size( + &self, + sizing: &ClientSizingState, + surface_id: SurfaceId, + attached_clients: Option<&HashSet>, + ) { + if let Some(runtime) = + self.surface(surface_id).and_then(|surface| surface.terminal_runtime_id()) + { + let Some(authority) = sizing.terminal_authorities.get(&runtime) else { + return; + }; + let Some((cols, rows)) = sizing .surfaces - .get(&id) - .and_then(|viewers| viewers.get(&client).copied()) + .get(&authority.placement) + .and_then(|viewers| viewers.get(&authority.client)) + .copied() + else { + return; + }; + let _ = self.resize_surface(authority.placement, cols, rows); + return; + } + let use_excluded = sizing.uses_excluded_fallback(surface_id, attached_clients); + if let Some((cols, rows)) = sizing.effective_size(surface_id, use_excluded) { + let _ = self.resize_surface(surface_id, cols, rows); + } else if let Some(surface) = self.surface(surface_id) { + let _ = surface.release_viewer_size(); } + } - /// Assign canonical PTY geometry to one explicit client/placement pair. - /// Other reports remain viewport hints and never resize the shared PTY. - pub fn claim_terminal_geometry(&self, surface: SurfaceId, client: u64) -> Option { - let _lifecycle = self.lock_client_sizing_lifecycle(); - let runtime = self.surface(surface)?.terminal_runtime_id()?; - if client != 0 && !self.control_clients.contains(client) { - return None; + fn apply_effective_client_sizes( + &self, + sizing: &ClientSizingState, + affected: impl IntoIterator, + attached_clients: &HashMap>, + ) { + let mut affected = affected.into_iter().collect::>(); + affected.sort_unstable(); + affected.dedup(); + for surface_id in affected { + self.apply_effective_client_size(sizing, surface_id, attached_clients.get(&surface_id)); + } + } + + pub fn remove_surface_size_client(&self, id: SurfaceId, client: u64) { + // Removal participates in the same ordering as size reports. + let terminal_runtime = self.surface(id).and_then(|surface| surface.terminal_runtime_id()); + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(id); + // Final-stream cleanup runs after the registry removes this + // attachment. Reconstruct the preceding attachment set so an + // unreported client only triggers geometry when its removal actually + // changes excluded-report fallback. + let mut attached_clients_before = attached_clients.clone(); + attached_clients_before.insert(client); + let fallback_before = sizing.uses_excluded_fallback(id, Some(&attached_clients_before)); + let removed = { + let removed = sizing + .surfaces + .get_mut(&id) + .is_some_and(|viewers| viewers.remove(&client).is_some()); + if sizing.surfaces.get(&id).is_some_and(HashMap::is_empty) { + sizing.surfaces.remove(&id); + sizing.terminal_runtime_by_placement.remove(&id); } - let mut sizing = self.client_sizing.lock().unwrap(); - let authority = TerminalGeometryAuthority { placement: surface, client }; - sizing.terminal_runtime_by_placement.insert(surface, runtime); - let previous = sizing.terminal_authorities.insert(runtime, authority); - let changed = previous != Some(authority); - let claimed_size = - sizing.surfaces.get(&surface).and_then(|viewers| viewers.get(&client)).copied(); - drop(sizing); - if let Some((cols, rows)) = claimed_size { - let _ = self.resize_surface(surface, cols, rows); + removed + }; + sizing.report_order.remove(&(id, client)); + if let Some(runtime) = terminal_runtime { + let released = sizing.owns_terminal_geometry(runtime, id, client); + if released { + sizing.terminal_authorities.remove(&runtime); } - if changed { - let mut changed_clients = vec![client]; - if let Some(previous) = previous - && previous.client != client - { - changed_clients.push(previous.client); - } - self.emit_client_sizing_changes(changed_clients); + drop(sizing); + if released { + self.emit_client_sizing_changes([client]); } - Some(changed) + return; + } + let fallback_after = sizing.uses_excluded_fallback(id, Some(&attached_clients)); + // A final unreported attachment can be the only thing suppressing + // this terminal's excluded-report fallback even though it had no + // visibility lease of its own to remove. + if !removed && fallback_before == fallback_after { + return; + } + #[cfg(test)] + let before_apply = self.client_resize_before_apply.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = before_apply { + hook(); } + self.apply_effective_client_size(&sizing, id, Some(&attached_clients)); + drop(sizing); + } - pub fn release_terminal_geometry(&self, surface: SurfaceId) -> Option { - let _lifecycle = self.lock_client_sizing_lifecycle(); - let runtime = self.surface(surface)?.terminal_runtime_id()?; - let mut sizing = self.client_sizing.lock().unwrap(); - let owner = sizing.terminal_authorities.get(&runtime).copied(); - if owner.is_some_and(|authority| { - authority.client != 0 && !self.control_clients.contains(authority.client) - }) { - return None; + #[cfg(test)] + pub fn remove_size_client(&self, client: u64) { + self.remove_size_client_from_attached_surfaces(client, []); + } + + pub(crate) fn remove_size_client_from_attached_surfaces( + &self, + client: u64, + attached_surfaces: impl IntoIterator, + ) { + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_by_surface(); + // The registry snapshot no longer contains this client. Reconstruct + // whether each old attachment suppressed excluded-report fallback so + // an unsized disconnect only reapplies geometry when that changed. + let detached_fallbacks = attached_surfaces + .into_iter() + .map(|surface| { + let used_fallback = if sizing.client_participates(surface, client) { + false + } else { + sizing.uses_excluded_fallback(surface, attached_clients.get(&surface)) + }; + (surface, used_fallback) + }) + .collect::>(); + let mut affected = HashSet::new(); + for (surface, viewers) in &mut sizing.surfaces { + if viewers.remove(&client).is_some() { + affected.insert(*surface); } - let released = sizing.terminal_authorities.remove(&runtime); - drop(sizing); - if let Some(released) = released { - self.emit_client_sizing_changes([released.client]); - Some(true) + } + sizing.surfaces.retain(|_, viewers| !viewers.is_empty()); + let reported_placements = sizing.surfaces.keys().copied().collect::>(); + sizing + .terminal_runtime_by_placement + .retain(|surface, _| reported_placements.contains(surface)); + sizing.report_order.retain(|(surface, reporter), _| { + if *reporter != client { + return true; + } + affected.insert(*surface); + false + }); + sizing.terminal_authorities.retain(|_, authority| authority.client != client); + let mut restored_surfaces = HashSet::new(); + for (surface, policy) in &mut sizing.policies { + let changed = if policy.exclusive_client == Some(client) { + policy.exclusive_client = None; + policy.excluded_clients.clear(); + restored_surfaces.insert(*surface); + true } else { - Some(false) + policy.excluded_clients.remove(&client) + }; + if changed { + affected.insert(*surface); } } - - fn emit_client_sizing_changes(&self, clients: impl IntoIterator) { - for client in clients { - let (name, kind) = self.control_clients.client_info(client).unwrap_or((None, None)); - self.emit(MuxEvent::ClientChanged { client, name, kind }); + sizing.policies.retain(|_, policy| { + policy.exclusive_client.is_some() || !policy.excluded_clients.is_empty() + }); + for (surface, fallback_before) in detached_fallbacks { + let fallback_after = + sizing.uses_excluded_fallback(surface, attached_clients.get(&surface)); + if fallback_before != fallback_after { + affected.insert(surface); } } - - /// Claim or release a terminal's canonical geometry for one live client, - /// or update the legacy shared-size policy for a non-terminal surface. - pub fn set_client_size_participation( - &self, - surface: SurfaceId, - client: u64, - participating: bool, - ) -> Option { - if self.surface(surface)?.terminal_runtime_id().is_some() { - if participating { - return self.claim_terminal_geometry(surface, client); - } - let _lifecycle = self.lock_client_sizing_lifecycle(); - // Revalidate after acquiring the sizing lifecycle fence. A - // disconnect may have removed the client while this action was - // waiting for the fence, and a stale release must not report a - // successful no-op against a dead client. - if client != 0 && !self.control_clients.contains(client) { - return None; - } - let runtime = self.surface(surface)?.terminal_runtime_id()?; - let mut sizing = self.client_sizing.lock().unwrap(); - let owns = sizing.owns_terminal_geometry(runtime, surface, client); - if owns { - sizing.terminal_authorities.remove(&runtime); - drop(sizing); - self.emit_client_sizing_changes([client]); - } - return Some(owns); + let mut changed_clients = HashSet::new(); + for surface in restored_surfaces { + if let Some(clients) = attached_clients.get(&surface) { + changed_clients.extend(clients.iter().copied()); } - let _lifecycle = self.lock_client_sizing_lifecycle(); - self.surface(surface)?; - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); - let mut known_clients = attached_clients.clone(); if let Some(reporters) = sizing.surfaces.get(&surface) { - known_clients.extend(reporters.keys().copied()); - } - if !known_clients.contains(&client) { - return None; + changed_clients.extend(reporters.keys().copied()); } - if sizing.client_participates(surface, client) == participating { - return Some(false); - } - let policy = sizing.policies.entry(surface).or_default(); - if let Some(exclusive) = policy.exclusive_client.take() { - policy.excluded_clients.extend( - known_clients.iter().copied().filter(|candidate| *candidate != exclusive), - ); - policy.excluded_clients.remove(&exclusive); - } - if participating { - policy.excluded_clients.remove(&client); - } else { - policy.excluded_clients.insert(client); - } - if policy.excluded_clients.is_empty() { - sizing.policies.remove(&surface); - } - self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); - drop(sizing); - self.emit_client_sizing_changes([client]); - Some(true) } + changed_clients.remove(&client); + self.apply_effective_client_sizes(&sizing, affected, &attached_clients); + drop(sizing); + self.emit_client_sizing_changes(changed_clients); + } - /// Atomically grant one client/placement canonical terminal geometry. - pub fn use_only_client_size(&self, surface: SurfaceId, target: u64) -> Option { - if self.surface(surface)?.terminal_runtime_id().is_some() { - if target != 0 - && !self - .control_clients - .attached_client_ids_for_surface(surface) - .contains(&target) - { - return None; - } - self.client_surface_size(surface, target)?; - return self.claim_terminal_geometry(surface, target); - } - let _lifecycle = self.lock_client_sizing_lifecycle(); - self.surface(surface)?; - let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); - let reporters = sizing.surfaces.get(&surface); - let target_is_reporting = - reporters.is_some_and(|viewers| viewers.contains_key(&target)); - if !target_is_reporting { - return None; - } - let mut known_clients = attached_clients.clone(); - if let Some(reporters) = reporters { - known_clients.extend(reporters.keys().copied()); - } - let excluded = known_clients - .iter() - .copied() - .filter(|client| *client != target) - .collect::>(); - let policy = sizing.policies.entry(surface).or_default(); - if policy.excluded_clients == excluded && policy.exclusive_client == Some(target) { - return Some(false); + pub fn client_surface_size(&self, id: SurfaceId, client: u64) -> Option<(u16, u16)> { + self.client_sizing + .lock() + .unwrap() + .surfaces + .get(&id) + .and_then(|viewers| viewers.get(&client).copied()) + } + + /// Assign canonical PTY geometry to one explicit client/placement pair. + /// Other reports remain viewport hints and never resize the shared PTY. + pub fn claim_terminal_geometry(&self, surface: SurfaceId, client: u64) -> Option { + let _lifecycle = self.lock_client_sizing_lifecycle(); + let runtime = self.surface(surface)?.terminal_runtime_id()?; + if client != 0 && !self.control_clients.contains(client) { + return None; + } + let mut sizing = self.client_sizing.lock().unwrap(); + let authority = TerminalGeometryAuthority { placement: surface, client }; + sizing.terminal_runtime_by_placement.insert(surface, runtime); + let previous = sizing.terminal_authorities.insert(runtime, authority); + let changed = previous != Some(authority); + let claimed_size = + sizing.surfaces.get(&surface).and_then(|viewers| viewers.get(&client)).copied(); + drop(sizing); + if let Some((cols, rows)) = claimed_size { + let _ = self.resize_surface(surface, cols, rows); + } + if changed { + let mut changed_clients = vec![client]; + if let Some(previous) = previous + && previous.client != client + { + changed_clients.push(previous.client); } - policy.excluded_clients = excluded; - policy.exclusive_client = Some(target); - self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); - drop(sizing); - self.emit_client_sizing_changes(known_clients); + self.emit_client_sizing_changes(changed_clients); + } + Some(changed) + } + + pub fn release_terminal_geometry(&self, surface: SurfaceId) -> Option { + let _lifecycle = self.lock_client_sizing_lifecycle(); + let runtime = self.surface(surface)?.terminal_runtime_id()?; + let mut sizing = self.client_sizing.lock().unwrap(); + let owner = sizing.terminal_authorities.get(&runtime).copied(); + if owner.is_some_and(|authority| { + authority.client != 0 && !self.control_clients.contains(authority.client) + }) { + return None; + } + let released = sizing.terminal_authorities.remove(&runtime); + drop(sizing); + if let Some(released) = released { + self.emit_client_sizing_changes([released.client]); Some(true) + } else { + Some(false) + } + } + + fn emit_client_sizing_changes(&self, clients: impl IntoIterator) { + for client in clients { + let (name, kind) = self.control_clients.client_info(client).unwrap_or((None, None)); + self.emit(MuxEvent::ClientChanged { client, name, kind }); } + } - /// Release canonical terminal geometry, freezing the current PTY size. - pub fn use_all_client_sizes(&self, surface: SurfaceId) -> Option { - if self.surface(surface)?.terminal_runtime_id().is_some() { - return self.release_terminal_geometry(surface); + /// Claim or release a terminal's canonical geometry for one live client, + /// or update the legacy shared-size policy for a non-terminal surface. + pub fn set_client_size_participation( + &self, + surface: SurfaceId, + client: u64, + participating: bool, + ) -> Option { + if self.surface(surface)?.terminal_runtime_id().is_some() { + if participating { + return self.claim_terminal_geometry(surface, client); } let _lifecycle = self.lock_client_sizing_lifecycle(); - self.surface(surface)?; + // Revalidate after acquiring the sizing lifecycle fence. A + // disconnect may have removed the client while this action was + // waiting for the fence, and a stale release must not report a + // successful no-op against a dead client. + if client != 0 && !self.control_clients.contains(client) { + return None; + } + let runtime = self.surface(surface)?.terminal_runtime_id()?; let mut sizing = self.client_sizing.lock().unwrap(); - let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); - let Some(_) = sizing.policies.remove(&surface) else { - return Some(false); - }; - let mut known_clients = attached_clients.clone(); - if let Some(reporters) = sizing.surfaces.get(&surface) { - known_clients.extend(reporters.keys().copied()); + let owns = sizing.owns_terminal_geometry(runtime, surface, client); + if owns { + sizing.terminal_authorities.remove(&runtime); + drop(sizing); + self.emit_client_sizing_changes([client]); } - self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); - drop(sizing); - self.emit_client_sizing_changes(known_clients); - Some(true) + return Some(owns); + } + let _lifecycle = self.lock_client_sizing_lifecycle(); + self.surface(surface)?; + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); + let mut known_clients = attached_clients.clone(); + if let Some(reporters) = sizing.surfaces.get(&surface) { + known_clients.extend(reporters.keys().copied()); + } + if !known_clients.contains(&client) { + return None; + } + if sizing.client_participates(surface, client) == participating { + return Some(false); + } + let policy = sizing.policies.entry(surface).or_default(); + if let Some(exclusive) = policy.exclusive_client.take() { + policy + .excluded_clients + .extend(known_clients.iter().copied().filter(|candidate| *candidate != exclusive)); + policy.excluded_clients.remove(&exclusive); } + if participating { + policy.excluded_clients.remove(&client); + } else { + policy.excluded_clients.insert(client); + } + if policy.excluded_clients.is_empty() { + sizing.policies.remove(&surface); + } + self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); + drop(sizing); + self.emit_client_sizing_changes([client]); + Some(true) + } - pub fn client_size_participates(&self, surface: SurfaceId, client: u64) -> bool { - if let Some(runtime) = - self.surface(surface).and_then(|surface| surface.terminal_runtime_id()) - { - return self - .client_sizing - .lock() - .unwrap() - .owns_terminal_geometry(runtime, surface, client); - } - self.client_sizing.lock().unwrap().client_participates(surface, client) - } - - pub fn control_clients_json(&self, requesting_client: u64) -> Value { - let mut clients = self.control_clients.list_json(requesting_client); - let sizing = self.client_sizing.lock().unwrap(); - if let Some(clients) = clients.as_array_mut() { - for info in clients { - let id = info.get("client").and_then(Value::as_u64).unwrap_or_default(); - if let Some(sizes) = info.get_mut("sizes").and_then(Value::as_array_mut) { - for size in sizes { - let surface = - size.get("surface").and_then(Value::as_u64).unwrap_or_default(); - size["size_participating"] = - serde_json::json!(sizing.report_participates(surface, id)); - } - } - } - } - let local_sizes = sizing - .surfaces - .iter() - .filter_map(|(surface, viewers)| { - viewers.get(&0).map(|(cols, rows)| { - serde_json::json!({ - "surface": surface, - "cols": cols, - "rows": rows, - "size_participating": sizing.report_participates(*surface, 0), - }) - }) - }) - .collect::>(); - if !local_sizes.is_empty() - && let Some(clients) = clients.as_array_mut() + /// Atomically grant one client/placement canonical terminal geometry. + pub fn use_only_client_size(&self, surface: SurfaceId, target: u64) -> Option { + if self.surface(surface)?.terminal_runtime_id().is_some() { + if target != 0 + && !self.control_clients.attached_client_ids_for_surface(surface).contains(&target) { - clients.insert( - 0, - serde_json::json!({ - "client": 0, - "transport": "local", - "name": "This TUI", - "kind": "tui", - "connected_seconds": 0, - "attached": local_sizes.iter().filter_map(|size| size.get("surface")).cloned().collect::>(), - "sizes": local_sizes, - "self": requesting_client == 0, - }), - ); + return None; } - clients + self.client_surface_size(surface, target)?; + return self.claim_terminal_geometry(surface, target); } - - #[cfg(test)] - fn set_client_resize_before_apply(&self, hook: Option>) { - *self.client_resize_before_apply.lock().unwrap() = hook; + let _lifecycle = self.lock_client_sizing_lifecycle(); + self.surface(surface)?; + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); + let reporters = sizing.surfaces.get(&surface); + let target_is_reporting = reporters.is_some_and(|viewers| viewers.contains_key(&target)); + if !target_is_reporting { + return None; } - - #[cfg(test)] - pub(crate) fn set_client_rollback_before_wait( - &self, - hook: Option>, - ) { - *self.client_rollback_before_wait.lock().unwrap() = hook; + let mut known_clients = attached_clients.clone(); + if let Some(reporters) = reporters { + known_clients.extend(reporters.keys().copied()); } - - #[cfg(test)] - fn set_terminal_move_before_projection(&self, hook: Option>) { - *self.terminal_move_before_projection.lock().unwrap() = hook; + let excluded = known_clients + .iter() + .copied() + .filter(|client| *client != target) + .collect::>(); + let policy = sizing.policies.entry(surface).or_default(); + if policy.excluded_clients == excluded && policy.exclusive_client == Some(target) { + return Some(false); } + policy.excluded_clients = excluded; + policy.exclusive_client = Some(target); + self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); + drop(sizing); + self.emit_client_sizing_changes(known_clients); + Some(true) + } - #[cfg(test)] - fn set_shutdown_owner_capacity_for_test(&self, capacity: usize) { - self.shutdown_owner_capacity.store(capacity, Ordering::Release); + /// Release canonical terminal geometry, freezing the current PTY size. + pub fn use_all_client_sizes(&self, surface: SurfaceId) -> Option { + if self.surface(surface)?.terminal_runtime_id().is_some() { + return self.release_terminal_geometry(surface); + } + let _lifecycle = self.lock_client_sizing_lifecycle(); + self.surface(surface)?; + let mut sizing = self.client_sizing.lock().unwrap(); + let attached_clients = self.control_clients.attached_client_ids_for_surface(surface); + let Some(_) = sizing.policies.remove(&surface) else { + return Some(false); + }; + let mut known_clients = attached_clients.clone(); + if let Some(reporters) = sizing.surfaces.get(&surface) { + known_clients.extend(reporters.keys().copied()); } + self.apply_effective_client_size(&sizing, surface, Some(&attached_clients)); + drop(sizing); + self.emit_client_sizing_changes(known_clients); + Some(true) + } - #[cfg(test)] - fn last_resource_mutation_metrics(&self) -> ResourceMutationMetrics { - self.resource_mutation_metrics + pub fn client_size_participates(&self, surface: SurfaceId, client: u64) -> bool { + if let Some(runtime) = + self.surface(surface).and_then(|surface| surface.terminal_runtime_id()) + { + return self + .client_sizing .lock() .unwrap() - .expect("resource mutation did not record metrics") - } - - #[cfg(all(test, unix))] - pub(crate) fn seed_running_terminal_for_test( - self: &Arc, - terminal_id: &str, - incarnation: &str, - workspace_key: &str, - ) -> anyhow::Result { - let _creation = self.begin_surface_creation()?; - let mut registry = self.workspace_registry.lock().unwrap(); - commit_terminal_transition( - &mut registry, - "terminal-reserved", - "seed-terminal-reservation", - &RegistryTerminal { - terminal_id: terminal_id.to_string(), - workspace_key: workspace_key.to_string(), - incarnation: None, - lifecycle: TerminalLifecycle::Launching, - launch_spec: serde_json::json!({}), - exit: None, - }, - )?; - commit_terminal_lifecycle( - &mut registry, - "terminal-ready", - "seed-running-terminal", - terminal_id, - TerminalLifecycle::Running, - Some(incarnation), - None, - )?; - let surface = Surface::exited_terminal_placeholder( - self.next_id(), - self.surface_options.lock().unwrap().clone(), - Arc::downgrade(self), - TerminalHostIdentity { - terminal_id: terminal_id.to_string(), - incarnation: incarnation.to_string(), - }, - )?; - let mut state = self.state.lock().unwrap(); - insert_surface_checked(self, &mut state, surface.clone())?; - let (placement, changed) = self.project_terminal_to_workspace_in_state( - &mut state, - terminal_id, - workspace_key, - )?; - anyhow::ensure!(changed, "seeded terminal did not change topology"); - anyhow::ensure!( - placement.is_some_and(|placement| placement.surface == surface.id), - "seeded terminal projection returned the wrong surface" + .owns_terminal_geometry(runtime, surface, client); + } + self.client_sizing.lock().unwrap().client_participates(surface, client) + } + + pub fn control_clients_json(&self, requesting_client: u64) -> Value { + let mut clients = self.control_clients.list_json(requesting_client); + let sizing = self.client_sizing.lock().unwrap(); + if let Some(clients) = clients.as_array_mut() { + for info in clients { + let id = info.get("client").and_then(Value::as_u64).unwrap_or_default(); + if let Some(sizes) = info.get_mut("sizes").and_then(Value::as_array_mut) { + for size in sizes { + let surface = + size.get("surface").and_then(Value::as_u64).unwrap_or_default(); + size["size_participating"] = + serde_json::json!(sizing.report_participates(surface, id)); + } + } + } + } + let local_sizes = sizing + .surfaces + .iter() + .filter_map(|(surface, viewers)| { + viewers.get(&0).map(|(cols, rows)| { + serde_json::json!({ + "surface": surface, + "cols": cols, + "rows": rows, + "size_participating": sizing.report_participates(*surface, 0), + }) + }) + }) + .collect::>(); + if !local_sizes.is_empty() + && let Some(clients) = clients.as_array_mut() + { + clients.insert( + 0, + serde_json::json!({ + "client": 0, + "transport": "local", + "name": "This TUI", + "kind": "tui", + "connected_seconds": 0, + "attached": local_sizes.iter().filter_map(|size| size.get("surface")).cloned().collect::>(), + "sizes": local_sizes, + "self": requesting_client == 0, + }), ); - drop(state); - drop(registry); - self.commit_ordinary_full_resource_projection( - "test.terminal.seed", - serde_json::json!({}), - )?; - Ok(surface.id) } + clients + } - #[cfg(test)] - pub(crate) fn set_terminal_close_failure_for_test( - &self, - enabled: bool, - ) -> anyhow::Result<()> { - self.workspace_registry.lock().unwrap().set_terminal_close_failure(enabled) - } + #[cfg(test)] + fn set_client_resize_before_apply(&self, hook: Option>) { + *self.client_resize_before_apply.lock().unwrap() = hook; + } - fn browser_runtime(&self) -> anyhow::Result> { - loop { - let mut state = self.browser_runtime.state.lock().unwrap(); - if self.is_shutting_down() { - anyhow::bail!("server is shutting down"); + #[cfg(test)] + pub(crate) fn set_client_rollback_before_wait( + &self, + hook: Option>, + ) { + *self.client_rollback_before_wait.lock().unwrap() = hook; + } + + #[cfg(test)] + fn set_terminal_move_before_projection(&self, hook: Option>) { + *self.terminal_move_before_projection.lock().unwrap() = hook; + } + + #[cfg(test)] + fn set_shutdown_owner_capacity_for_test(&self, capacity: usize) { + self.shutdown_owner_capacity.store(capacity, Ordering::Release); + } + + #[cfg(test)] + fn last_resource_mutation_metrics(&self) -> ResourceMutationMetrics { + self.resource_mutation_metrics + .lock() + .unwrap() + .expect("resource mutation did not record metrics") + } + + #[cfg(all(test, unix))] + pub(crate) fn seed_running_terminal_for_test( + self: &Arc, + terminal_id: &str, + incarnation: &str, + workspace_key: &str, + ) -> anyhow::Result { + let _creation = self.begin_surface_creation()?; + let mut registry = self.workspace_registry.lock().unwrap(); + commit_terminal_transition( + &mut registry, + "terminal-reserved", + "seed-terminal-reservation", + &RegistryTerminal { + terminal_id: terminal_id.to_string(), + workspace_key: workspace_key.to_string(), + incarnation: None, + lifecycle: TerminalLifecycle::Launching, + launch_spec: serde_json::json!({}), + exit: None, + }, + )?; + commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "seed-running-terminal", + terminal_id, + TerminalLifecycle::Running, + Some(incarnation), + None, + )?; + let surface = Surface::exited_terminal_placeholder( + self.next_id(), + self.surface_options.lock().unwrap().clone(), + Arc::downgrade(self), + TerminalHostIdentity { + terminal_id: terminal_id.to_string(), + incarnation: incarnation.to_string(), + }, + )?; + let mut state = self.state.lock().unwrap(); + insert_surface_checked(self, &mut state, surface.clone())?; + let (placement, changed) = + self.project_terminal_to_workspace_in_state(&mut state, terminal_id, workspace_key)?; + anyhow::ensure!(changed, "seeded terminal did not change topology"); + anyhow::ensure!( + placement.is_some_and(|placement| placement.surface == surface.id), + "seeded terminal projection returned the wrong surface" + ); + drop(state); + drop(registry); + self.commit_ordinary_full_resource_projection("test.terminal.seed", serde_json::json!({}))?; + Ok(surface.id) + } + + #[cfg(test)] + pub(crate) fn set_terminal_close_failure_for_test(&self, enabled: bool) -> anyhow::Result<()> { + self.workspace_registry.lock().unwrap().set_terminal_close_failure(enabled) + } + + fn browser_runtime(&self) -> anyhow::Result> { + loop { + let mut state = self.browser_runtime.state.lock().unwrap(); + if self.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } + match &*state { + BrowserRuntimeSlotState::Ready(runtime) if !runtime.is_closed() => { + return Ok(runtime.clone()); } - match &*state { - BrowserRuntimeSlotState::Ready(runtime) if !runtime.is_closed() => { - return Ok(runtime.clone()); - } - BrowserRuntimeSlotState::Ready(_) => { - *state = BrowserRuntimeSlotState::Empty; - } - BrowserRuntimeSlotState::Empty => { - *state = BrowserRuntimeSlotState::Connecting; - break; - } - BrowserRuntimeSlotState::Connecting => { - drop(self.browser_runtime.changed.wait(state).unwrap()); - } - BrowserRuntimeSlotState::Stopping(_) => { - anyhow::bail!("server is shutting down"); - } + BrowserRuntimeSlotState::Ready(_) => { + *state = BrowserRuntimeSlotState::Empty; + } + BrowserRuntimeSlotState::Empty => { + *state = BrowserRuntimeSlotState::Connecting; + break; + } + BrowserRuntimeSlotState::Connecting => { + drop(self.browser_runtime.changed.wait(state).unwrap()); + } + BrowserRuntimeSlotState::Stopping(_) => { + anyhow::bail!("server is shutting down"); } } + } - let opts = self.surface_options.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { - hook(); - } - let connected = BrowserRuntime::connect(&opts); - let mut state = self.browser_runtime.state.lock().unwrap(); - match connected { + let opts = self.surface_options.lock().unwrap().clone(); + #[cfg(test)] + if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { + hook(); + } + let connected = BrowserRuntime::connect(&opts); + let mut state = self.browser_runtime.state.lock().unwrap(); + match connected { + Ok(created) + if matches!(*state, BrowserRuntimeSlotState::Connecting) + && !self.is_shutting_down() => + { + *state = BrowserRuntimeSlotState::Ready(created.clone()); + self.browser_runtime.changed.notify_all(); Ok(created) - if matches!(*state, BrowserRuntimeSlotState::Connecting) - && !self.is_shutting_down() => - { - *state = BrowserRuntimeSlotState::Ready(created.clone()); - self.browser_runtime.changed.notify_all(); - Ok(created) + } + Ok(created) => { + self.browser_runtime.changed.notify_all(); + drop(state); + created.shutdown(); + anyhow::bail!("server is shutting down"); + } + Err(error) => { + let stopping = self.is_shutting_down() + || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); + if matches!(*state, BrowserRuntimeSlotState::Connecting) { + *state = BrowserRuntimeSlotState::Empty; } - Ok(created) => { - self.browser_runtime.changed.notify_all(); - drop(state); - created.shutdown(); + self.browser_runtime.changed.notify_all(); + drop(state); + if stopping { anyhow::bail!("server is shutting down"); } - Err(error) => { - let stopping = self.is_shutting_down() - || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); - if matches!(*state, BrowserRuntimeSlotState::Connecting) { - *state = BrowserRuntimeSlotState::Empty; - } - self.browser_runtime.changed.notify_all(); - drop(state); - if stopping { - anyhow::bail!("server is shutting down"); - } - Err(error) - } + Err(error) } } + } - pub(crate) fn register_browser_provider( - self: &Arc, - client: u64, - registration: BrowserProviderRegistration, - ) -> anyhow::Result { - let snapshot = self.browser_providers.register(client, registration)?; - self.reconcile_provider_browser_surfaces(); - Ok(snapshot) - } + pub(crate) fn register_browser_provider( + self: &Arc, + client: u64, + registration: BrowserProviderRegistration, + ) -> anyhow::Result { + let snapshot = self.browser_providers.register(client, registration)?; + self.reconcile_provider_browser_surfaces(); + Ok(snapshot) + } - pub(crate) fn unregister_browser_provider(self: &Arc, client: u64) -> bool { - let removed = self.browser_providers.unregister(client); - if removed { - self.reconcile_provider_browser_surfaces(); - } - removed + pub(crate) fn unregister_browser_provider(self: &Arc, client: u64) -> bool { + let removed = self.browser_providers.unregister(client); + if removed { + self.reconcile_provider_browser_surfaces(); } + removed + } - pub(crate) fn browser_provider_snapshot(&self) -> Option { - self.browser_providers.snapshot() - } + pub(crate) fn browser_provider_snapshot(&self) -> Option { + self.browser_providers.snapshot() + } - fn reconcile_provider_browser_surfaces(self: &Arc) { - let surfaces = { - let state = self.state.lock().unwrap(); - state - .surfaces - .values() - .filter_map(|surface| { - let identity = surface.resource_identity()?; - matches!(&identity.content_id, ContentPublicId::Browser(_)) - .then(|| (surface.clone(), identity.tab_id.clone())) - }) - .collect::>() - }; - for (surface, tab_id) in surfaces { - let lease = self.browser_providers.target(&tab_id); - let Surface::Browser(browser) = surface.as_ref() else { continue }; - if browser.prepare_provider_lease_replacement(lease.as_ref()) { - self.restart_provider_browser_surface(surface); - } + fn reconcile_provider_browser_surfaces(self: &Arc) { + let surfaces = { + let state = self.state.lock().unwrap(); + state + .surfaces + .values() + .filter_map(|surface| { + let identity = surface.resource_identity()?; + matches!(&identity.content_id, ContentPublicId::Browser(_)) + .then(|| (surface.clone(), identity.tab_id.clone())) + }) + .collect::>() + }; + for (surface, tab_id) in surfaces { + let lease = self.browser_providers.target(&tab_id); + let Surface::Browser(browser) = surface.as_ref() else { continue }; + if browser.prepare_provider_lease_replacement(lease.as_ref()) { + self.restart_provider_browser_surface(surface); } } + } - fn browser_runtime_for_provider( - &self, - lease: &BrowserProviderTargetLease, - ) -> anyhow::Result> { - loop { - let mut state = self.browser_runtime.state.lock().unwrap(); - if self.is_shutting_down() { - anyhow::bail!("server is shutting down"); + fn browser_runtime_for_provider( + &self, + lease: &BrowserProviderTargetLease, + ) -> anyhow::Result> { + loop { + let mut state = self.browser_runtime.state.lock().unwrap(); + if self.is_shutting_down() { + anyhow::bail!("server is shutting down"); + } + match &*state { + BrowserRuntimeSlotState::Ready(runtime) + if !runtime.is_closed() + && runtime.matches_provider(&lease.endpoint, &lease.authentication) => + { + return Ok(runtime.clone()); } - match &*state { - BrowserRuntimeSlotState::Ready(runtime) - if !runtime.is_closed() - && runtime.matches_provider(&lease.endpoint, &lease.authentication) => - { - return Ok(runtime.clone()); - } - BrowserRuntimeSlotState::Ready(_) => { - *state = BrowserRuntimeSlotState::Empty; - } - BrowserRuntimeSlotState::Empty => { - *state = BrowserRuntimeSlotState::Connecting; - break; - } - BrowserRuntimeSlotState::Connecting => { - drop(self.browser_runtime.changed.wait(state).unwrap()); - } - BrowserRuntimeSlotState::Stopping(_) => { - anyhow::bail!("server is shutting down"); - } + BrowserRuntimeSlotState::Ready(_) => { + *state = BrowserRuntimeSlotState::Empty; + } + BrowserRuntimeSlotState::Empty => { + *state = BrowserRuntimeSlotState::Connecting; + break; + } + BrowserRuntimeSlotState::Connecting => { + drop(self.browser_runtime.changed.wait(state).unwrap()); + } + BrowserRuntimeSlotState::Stopping(_) => { + anyhow::bail!("server is shutting down"); } } + } - let connected = - BrowserRuntime::connect_provider(&lease.endpoint, &lease.authentication); - let mut state = self.browser_runtime.state.lock().unwrap(); - match connected { + let connected = BrowserRuntime::connect_provider(&lease.endpoint, &lease.authentication); + let mut state = self.browser_runtime.state.lock().unwrap(); + match connected { + Ok(created) + if matches!(*state, BrowserRuntimeSlotState::Connecting) + && !self.is_shutting_down() => + { + *state = BrowserRuntimeSlotState::Ready(created.clone()); + self.browser_runtime.changed.notify_all(); Ok(created) - if matches!(*state, BrowserRuntimeSlotState::Connecting) - && !self.is_shutting_down() => - { - *state = BrowserRuntimeSlotState::Ready(created.clone()); - self.browser_runtime.changed.notify_all(); - Ok(created) + } + Ok(created) => { + self.browser_runtime.changed.notify_all(); + drop(state); + created.shutdown(); + anyhow::bail!("server is shutting down"); + } + Err(error) => { + let stopping = self.is_shutting_down() + || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); + if matches!(*state, BrowserRuntimeSlotState::Connecting) { + *state = BrowserRuntimeSlotState::Empty; } - Ok(created) => { - self.browser_runtime.changed.notify_all(); - drop(state); - created.shutdown(); + self.browser_runtime.changed.notify_all(); + drop(state); + if stopping { anyhow::bail!("server is shutting down"); } - Err(error) => { - let stopping = self.is_shutting_down() - || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); - if matches!(*state, BrowserRuntimeSlotState::Connecting) { - *state = BrowserRuntimeSlotState::Empty; - } - self.browser_runtime.changed.notify_all(); - drop(state); - if stopping { - anyhow::bail!("server is shutting down"); - } - Err(error) - } + Err(error) } } + } - fn start_browser_bootstrap( - self: &Arc, - surface: Arc, - bootstrap: BrowserBootstrap, - runtime: Option>, - ) { - let bootstrap_guard = match self.async_surface_creations.begin() { - Ok(guard) => guard, - Err(error) => { - self.report_browser_bootstrap_failure(&surface, error.to_string()); - return; - } - }; - let provider_bootstrap = matches!(&bootstrap, BrowserBootstrap::Provider { .. }); - let weak_mux = Arc::downgrade(self); - let providers = self.browser_providers.clone(); - let id = surface.id; - let thread_surface = surface.clone(); - let spawn = std::thread::Builder::new() - .name(format!("browser-surface-{id}-bootstrap")) - .spawn(move || { - let _bootstrap_guard = bootstrap_guard; - let result = (|| -> anyhow::Result<()> { - match bootstrap { - BrowserBootstrap::Provider { tab_id, url } => { - anyhow::ensure!( - runtime.is_none(), - "provider bootstrap cannot override its CDP runtime" - ); - let mut retry_delay = Duration::from_millis(250); - loop { - let canceled = || { - thread_surface.is_dead() - || weak_mux.upgrade().is_none_or(|mux| { - mux.shutting_down.load(Ordering::Acquire) - }) + fn start_browser_bootstrap( + self: &Arc, + surface: Arc, + bootstrap: BrowserBootstrap, + runtime: Option>, + ) { + let bootstrap_guard = match self.async_surface_creations.begin() { + Ok(guard) => guard, + Err(error) => { + self.report_browser_bootstrap_failure(&surface, error.to_string()); + return; + } + }; + let provider_bootstrap = matches!(&bootstrap, BrowserBootstrap::Provider { .. }); + let weak_mux = Arc::downgrade(self); + let providers = self.browser_providers.clone(); + let id = surface.id; + let thread_surface = surface.clone(); + let spawn = std::thread::Builder::new() + .name(format!("browser-surface-{id}-bootstrap")) + .spawn(move || { + let _bootstrap_guard = bootstrap_guard; + let result = (|| -> anyhow::Result<()> { + match bootstrap { + BrowserBootstrap::Provider { tab_id, url } => { + anyhow::ensure!( + runtime.is_none(), + "provider bootstrap cannot override its CDP runtime" + ); + let mut retry_delay = Duration::from_millis(250); + loop { + let canceled = || { + thread_surface.is_dead() + || weak_mux.upgrade().is_none_or(|mux| { + mux.shutting_down.load(Ordering::Acquire) + }) + }; + let lease = + providers.wait_for_target(&tab_id, canceled).ok_or_else( + || anyhow::anyhow!("browser provider wait was canceled"), + )?; + let attempt = (|| -> anyhow::Result<()> { + let mux = weak_mux.upgrade().ok_or_else(|| { + anyhow::anyhow!("browser mux was dropped") + })?; + let runtime = mux.browser_runtime_for_provider(&lease)?; + let Surface::Browser(browser) = thread_surface.as_ref() else { + anyhow::bail!( + "browser bootstrap got a non-browser surface" + ); }; - let lease = providers - .wait_for_target(&tab_id, canceled) - .ok_or_else(|| { - anyhow::anyhow!("browser provider wait was canceled") - })?; - let attempt = (|| -> anyhow::Result<()> { - let mux = weak_mux.upgrade().ok_or_else(|| { - anyhow::anyhow!("browser mux was dropped") - })?; - let runtime = mux.browser_runtime_for_provider(&lease)?; + anyhow::ensure!( + browser.prepare_provider_bootstrap_attempt(), + "browser provider wait was canceled" + ); + runtime.bootstrap_surface_sync( + thread_surface.clone(), + BrowserBootstrap::ExistingTarget { + target_id: lease.target_id.clone(), + url: url.clone(), + }, + weak_mux.clone(), + ) + })(); + match attempt { + Ok(()) => { + let current_lease = providers.target(&tab_id); let Surface::Browser(browser) = thread_surface.as_ref() else { anyhow::bail!( "browser bootstrap got a non-browser surface" ); }; - anyhow::ensure!( - browser.prepare_provider_bootstrap_attempt(), - "browser provider wait was canceled" - ); - runtime.bootstrap_surface_sync( - thread_surface.clone(), - BrowserBootstrap::ExistingTarget { - target_id: lease.target_id.clone(), - url: url.clone(), - }, - weak_mux.clone(), - ) - })(); - match attempt { - Ok(()) => { - let current_lease = providers.target(&tab_id); - let Surface::Browser(browser) = thread_surface.as_ref() - else { - anyhow::bail!( - "browser bootstrap got a non-browser surface" - ); - }; - // Registration can change while CDP - // setup is in flight. Never publish a - // now-stale target merely because its - // attach finished after the provider - // revision advanced. - if browser.prepare_provider_lease_replacement( - current_lease.as_ref(), - ) { - retry_delay = Duration::from_millis(250); - continue; - } - return Ok(()); + // Registration can change while CDP + // setup is in flight. Never publish a + // now-stale target merely because its + // attach finished after the provider + // revision advanced. + if browser.prepare_provider_lease_replacement( + current_lease.as_ref(), + ) { + retry_delay = Duration::from_millis(250); + continue; } - Err(error) if !canceled() => { - let message = error.to_string(); - let Surface::Browser(browser) = thread_surface.as_ref() - else { - return Err(error); - }; - let changed = browser.status() - != crate::BrowserStatus::Failed(message.clone()); - if changed { - browser.mark_failed(message.clone()); - if let Some(mux) = weak_mux.upgrade() { - mux.emit(MuxEvent::Status(format!( + return Ok(()); + } + Err(error) if !canceled() => { + let message = error.to_string(); + let Surface::Browser(browser) = thread_surface.as_ref() + else { + return Err(error); + }; + let changed = browser.status() + != crate::BrowserStatus::Failed(message.clone()); + if changed { + browser.mark_failed(message.clone()); + if let Some(mux) = weak_mux.upgrade() { + mux.emit(MuxEvent::Status(format!( "cmux-browser provider unavailable: {message}" ))); - mux.emit(MuxEvent::TitleChanged { - surface: id, - title: thread_surface.title().into(), - }); - mux.emit(MuxEvent::SurfaceOutput(id)); - } - } - if !providers.wait_for_revision_change( - lease.revision, - canceled, - retry_delay, - ) { - anyhow::bail!("browser provider wait was canceled"); + mux.emit(MuxEvent::TitleChanged { + surface: id, + title: thread_surface.title().into(), + }); + mux.emit(MuxEvent::SurfaceOutput(id)); } - retry_delay = retry_delay - .saturating_mul(2) - .min(Duration::from_secs(2)); } - Err(error) => return Err(error), + if !providers.wait_for_revision_change( + lease.revision, + canceled, + retry_delay, + ) { + anyhow::bail!("browser provider wait was canceled"); + } + retry_delay = retry_delay + .saturating_mul(2) + .min(Duration::from_secs(2)); } + Err(error) => return Err(error), } } - bootstrap => { - let mux = weak_mux - .upgrade() - .ok_or_else(|| anyhow::anyhow!("browser mux was dropped"))?; - let runtime = match runtime { - Some(runtime) => runtime, - None => mux.browser_runtime()?, - }; - runtime.bootstrap_surface_sync( - thread_surface.clone(), - bootstrap, - weak_mux.clone(), - ) - } - } - })(); - if let Err(err) = result { - if !thread_surface.is_dead() - && !provider_bootstrap - && let Surface::Browser(browser) = thread_surface.as_ref() - { - browser.mark_failed(err.to_string()); } - if !provider_bootstrap - && let Some(mux) = weak_mux.upgrade() - && !thread_surface.is_dead() - { - mux.emit(MuxEvent::Status(format!("browser failed: {err}"))); - mux.emit(MuxEvent::TitleChanged { - surface: id, - title: thread_surface.title().into(), - }); - mux.emit(MuxEvent::SurfaceOutput(id)); + bootstrap => { + let mux = weak_mux + .upgrade() + .ok_or_else(|| anyhow::anyhow!("browser mux was dropped"))?; + let runtime = match runtime { + Some(runtime) => runtime, + None => mux.browser_runtime()?, + }; + runtime.bootstrap_surface_sync( + thread_surface.clone(), + bootstrap, + weak_mux.clone(), + ) } } - }); - if let Err(error) = spawn - && !surface.is_dead() - && let Surface::Browser(browser) = surface.as_ref() - { - browser.mark_failed(format!("could not start browser bootstrap: {error}")); - } + })(); + if let Err(err) = result { + if !thread_surface.is_dead() + && !provider_bootstrap + && let Surface::Browser(browser) = thread_surface.as_ref() + { + browser.mark_failed(err.to_string()); + } + if !provider_bootstrap + && let Some(mux) = weak_mux.upgrade() + && !thread_surface.is_dead() + { + mux.emit(MuxEvent::Status(format!("browser failed: {err}"))); + mux.emit(MuxEvent::TitleChanged { + surface: id, + title: thread_surface.title().into(), + }); + mux.emit(MuxEvent::SurfaceOutput(id)); + } + } + }); + if let Err(error) = spawn + && !surface.is_dead() + && let Surface::Browser(browser) = surface.as_ref() + { + browser.mark_failed(format!("could not start browser bootstrap: {error}")); } + } - pub(crate) fn restart_provider_browser_surface(self: &Arc, surface: Arc) { - let Some(identity) = surface.resource_identity() else { return }; - if !matches!(identity.content_id, ContentPublicId::Browser(_)) { - return; - } - let tab_id = identity.tab_id.clone(); - let url = surface.browser_url().unwrap_or_else(|| "about:blank".to_string()); - self.start_browser_bootstrap(surface, BrowserBootstrap::Provider { tab_id, url }, None); + pub(crate) fn restart_provider_browser_surface(self: &Arc, surface: Arc) { + let Some(identity) = surface.resource_identity() else { return }; + if !matches!(identity.content_id, ContentPublicId::Browser(_)) { + return; } + let tab_id = identity.tab_id.clone(); + let url = surface.browser_url().unwrap_or_else(|| "about:blank".to_string()); + self.start_browser_bootstrap(surface, BrowserBootstrap::Provider { tab_id, url }, None); + } - /// A fresh single-tab pane wrapping `surface`. - fn make_pane(&self, surface: SurfaceId) -> anyhow::Result<(PaneId, Pane)> { - let id = self.next_id(); - let active_at = self.next_active_at(); - Ok(( + /// A fresh single-tab pane wrapping `surface`. + fn make_pane(&self, surface: SurfaceId) -> anyhow::Result<(PaneId, Pane)> { + let id = self.next_id(); + let active_at = self.next_active_at(); + Ok(( + id, + Pane { id, - Pane { - id, - public_id: PanePublicId::random()?, - name: None, - tabs: vec![surface], - active_tab: 0, - active_at, - focused_at: 0, - }, - )) - } + public_id: PanePublicId::random()?, + name: None, + tabs: vec![surface], + active_tab: 0, + active_at, + focused_at: 0, + }, + )) + } + + pub fn surface(&self, id: SurfaceId) -> Option> { + let state = self.state.lock().unwrap(); + state.surfaces.get(&id).or_else(|| state.terminal_runtime_by_id(id)).cloned() + } - pub fn surface(&self, id: SurfaceId) -> Option> { - let state = self.state.lock().unwrap(); - state.surfaces.get(&id).or_else(|| state.terminal_runtime_by_id(id)).cloned() - } + pub(crate) fn terminal_resource_surface( + &self, + terminal_id: &TerminalPublicId, + ) -> Option> { + self.state.lock().unwrap().terminal_catalog.get(terminal_id).cloned() + } - pub(crate) fn terminal_resource_surface( - &self, - terminal_id: &TerminalPublicId, - ) -> Option> { - self.state.lock().unwrap().terminal_catalog.get(terminal_id).cloned() - } + #[cfg(test)] + pub(crate) fn remove_surface_runtime_for_test(&self, id: SurfaceId) -> Option> { + self.state.lock().unwrap().surfaces.remove(&id) + } - #[cfg(test)] - pub(crate) fn remove_surface_runtime_for_test( - &self, - id: SurfaceId, - ) -> Option> { - self.state.lock().unwrap().surfaces.remove(&id) - } + #[cfg(test)] + pub(crate) fn insert_surface_runtime_for_test(&self, surface: Arc) { + let previous = self.state.lock().unwrap().surfaces.insert(surface.id, surface); + assert!(previous.is_none(), "test surface id already exists"); + } - #[cfg(test)] - pub(crate) fn insert_surface_runtime_for_test(&self, surface: Arc) { - let previous = self.state.lock().unwrap().surfaces.insert(surface.id, surface); - assert!(previous.is_none(), "test surface id already exists"); + #[cfg(test)] + pub(crate) fn remove_terminal_catalog_for_test( + &self, + terminal_id: &TerminalPublicId, + ) -> Option> { + let mut state = self.state.lock().unwrap(); + let removed = state.terminal_catalog.remove(terminal_id)?; + if let Some(runtime_id) = removed.terminal_runtime_id() { + state.terminal_catalog_by_runtime.remove(&runtime_id); } + Some(removed) + } - #[cfg(test)] - pub(crate) fn remove_terminal_catalog_for_test( - &self, - terminal_id: &TerminalPublicId, - ) -> Option> { - let mut state = self.state.lock().unwrap(); - let removed = state.terminal_catalog.remove(terminal_id)?; - if let Some(runtime_id) = removed.terminal_runtime_id() { - state.terminal_catalog_by_runtime.remove(&runtime_id); - } - Some(removed) - } + /// Resolve a process-stable terminal UUID and one live view placement. + /// A catalog-owned terminal with zero views resolves successfully with a + /// null surface. This is lookup-only and never creates a replacement shell. + pub fn resolve_terminal( + &self, + terminal_id: &str, + ) -> anyhow::Result> { + validate_terminal_hex(terminal_id, "invalid_terminal_id")?; + let (terminal, terminal_revision) = { + let registry = self.workspace_registry.lock().unwrap(); + let snapshot = registry.terminal_snapshot()?; + (registry.terminal_record(terminal_id)?, snapshot.revision) + }; + let Some(terminal) = terminal else { + return Ok(None); + }; + let state = self.state.lock().unwrap(); + let surface = self + .catalog_terminal_by_host(&state, terminal_id)? + .and_then(|runtime| terminal_placement_for_runtime(&state, &runtime)); + Ok(Some(TerminalResolution { surface, terminal, terminal_revision })) + } - /// Resolve a process-stable terminal UUID and one live view placement. - /// A catalog-owned terminal with zero views resolves successfully with a - /// null surface. This is lookup-only and never creates a replacement shell. - pub fn resolve_terminal( - &self, - terminal_id: &str, - ) -> anyhow::Result> { - validate_terminal_hex(terminal_id, "invalid_terminal_id")?; - let (terminal, terminal_revision) = { - let registry = self.workspace_registry.lock().unwrap(); - let snapshot = registry.terminal_snapshot()?; - (registry.terminal_record(terminal_id)?, snapshot.revision) - }; - let Some(terminal) = terminal else { - return Ok(None); - }; - let state = self.state.lock().unwrap(); - let surface = self - .catalog_terminal_by_host(&state, terminal_id)? - .and_then(|runtime| terminal_placement_for_runtime(&state, &runtime)); - Ok(Some(TerminalResolution { surface, terminal, terminal_revision })) - } - - /// Atomically resolve, incarnation-check, and remove a hosted terminal by - /// process-stable identity. The host is terminated only after the state - /// lock has made the removal authoritative for this daemon generation. - pub fn close_terminal( - &self, - terminal_id: &str, - terminal_incarnation: &str, - ) -> anyhow::Result { - self.close_terminal_with_mutation( - terminal_id, - Some(terminal_incarnation), - None, - None, - &WorkspaceMutation::local("cmux-tui"), - ) - } + /// Atomically resolve, incarnation-check, and remove a hosted terminal by + /// process-stable identity. The host is terminated only after the state + /// lock has made the removal authoritative for this daemon generation. + pub fn close_terminal( + &self, + terminal_id: &str, + terminal_incarnation: &str, + ) -> anyhow::Result { + self.close_terminal_with_mutation( + terminal_id, + Some(terminal_incarnation), + None, + None, + &WorkspaceMutation::local("cmux-tui"), + ) + } - pub fn close_terminal_with_mutation( - &self, - terminal_id: &str, - terminal_incarnation: Option<&str>, - expected_generation: Option<&str>, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - validate_terminal_hex(terminal_id, "invalid_terminal_id")?; - if let Some(incarnation) = terminal_incarnation { - validate_terminal_hex(incarnation, "invalid_terminal_incarnation")?; - } - let (commit, terminal_incarnation, public_id, notify_public_id) = { - let mut registry = self.workspace_registry.lock().unwrap(); - let public_id = registry.terminal_resource_id(terminal_id)?; - let commit = registry.close_terminal( - mutation, - expected_generation, - expected_revision, - terminal_id, - terminal_incarnation, - )?; - let newly_closed = - !commit.replayed && !commit.result["already_closed"].as_bool().unwrap_or(false); - if newly_closed { - self.emit_terminal_registry_changed(®istry, commit.revision); - } - let incarnation = registry - .terminal_record(terminal_id)? - .and_then(|terminal| terminal.incarnation); - ( - commit, - incarnation, - public_id.clone(), - newly_closed.then_some(public_id).flatten(), - ) - }; - self.notify_terminal_exit_waiters(notify_public_id); - let (target, removed, runtime, changed_screens, empty_revision) = { - let mut state = self.state.lock().unwrap(); - let catalog_public_id = public_id.or_else(|| { - state.terminal_catalog.iter().find_map(|(public_id, surface)| { - self.resource_terminal_host_identity(surface) - .is_some_and(|identity| identity.terminal_id == terminal_id) - .then(|| public_id.clone()) - }) - }); - let runtime = catalog_public_id - .as_ref() - .and_then(|public_id| state.terminal_catalog.get(public_id)) - .cloned(); - // The durable close has committed. From this point cleanup must - // finish even if an in-memory host incarnation was stale; leaving - // a live runtime behind would contradict the terminal tombstone. - let content_id = catalog_public_id.map(ContentPublicId::Terminal); - let mut targets = content_id - .as_ref() - .map(|content_id| state.placements_of_content(content_id).to_vec()) - .unwrap_or_default(); - if targets.is_empty() { - targets.extend(state.surfaces.iter().filter_map(|(surface_id, surface)| { - self.resource_terminal_host_identity(surface) - .is_some_and(|identity| identity.terminal_id == terminal_id) - .then_some(*surface_id) - })); - } - let target = targets.first().copied(); - let changed_screens = unique_screen_ids( - targets.iter().filter_map(|surface| surface_screen_id(&state, *surface)), - ); - let removed = if let Some(runtime) = runtime.as_ref() { - remove_terminal_runtime_from_state(self, &mut state, runtime).0 - } else { - let mut removed = Vec::with_capacity(targets.len()); - let mut split_index_dirty = false; - for target in targets { - let (surface, topology_changed) = remove_surface(self, &mut state, target); - split_index_dirty |= topology_changed; - if let Some(surface) = surface { - removed.push(surface); - } - } - if split_index_dirty { - Self::rebuild_split_screen_index(&mut state); - } - removed - }; - let empty_revision = - state.workspaces.is_empty().then_some(state.workspace_revision); - (target, removed, runtime, changed_screens, empty_revision) - }; - for surface in removed { - self.purge_surface_side_tables(surface.id); - } - let had_runtime = runtime.is_some(); - if let Some(runtime) = runtime { - self.purge_terminal_runtime_side_tables(&runtime); - self.terminate_terminal_runtime(&runtime); - } - if target.is_some() { - self.emit(MuxEvent::TreeChanged); + pub fn close_terminal_with_mutation( + &self, + terminal_id: &str, + terminal_incarnation: Option<&str>, + expected_generation: Option<&str>, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + validate_terminal_hex(terminal_id, "invalid_terminal_id")?; + if let Some(incarnation) = terminal_incarnation { + validate_terminal_hex(incarnation, "invalid_terminal_incarnation")?; + } + let (commit, terminal_incarnation, public_id, notify_public_id) = { + let mut registry = self.workspace_registry.lock().unwrap(); + let public_id = registry.terminal_resource_id(terminal_id)?; + let commit = registry.close_terminal( + mutation, + expected_generation, + expected_revision, + terminal_id, + terminal_incarnation, + )?; + let newly_closed = + !commit.replayed && !commit.result["already_closed"].as_bool().unwrap_or(false); + if newly_closed { + self.emit_terminal_registry_changed(®istry, commit.revision); } - for screen in changed_screens { - self.emit(MuxEvent::LayoutChanged(screen)); + let incarnation = + registry.terminal_record(terminal_id)?.and_then(|terminal| terminal.incarnation); + (commit, incarnation, public_id.clone(), newly_closed.then_some(public_id).flatten()) + }; + self.notify_terminal_exit_waiters(notify_public_id); + let (target, removed, runtime, changed_screens, empty_revision) = { + let mut state = self.state.lock().unwrap(); + let catalog_public_id = public_id.or_else(|| { + state.terminal_catalog.iter().find_map(|(public_id, surface)| { + self.resource_terminal_host_identity(surface) + .is_some_and(|identity| identity.terminal_id == terminal_id) + .then(|| public_id.clone()) + }) + }); + let runtime = catalog_public_id + .as_ref() + .and_then(|public_id| state.terminal_catalog.get(public_id)) + .cloned(); + // The durable close has committed. From this point cleanup must + // finish even if an in-memory host incarnation was stale; leaving + // a live runtime behind would contradict the terminal tombstone. + let content_id = catalog_public_id.map(ContentPublicId::Terminal); + let mut targets = content_id + .as_ref() + .map(|content_id| state.placements_of_content(content_id).to_vec()) + .unwrap_or_default(); + if targets.is_empty() { + targets.extend(state.surfaces.iter().filter_map(|(surface_id, surface)| { + self.resource_terminal_host_identity(surface) + .is_some_and(|identity| identity.terminal_id == terminal_id) + .then_some(*surface_id) + })); } - let termination = if !had_runtime { - self.terminate_discovered_terminal_host( - terminal_id, - terminal_incarnation.as_deref(), - ) + let target = targets.first().copied(); + let changed_screens = unique_screen_ids( + targets.iter().filter_map(|surface| surface_screen_id(&state, *surface)), + ); + let removed = if let Some(runtime) = runtime.as_ref() { + remove_terminal_runtime_from_state(self, &mut state, runtime).0 } else { - Ok(()) - }; - self.emit_empty_if_current(empty_revision); - termination?; - Ok(TerminalCloseResult { - surface: target, - terminal_id: terminal_id.to_string(), - terminal_incarnation, - already_closed: commit.result["already_closed"] - .as_bool() - .unwrap_or(commit.replayed), - terminal_revision: commit.revision, - }) - } - - /// A host can become Running before its topology binding is built. Keep - /// the durable lifecycle transition ahead of in-memory removal so a crash - /// at any point cannot resurrect an unbound Running terminal on restart. - fn fail_hosted_terminal_attachment( - &self, - surface: &Arc, - _operation: &str, - reason: &str, - ) -> anyhow::Result<()> { - let Some(identity) = self.resource_terminal_host_identity(surface) else { - let removed = { - let mut state = self.state.lock().unwrap(); - remove_terminal_runtime_from_state(self, &mut state, surface).0 - }; - for placement in removed { - self.purge_surface_side_tables(placement.id); + let mut removed = Vec::with_capacity(targets.len()); + let mut split_index_dirty = false; + for target in targets { + let (surface, topology_changed) = remove_surface(self, &mut state, target); + split_index_dirty |= topology_changed; + if let Some(surface) = surface { + removed.push(surface); + } } - self.purge_terminal_runtime_side_tables(surface); - if !surface.is_dead() { - surface.kill(); + if split_index_dirty { + Self::rebuild_split_screen_index(&mut state); } - return Ok(()); + removed }; - self.persist_terminal_exit( - &identity.terminal_id, - Some(&identity.incarnation), - &TerminalExit::unknown(reason), - )?; + let empty_revision = state.workspaces.is_empty().then_some(state.workspace_revision); + (target, removed, runtime, changed_screens, empty_revision) + }; + for surface in removed { + self.purge_surface_side_tables(surface.id); + } + let had_runtime = runtime.is_some(); + if let Some(runtime) = runtime { + self.purge_terminal_runtime_side_tables(&runtime); + self.terminate_terminal_runtime(&runtime); + } + if target.is_some() { + self.emit(MuxEvent::TreeChanged); + } + for screen in changed_screens { + self.emit(MuxEvent::LayoutChanged(screen)); + } + let termination = if !had_runtime { + self.terminate_discovered_terminal_host(terminal_id, terminal_incarnation.as_deref()) + } else { + Ok(()) + }; + self.emit_empty_if_current(empty_revision); + termination?; + Ok(TerminalCloseResult { + surface: target, + terminal_id: terminal_id.to_string(), + terminal_incarnation, + already_closed: commit.result["already_closed"].as_bool().unwrap_or(commit.replayed), + terminal_revision: commit.revision, + }) + } + + /// A host can become Running before its topology binding is built. Keep + /// the durable lifecycle transition ahead of in-memory removal so a crash + /// at any point cannot resurrect an unbound Running terminal on restart. + fn fail_hosted_terminal_attachment( + &self, + surface: &Arc, + _operation: &str, + reason: &str, + ) -> anyhow::Result<()> { + let Some(identity) = self.resource_terminal_host_identity(surface) else { let removed = { let mut state = self.state.lock().unwrap(); remove_terminal_runtime_from_state(self, &mut state, surface).0 @@ -9382,984 +9202,981 @@ impl Mux { if !surface.is_dead() { surface.kill(); } - Ok(()) + return Ok(()); + }; + self.persist_terminal_exit( + &identity.terminal_id, + Some(&identity.incarnation), + &TerminalExit::unknown(reason), + )?; + let removed = { + let mut state = self.state.lock().unwrap(); + remove_terminal_runtime_from_state(self, &mut state, surface).0 + }; + for placement in removed { + self.purge_surface_side_tables(placement.id); } - - pub(crate) fn terminate_discovered_terminal_host( - &self, - terminal_id: &str, - incarnation: Option<&str>, - ) -> anyhow::Result<()> { - self.terminate_discovered_terminal_hosts(&[( - terminal_id.to_string(), - incarnation.map(str::to_string), - )]) + self.purge_terminal_runtime_side_tables(surface); + if !surface.is_dead() { + surface.kill(); } + Ok(()) + } - fn terminate_discovered_terminal_hosts( - &self, - terminals: &[(String, Option)], - ) -> anyhow::Result<()> { - if terminals.is_empty() { - return Ok(()); - } - #[cfg(test)] - self.discovered_terminal_termination_requests - .lock() - .unwrap() - .extend(terminals.iter().cloned()); - #[cfg(unix)] - { - let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); - let Some(root) = root else { return Ok(()) }; - for (terminal_id, expected_incarnation) in terminals { - let Some((path, record)) = - crate::terminal_host_runtime::load_terminal_host_record(&root, terminal_id) - .with_context(|| { - format!("load exact terminal-host record for {terminal_id}") - })? - else { - continue; - }; - anyhow::ensure!( - expected_incarnation - .as_deref() - .is_none_or(|expected| record.incarnation == expected), - "terminal-host incarnation changed while closing {terminal_id}" - ); - anyhow::ensure!( - cleanup_terminal_host_record(&record, &path), - "could not clean up terminal host {terminal_id}" - ); - } - Ok(()) - } - #[cfg(not(unix))] - { - let _ = terminals; - Ok(()) - } - } + pub(crate) fn terminate_discovered_terminal_host( + &self, + terminal_id: &str, + incarnation: Option<&str>, + ) -> anyhow::Result<()> { + self.terminate_discovered_terminal_hosts(&[( + terminal_id.to_string(), + incarnation.map(str::to_string), + )]) + } - #[cfg(test)] - pub(crate) fn take_discovered_terminal_termination_requests_for_test( - &self, - ) -> Vec<(String, Option)> { - std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) + fn terminate_discovered_terminal_hosts( + &self, + terminals: &[(String, Option)], + ) -> anyhow::Result<()> { + if terminals.is_empty() { + return Ok(()); } - - fn prepare_terminal_host_cleanup_at_root( - &self, - root: Option<&Path>, - terminals: &[(String, Option)], - ) -> anyhow::Result { - anyhow::ensure!( - terminals.len() <= MAX_TERMINAL_HOST_CLEANUP_IDENTITIES, - "terminal-host cleanup target exceeds capacity" - ); - let mut seen = HashSet::with_capacity(terminals.len()); - for (terminal_id, incarnation) in terminals { - validate_terminal_hex(terminal_id, "terminal-host cleanup id is invalid")?; + #[cfg(test)] + self.discovered_terminal_termination_requests + .lock() + .unwrap() + .extend(terminals.iter().cloned()); + #[cfg(unix)] + { + let root = self.surface_options.lock().unwrap().terminal_host_root.clone(); + let Some(root) = root else { return Ok(()) }; + for (terminal_id, expected_incarnation) in terminals { + let Some((path, record)) = + crate::terminal_host_runtime::load_terminal_host_record(&root, terminal_id) + .with_context(|| { + format!("load exact terminal-host record for {terminal_id}") + })? + else { + continue; + }; anyhow::ensure!( - seen.insert(terminal_id.as_str()), - "terminal-host cleanup target contains a duplicate id" + expected_incarnation + .as_deref() + .is_none_or(|expected| record.incarnation == expected), + "terminal-host incarnation changed while closing {terminal_id}" ); - if let Some(incarnation) = incarnation { - validate_terminal_hex( - incarnation, - "terminal-host cleanup incarnation is invalid", - )?; - } - } - #[cfg(unix)] - { - let Some(root) = root else { return Ok(Vec::new()) }; - let mut records = Vec::new(); - for (terminal_id, expected_incarnation) in terminals { - let Some((path, record)) = - crate::terminal_host_runtime::load_terminal_host_record(root, terminal_id) - .with_context(|| { - format!("load exact terminal-host record for {terminal_id}") - })? - else { - continue; - }; - anyhow::ensure!( - expected_incarnation - .as_deref() - .is_none_or(|expected| record.incarnation == expected), - "terminal-host incarnation changed while closing {terminal_id}" - ); - records.push((path, record)); - } - #[cfg(test)] - self.discovered_terminal_termination_requests.lock().unwrap().extend( - records.iter().map(|(_, record)| { - (record.terminal_id.clone(), Some(record.incarnation.clone())) - }), + anyhow::ensure!( + cleanup_terminal_host_record(&record, &path), + "could not clean up terminal host {terminal_id}" ); - Ok(records) - } - #[cfg(not(unix))] - { - let _ = root; - let _ = terminals; - Ok(()) } + Ok(()) + } + #[cfg(not(unix))] + { + let _ = terminals; + Ok(()) } + } - fn terminate_prepared_terminal_hosts( - &self, - records: TerminalHostRecords, - ) -> anyhow::Result<()> { - #[cfg(test)] - if let Some(hook) = self.terminal_host_cleanup_hook.lock().unwrap().clone() { - anyhow::ensure!(hook(), "forced terminal-host cleanup failure"); - } - #[cfg(unix)] - { - for (path, record) in records { - anyhow::ensure!( - cleanup_terminal_host_record(&record, &path), - "could not clean up terminal host {}", - record.terminal_id - ); - } - Ok(()) + #[cfg(test)] + pub(crate) fn take_discovered_terminal_termination_requests_for_test( + &self, + ) -> Vec<(String, Option)> { + std::mem::take(&mut *self.discovered_terminal_termination_requests.lock().unwrap()) + } + + fn prepare_terminal_host_cleanup_at_root( + &self, + root: Option<&Path>, + terminals: &[(String, Option)], + ) -> anyhow::Result { + anyhow::ensure!( + terminals.len() <= MAX_TERMINAL_HOST_CLEANUP_IDENTITIES, + "terminal-host cleanup target exceeds capacity" + ); + let mut seen = HashSet::with_capacity(terminals.len()); + for (terminal_id, incarnation) in terminals { + validate_terminal_hex(terminal_id, "terminal-host cleanup id is invalid")?; + anyhow::ensure!( + seen.insert(terminal_id.as_str()), + "terminal-host cleanup target contains a duplicate id" + ); + if let Some(incarnation) = incarnation { + validate_terminal_hex(incarnation, "terminal-host cleanup incarnation is invalid")?; } - #[cfg(not(unix))] - { - let _ = records; - Ok(()) + } + #[cfg(unix)] + { + let Some(root) = root else { return Ok(Vec::new()) }; + let mut records = Vec::new(); + for (terminal_id, expected_incarnation) in terminals { + let Some((path, record)) = + crate::terminal_host_runtime::load_terminal_host_record(root, terminal_id) + .with_context(|| { + format!("load exact terminal-host record for {terminal_id}") + })? + else { + continue; + }; + anyhow::ensure!( + expected_incarnation + .as_deref() + .is_none_or(|expected| record.incarnation == expected), + "terminal-host incarnation changed while closing {terminal_id}" + ); + records.push((path, record)); } + #[cfg(test)] + self.discovered_terminal_termination_requests.lock().unwrap().extend( + records.iter().map(|(_, record)| { + (record.terminal_id.clone(), Some(record.incarnation.clone())) + }), + ); + Ok(records) + } + #[cfg(not(unix))] + { + let _ = root; + let _ = terminals; + Ok(()) } + } - fn terminate_workspace_hosts_from_receipt(&self, result: &Value) -> anyhow::Result<()> { - let identities = terminal_host_cleanup_identities(result)?; - let mut runtimes = Vec::new(); - let mut removed = Vec::new(); - let mut missing = Vec::new(); - { - let mut state = self.state.lock().unwrap(); - for (terminal_id, expected_incarnation) in identities { - let Some(runtime) = self.catalog_terminal_by_host(&state, &terminal_id)? else { - missing.push((terminal_id, expected_incarnation)); - continue; - }; - let identity = self - .resource_terminal_host_identity(&runtime) - .context("catalog terminal has no host identity")?; - anyhow::ensure!( - expected_incarnation - .as_deref() - .is_none_or(|expected| identity.incarnation == expected), - "terminal-host incarnation changed while closing {terminal_id}" - ); - removed - .extend(remove_terminal_runtime_from_state(self, &mut state, &runtime).0); - runtimes.push(runtime); - } - } - for placement in removed { - self.purge_surface_side_tables(placement.id); - } - for runtime in runtimes { - self.purge_terminal_runtime_side_tables(&runtime); - self.terminate_terminal_runtime(&runtime); + fn terminate_prepared_terminal_hosts( + &self, + records: TerminalHostRecords, + ) -> anyhow::Result<()> { + #[cfg(test)] + if let Some(hook) = self.terminal_host_cleanup_hook.lock().unwrap().clone() { + anyhow::ensure!(hook(), "forced terminal-host cleanup failure"); + } + #[cfg(unix)] + { + for (path, record) in records { + anyhow::ensure!( + cleanup_terminal_host_record(&record, &path), + "could not clean up terminal host {}", + record.terminal_id + ); } - self.terminate_discovered_terminal_hosts(&missing) + Ok(()) + } + #[cfg(not(unix))] + { + let _ = records; + Ok(()) } + } - fn terminate_terminal_runtime(&self, runtime: &Arc) { - let identity = self.resource_terminal_host_identity(runtime); - #[cfg(unix)] - let acknowledged = match runtime - .terminate_host_and_wait_for_exit(Instant::now() + TERMINAL_HOST_CLOSE_WAIT) - { - Ok(Some((path, exit))) => acknowledge_exact_terminal_host_exit(&path, &exit), - Ok(None) => false, - Err(error) => { - if let Some(identity) = identity.as_ref() { - eprintln!( - "cmux-tui: terminal {} close could not await host exit: {error:#}", - identity.terminal_id - ); - } - false - } - }; - runtime.kill(); - #[cfg(unix)] - if !acknowledged && let Some(identity) = identity { - let _ = self.terminate_discovered_terminal_host( - &identity.terminal_id, - Some(&identity.incarnation), + fn terminate_workspace_hosts_from_receipt(&self, result: &Value) -> anyhow::Result<()> { + let identities = terminal_host_cleanup_identities(result)?; + let mut runtimes = Vec::new(); + let mut removed = Vec::new(); + let mut missing = Vec::new(); + { + let mut state = self.state.lock().unwrap(); + for (terminal_id, expected_incarnation) in identities { + let Some(runtime) = self.catalog_terminal_by_host(&state, &terminal_id)? else { + missing.push((terminal_id, expected_incarnation)); + continue; + }; + let identity = self + .resource_terminal_host_identity(&runtime) + .context("catalog terminal has no host identity")?; + anyhow::ensure!( + expected_incarnation + .as_deref() + .is_none_or(|expected| identity.incarnation == expected), + "terminal-host incarnation changed while closing {terminal_id}" ); + removed.extend(remove_terminal_runtime_from_state(self, &mut state, &runtime).0); + runtimes.push(runtime); } - #[cfg(not(unix))] - let _ = identity; } - - /// Run `f` with the session state. - /// - /// The state lock is held for the duration of `f`; do not call back - /// into `Mux` methods that take it (`surface()`, `close_pane()`, ...). - pub fn with_state(&self, f: impl FnOnce(&State) -> R) -> R { - f(&self.state.lock().unwrap()) + for placement in removed { + self.purge_surface_side_tables(placement.id); } - - pub fn surface_count(&self) -> usize { - self.state.lock().unwrap().surfaces.len() + for runtime in runtimes { + self.purge_terminal_runtime_side_tables(&runtime); + self.terminate_terminal_runtime(&runtime); } + self.terminate_discovered_terminal_hosts(&missing) + } - pub fn surface_notification(&self, surface: SurfaceId) -> Option { - let state = self.state.lock().unwrap(); - let terminal_id = state - .surfaces - .get(&surface) - .or_else(|| state.terminal_runtime_by_id(surface)) - .and_then(|surface| surface.terminal_public_id().cloned()); - drop(state); - match terminal_id { - Some(terminal_id) => { - self.terminal_notifications.lock().unwrap().get(&terminal_id).copied() + fn terminate_terminal_runtime(&self, runtime: &Arc) { + let identity = self.resource_terminal_host_identity(runtime); + #[cfg(unix)] + let acknowledged = match runtime + .terminate_host_and_wait_for_exit(Instant::now() + TERMINAL_HOST_CLOSE_WAIT) + { + Ok(Some((path, exit))) => acknowledge_exact_terminal_host_exit(&path, &exit), + Ok(None) => false, + Err(error) => { + if let Some(identity) = identity.as_ref() { + eprintln!( + "cmux-tui: terminal {} close could not await host exit: {error:#}", + identity.terminal_id + ); } - None => self.placement_notifications.lock().unwrap().get(&surface).copied(), + false } + }; + runtime.kill(); + #[cfg(unix)] + if !acknowledged && let Some(identity) = identity { + let _ = self.terminate_discovered_terminal_host( + &identity.terminal_id, + Some(&identity.incarnation), + ); } + #[cfg(not(unix))] + let _ = identity; + } - pub(crate) fn terminal_notification( - &self, - terminal_id: &TerminalPublicId, - ) -> Option { - self.terminal_notifications.lock().unwrap().get(terminal_id).copied() - } + /// Run `f` with the session state. + /// + /// The state lock is held for the duration of `f`; do not call back + /// into `Mux` methods that take it (`surface()`, `close_pane()`, ...). + pub fn with_state(&self, f: impl FnOnce(&State) -> R) -> R { + f(&self.state.lock().unwrap()) + } - pub fn surface_notifications(&self) -> HashMap { - let state = self.state.lock().unwrap(); - let placement_notifications = self.placement_notifications.lock().unwrap(); - let terminal_notifications = self.terminal_notifications.lock().unwrap(); - let mut result = HashMap::new(); - for (surface_id, surface) in &state.surfaces { - let notification = match surface.terminal_public_id() { - Some(terminal_id) => terminal_notifications.get(terminal_id), - None => placement_notifications.get(surface_id), - }; - if let Some(notification) = notification { - result.insert(*surface_id, *notification); - } + pub fn surface_count(&self) -> usize { + self.state.lock().unwrap().surfaces.len() + } + + pub fn surface_notification(&self, surface: SurfaceId) -> Option { + let state = self.state.lock().unwrap(); + let terminal_id = state + .surfaces + .get(&surface) + .or_else(|| state.terminal_runtime_by_id(surface)) + .and_then(|surface| surface.terminal_public_id().cloned()); + drop(state); + match terminal_id { + Some(terminal_id) => { + self.terminal_notifications.lock().unwrap().get(&terminal_id).copied() } - result + None => self.placement_notifications.lock().unwrap().get(&surface).copied(), } + } - pub fn clear_surface_notification(&self, surface: SurfaceId) -> bool { - let state = self.state.lock().unwrap(); - let terminal_id = state - .surfaces - .get(&surface) - .or_else(|| state.terminal_runtime_by_id(surface)) - .and_then(|surface| surface.terminal_public_id().cloned()); - drop(state); - let cleared = match terminal_id { - Some(terminal_id) => { - self.terminal_notifications.lock().unwrap().remove(&terminal_id).is_some() - } - None => self.placement_notifications.lock().unwrap().remove(&surface).is_some(), + pub(crate) fn terminal_notification( + &self, + terminal_id: &TerminalPublicId, + ) -> Option { + self.terminal_notifications.lock().unwrap().get(terminal_id).copied() + } + + pub fn surface_notifications(&self) -> HashMap { + let state = self.state.lock().unwrap(); + let placement_notifications = self.placement_notifications.lock().unwrap(); + let terminal_notifications = self.terminal_notifications.lock().unwrap(); + let mut result = HashMap::new(); + for (surface_id, surface) in &state.surfaces { + let notification = match surface.terminal_public_id() { + Some(terminal_id) => terminal_notifications.get(terminal_id), + None => placement_notifications.get(surface_id), }; - if cleared { - self.emit(MuxEvent::TreeChanged); + if let Some(notification) = notification { + result.insert(*surface_id, *notification); } - cleared } + result + } - fn active_surface_in_state(state: &State) -> Option { - let pane = state.active_pane()?; - state.panes.get(&pane)?.active_surface() + pub fn clear_surface_notification(&self, surface: SurfaceId) -> bool { + let state = self.state.lock().unwrap(); + let terminal_id = state + .surfaces + .get(&surface) + .or_else(|| state.terminal_runtime_by_id(surface)) + .and_then(|surface| surface.terminal_public_id().cloned()); + drop(state); + let cleared = match terminal_id { + Some(terminal_id) => { + self.terminal_notifications.lock().unwrap().remove(&terminal_id).is_some() + } + None => self.placement_notifications.lock().unwrap().remove(&surface).is_some(), + }; + if cleared { + self.emit(MuxEvent::TreeChanged); } + cleared + } + + fn active_surface_in_state(state: &State) -> Option { + let pane = state.active_pane()?; + state.panes.get(&pane)?.active_surface() + } - pub fn active_surface(&self) -> Option { - self.with_state(Self::active_surface_in_state) + pub fn active_surface(&self) -> Option { + self.with_state(Self::active_surface_in_state) + } + + /// Scroll one backend-owned view without mutating the terminal runtime + /// shared by the terminal's other placements. + pub fn scroll_surface_viewport(&self, surface: &Surface, delta: isize) -> anyhow::Result<()> { + let before = surface.view_scrollbar(); + let after = surface.view_scroll_delta(delta)?; + if after != before + && let Some(scrollbar) = after + { + self.emit(MuxEvent::ScrollChanged { + surface: surface.id, + offset: scrollbar.offset, + at_bottom: !scrollbar.scrolled_back(), + }); } + Ok(()) + } - /// Scroll one backend-owned view without mutating the terminal runtime - /// shared by the terminal's other placements. - pub fn scroll_surface_viewport( - &self, - surface: &Surface, - delta: isize, - ) -> anyhow::Result<()> { - let before = surface.view_scrollbar(); - let after = surface.view_scroll_delta(delta)?; - if after != before - && let Some(scrollbar) = after - { - self.emit(MuxEvent::ScrollChanged { - surface: surface.id, - offset: scrollbar.offset, - at_bottom: !scrollbar.scrolled_back(), - }); - } - Ok(()) + fn clear_viewed_notification(&self, surface: Option) { + let Some(surface) = surface else { return }; + let state = self.state.lock().unwrap(); + let terminal_id = state + .surfaces + .get(&surface) + .or_else(|| state.terminal_runtime_by_id(surface)) + .and_then(|surface| surface.terminal_public_id().cloned()); + drop(state); + if let Some(terminal_id) = terminal_id { + let _ = self.terminal_notifications.lock().unwrap().remove(&terminal_id); + } else { + let _ = self.placement_notifications.lock().unwrap().remove(&surface); } + } - fn clear_viewed_notification(&self, surface: Option) { - let Some(surface) = surface else { return }; + pub fn post_notification( + &self, + title: String, + body: String, + level: NotificationLevel, + surface: Option, + ) -> anyhow::Result { + let public_id = NotificationPublicId::random()?; + let terminal_id = surface.and_then(|surface| { let state = self.state.lock().unwrap(); - let terminal_id = state + state .surfaces .get(&surface) .or_else(|| state.terminal_runtime_by_id(surface)) - .and_then(|surface| surface.terminal_public_id().cloned()); - drop(state); - if let Some(terminal_id) = terminal_id { - let _ = self.terminal_notifications.lock().unwrap().remove(&terminal_id); - } else { - let _ = self.placement_notifications.lock().unwrap().remove(&surface); - } - } + .and_then(|surface| surface.terminal_public_id().cloned()) + }); + Ok(self.post_resource_notification( + public_id, + title, + body, + level, + surface, + terminal_id, + now_ms(), + )) + } - pub fn post_notification( - &self, - title: String, - body: String, - level: NotificationLevel, - surface: Option, - ) -> anyhow::Result { - let public_id = NotificationPublicId::random()?; - let terminal_id = surface.and_then(|surface| { - let state = self.state.lock().unwrap(); - state - .surfaces - .get(&surface) - .or_else(|| state.terminal_runtime_by_id(surface)) - .and_then(|surface| surface.terminal_public_id().cloned()) - }); - Ok(self.post_resource_notification( - public_id, - title, - body, + #[allow(clippy::too_many_arguments)] + pub(crate) fn post_resource_notification( + &self, + public_id: NotificationPublicId, + title: String, + body: String, + level: NotificationLevel, + surface: Option, + terminal_id: Option, + created_at_ms: u64, + ) -> u64 { + let id = self.next_notification_id(); + { + const NOTIFICATION_LEDGER_CAPACITY: usize = 256; + let mut ledger = self.notification_ledger.lock().unwrap(); + ledger.push_back(ResourceNotification { + id: public_id, + title: title.clone(), + body: body.clone(), level, + terminal_id: terminal_id.clone(), + created_at_ms, surface, - terminal_id, - now_ms(), - )) - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn post_resource_notification( - &self, - public_id: NotificationPublicId, - title: String, - body: String, - level: NotificationLevel, - surface: Option, - terminal_id: Option, - created_at_ms: u64, - ) -> u64 { - let id = self.next_notification_id(); - { - const NOTIFICATION_LEDGER_CAPACITY: usize = 256; - let mut ledger = self.notification_ledger.lock().unwrap(); - ledger.push_back(ResourceNotification { - id: public_id, - title: title.clone(), - body: body.clone(), - level, - terminal_id: terminal_id.clone(), - created_at_ms, - surface, - }); - while ledger.len() > NOTIFICATION_LEDGER_CAPACITY { - ledger.pop_front(); - } - } - // Shared topology focus is only a default projection. A frontend must - // explicitly acknowledge a viewed notification through its selection - // action, so local focus in one client cannot hide attention from the - // others. - let mut unread_changed = false; - match terminal_id { - Some(terminal_id) => { - self.terminal_notifications.lock().unwrap().insert( - terminal_id, - SurfaceNotification { notification: id, level, unread: true }, - ); - unread_changed = true; - } - None if surface.is_some() => { - let surface = surface.expect("checked notification surface"); - self.placement_notifications.lock().unwrap().insert( - surface, - SurfaceNotification { notification: id, level, unread: true }, - ); - unread_changed = true; - } - None => {} + }); + while ledger.len() > NOTIFICATION_LEDGER_CAPACITY { + ledger.pop_front(); + } + } + // Shared topology focus is only a default projection. A frontend must + // explicitly acknowledge a viewed notification through its selection + // action, so local focus in one client cannot hide attention from the + // others. + let mut unread_changed = false; + match terminal_id { + Some(terminal_id) => { + self.terminal_notifications.lock().unwrap().insert( + terminal_id, + SurfaceNotification { notification: id, level, unread: true }, + ); + unread_changed = true; } - self.emit(MuxEvent::Notification(NotificationEvent { - notification: id, - title, - body, - level, - surface, - })); - if unread_changed { - self.emit(MuxEvent::TreeChanged); + None if surface.is_some() => { + let surface = surface.expect("checked notification surface"); + self.placement_notifications + .lock() + .unwrap() + .insert(surface, SurfaceNotification { notification: id, level, unread: true }); + unread_changed = true; } - id + None => {} } + self.emit(MuxEvent::Notification(NotificationEvent { + notification: id, + title, + body, + level, + surface, + })); + if unread_changed { + self.emit(MuxEvent::TreeChanged); + } + id + } - pub fn resource_notifications(&self, limit: usize) -> Vec { - let mut notifications = self - .notification_ledger - .lock() - .unwrap() - .iter() - .rev() - .take(limit.min(256)) - .cloned() - .collect::>(); - let state = self.state.lock().unwrap(); - for notification in &mut notifications { - if let Some(terminal_id) = ¬ification.terminal_id { - notification.surface = state - .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) - .first() - .copied() - .or_else(|| { - state.terminal_catalog.get(terminal_id).map(|surface| surface.id) - }); - } + pub fn resource_notifications(&self, limit: usize) -> Vec { + let mut notifications = self + .notification_ledger + .lock() + .unwrap() + .iter() + .rev() + .take(limit.min(256)) + .cloned() + .collect::>(); + let state = self.state.lock().unwrap(); + for notification in &mut notifications { + if let Some(terminal_id) = ¬ification.terminal_id { + notification.surface = state + .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) + .first() + .copied() + .or_else(|| state.terminal_catalog.get(terminal_id).map(|surface| surface.id)); } - notifications } + notifications + } - pub fn report_agent( - &self, - surface: SurfaceId, - state: AgentState, - source: AgentSource, - session: Option, - ) -> anyhow::Result { - let mutation = WorkspaceMutation::new( - format!("raw-agent-{}", crate::workspace_registry::new_uuid_v4()), - "raw-control", - )?; - let fingerprint = serde_json::json!({ - "operation":"agent.report", - "surface":surface, - "state":state.as_str(), - "source":source.as_str(), - "source_session":session, - }); - let (_, record) = self.commit_agent_report( - AgentReportTarget::Surface(surface), - state, - source, - session, - None, - &mutation, - &fingerprint, - )?; - record.context("fresh raw agent report unexpectedly replayed") - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn resource_report_agent_selected( - &self, - selectors: crate::ResourceSelectors, - terminal_id: &TerminalPublicId, - agent_state: AgentState, - source: AgentSource, - source_session: Option, - expected_revision: Option, - mutation: &WorkspaceMutation, - ) -> anyhow::Result { - let fingerprint = serde_json::json!({ - "operation":"agent.report", - "selectors":selectors, - "terminal_id":terminal_id, - "state":agent_state.as_str(), - "source":source.as_str(), - "source_session":source_session, - }); - self.commit_agent_report( - AgentReportTarget::Resource { selectors: &selectors, terminal_id }, - agent_state, - source, - source_session, - expected_revision, - mutation, - &fingerprint, - ) - .map(|(commit, _)| commit) - } - - #[allow(clippy::too_many_arguments)] - fn commit_agent_report( - &self, - target: AgentReportTarget<'_>, - agent_state: AgentState, - source: AgentSource, - source_session: Option, - expected_revision: Option, - mutation: &WorkspaceMutation, - fingerprint: &Value, - ) -> anyhow::Result<(ResourcePatchCommit, Option)> { - let mut registry = self.workspace_registry.lock().unwrap(); - if let Some(replay) = - registry.replay_resource_patch(mutation, "agent.report", fingerprint)? - { - return Ok((replay, None)); + pub fn report_agent( + &self, + surface: SurfaceId, + state: AgentState, + source: AgentSource, + session: Option, + ) -> anyhow::Result { + let mutation = WorkspaceMutation::new( + format!("raw-agent-{}", crate::workspace_registry::new_uuid_v4()), + "raw-control", + )?; + let fingerprint = serde_json::json!({ + "operation":"agent.report", + "surface":surface, + "state":state.as_str(), + "source":source.as_str(), + "source_session":session, + }); + let (_, record) = self.commit_agent_report( + AgentReportTarget::Surface(surface), + state, + source, + session, + None, + &mutation, + &fingerprint, + )?; + record.context("fresh raw agent report unexpectedly replayed") + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn resource_report_agent_selected( + &self, + selectors: crate::ResourceSelectors, + terminal_id: &TerminalPublicId, + agent_state: AgentState, + source: AgentSource, + source_session: Option, + expected_revision: Option, + mutation: &WorkspaceMutation, + ) -> anyhow::Result { + let fingerprint = serde_json::json!({ + "operation":"agent.report", + "selectors":selectors, + "terminal_id":terminal_id, + "state":agent_state.as_str(), + "source":source.as_str(), + "source_session":source_session, + }); + self.commit_agent_report( + AgentReportTarget::Resource { selectors: &selectors, terminal_id }, + agent_state, + source, + source_session, + expected_revision, + mutation, + &fingerprint, + ) + .map(|(commit, _)| commit) + } + + #[allow(clippy::too_many_arguments)] + fn commit_agent_report( + &self, + target: AgentReportTarget<'_>, + agent_state: AgentState, + source: AgentSource, + source_session: Option, + expected_revision: Option, + mutation: &WorkspaceMutation, + fingerprint: &Value, + ) -> anyhow::Result<(ResourcePatchCommit, Option)> { + let mut registry = self.workspace_registry.lock().unwrap(); + if let Some(replay) = + registry.replay_resource_patch(mutation, "agent.report", fingerprint)? + { + return Ok((replay, None)); + } + let mut state = self.state.lock().unwrap(); + let (surface, terminal_id) = match target { + AgentReportTarget::Surface(surface) => { + let runtime = state + .surfaces + .get(&surface) + .or_else(|| state.terminal_runtime_by_id(surface)) + .with_context(|| format!("unknown surface {surface}"))?; + let identity = runtime.resource_identity().with_context(|| { + format!("surface {surface} has no durable resource identity") + })?; + let ContentPublicId::Terminal(terminal_id) = &identity.content_id else { + anyhow::bail!("surface {surface} is not a terminal"); + }; + (surface, terminal_id.clone()) } - let mut state = self.state.lock().unwrap(); - let (surface, terminal_id) = match target { - AgentReportTarget::Surface(surface) => { - let runtime = state - .surfaces - .get(&surface) - .or_else(|| state.terminal_runtime_by_id(surface)) - .with_context(|| format!("unknown surface {surface}"))?; - let identity = runtime.resource_identity().with_context(|| { - format!("surface {surface} has no durable resource identity") - })?; - let ContentPublicId::Terminal(terminal_id) = &identity.content_id else { - anyhow::bail!("surface {surface} is not a terminal"); - }; - (surface, terminal_id.clone()) - } - AgentReportTarget::Resource { selectors, terminal_id } => { - self.resolve_resource_path_in_state( - &state, - ®istry, - crate::ResourceTarget::Session, - selectors, - ) - .map_err(anyhow::Error::new)?; - let surface = state - .placements_of_content(&ContentPublicId::Terminal((*terminal_id).clone())) - .first() - .copied() - .or_else(|| { - state.terminal_catalog.get(terminal_id).map(|surface| surface.id) - }) - .with_context(|| format!("unknown terminal {terminal_id}"))?; - (surface, (*terminal_id).clone()) - } - }; - let now = now_ms(); - let mut records = self.agent_records.lock().unwrap(); - let record = match records.get(&terminal_id) { - Some(existing) - if existing.source == AgentSource::Hook && source == AgentSource::Socket => - { - existing.clone() - } - _ => TerminalAgentRecord { - state: agent_state, - source, - session: source_session, - updated_at_ms: now, - }, - }; - let digest = Sha256::digest(format!("cmux.protocol/2/agent/{terminal_id}").as_bytes()); - let payload = digest[..16].iter().map(|byte| format!("{byte:02x}")).collect::(); - let agent_id = - AgentPublicId::parse(format!("agent_{payload}")).map_err(anyhow::Error::new)?; - let session_id = registry.session_id().clone(); - let value = serde_json::json!({ - "id":agent_id, - "session_id":session_id, - "terminal_id":terminal_id, - "state":record.state.as_str(), - "source":record.source.as_str(), - "updated_at_ms":record.updated_at_ms.to_string(), - "source_session":record.session, - }); - let deltas = serde_json::json!([{ - "kind":"upsert", - "sequence":0, - "resource":"agent", - "id":agent_id, - "value":value, - }]); - let commit = registry.commit_agent_projection( - mutation, - fingerprint, - expected_revision, - &terminal_id, - &value, - &deltas, - )?; - state.resource_revision = commit.revision; - if !commit.replayed { - records.insert(terminal_id.clone(), record.clone()); + AgentReportTarget::Resource { selectors, terminal_id } => { + self.resolve_resource_path_in_state( + &state, + ®istry, + crate::ResourceTarget::Session, + selectors, + ) + .map_err(anyhow::Error::new)?; + let surface = state + .placements_of_content(&ContentPublicId::Terminal((*terminal_id).clone())) + .first() + .copied() + .or_else(|| state.terminal_catalog.get(terminal_id).map(|surface| surface.id)) + .with_context(|| format!("unknown terminal {terminal_id}"))?; + (surface, (*terminal_id).clone()) } - drop(records); - drop(state); - drop(registry); - let agent = AgentRecord { - surface, - terminal_id, - state: record.state, - source: record.source, - session: record.session, - updated_at_ms: record.updated_at_ms, - }; - if !commit.replayed { - self.publish_resource_event(); - self.emit(MuxEvent::AgentChanged { - surface: agent.surface, - state: Arc::from(agent.state.as_str()), - source: Arc::from(agent.source.as_str()), - session: agent.session.as_deref().map(Arc::from), - updated_at_ms: agent.updated_at_ms, - }); + }; + let now = now_ms(); + let mut records = self.agent_records.lock().unwrap(); + let record = match records.get(&terminal_id) { + Some(existing) + if existing.source == AgentSource::Hook && source == AgentSource::Socket => + { + existing.clone() } - Ok((commit, Some(agent))) + _ => TerminalAgentRecord { + state: agent_state, + source, + session: source_session, + updated_at_ms: now, + }, + }; + let digest = Sha256::digest(format!("cmux.protocol/2/agent/{terminal_id}").as_bytes()); + let payload = digest[..16].iter().map(|byte| format!("{byte:02x}")).collect::(); + let agent_id = + AgentPublicId::parse(format!("agent_{payload}")).map_err(anyhow::Error::new)?; + let session_id = registry.session_id().clone(); + let value = serde_json::json!({ + "id":agent_id, + "session_id":session_id, + "terminal_id":terminal_id, + "state":record.state.as_str(), + "source":record.source.as_str(), + "updated_at_ms":record.updated_at_ms.to_string(), + "source_session":record.session, + }); + let deltas = serde_json::json!([{ + "kind":"upsert", + "sequence":0, + "resource":"agent", + "id":agent_id, + "value":value, + }]); + let commit = registry.commit_agent_projection( + mutation, + fingerprint, + expected_revision, + &terminal_id, + &value, + &deltas, + )?; + state.resource_revision = commit.revision; + if !commit.replayed { + records.insert(terminal_id.clone(), record.clone()); } - - /// Drop per-surface metadata for a surface that has left the tree. - /// `SurfaceId` is monotonic, so without this every closed tab would - /// leak an entry forever and `list-agents` would keep reporting dead - /// surfaces as live agents. - fn purge_surface_side_tables(&self, surface: SurfaceId) { - let _lifecycle = self.lock_client_sizing_lifecycle(); - let mut sizing = self.client_sizing.lock().unwrap(); - sizing.surfaces.remove(&surface); - sizing.report_order.retain(|(reported_surface, _), _| *reported_surface != surface); - sizing.policies.remove(&surface); - sizing.terminal_runtime_by_placement.remove(&surface); - sizing.terminal_authorities.retain(|_, authority| authority.placement != surface); - drop(sizing); - self.placement_notifications.lock().unwrap().remove(&surface); + drop(records); + drop(state); + drop(registry); + let agent = AgentRecord { + surface, + terminal_id, + state: record.state, + source: record.source, + session: record.session, + updated_at_ms: record.updated_at_ms, + }; + if !commit.replayed { + self.publish_resource_event(); + self.emit(MuxEvent::AgentChanged { + surface: agent.surface, + state: Arc::from(agent.state.as_str()), + source: Arc::from(agent.source.as_str()), + session: agent.session.as_deref().map(Arc::from), + updated_at_ms: agent.updated_at_ms, + }); } + Ok((commit, Some(agent))) + } + + /// Drop per-surface metadata for a surface that has left the tree. + /// `SurfaceId` is monotonic, so without this every closed tab would + /// leak an entry forever and `list-agents` would keep reporting dead + /// surfaces as live agents. + fn purge_surface_side_tables(&self, surface: SurfaceId) { + let _lifecycle = self.lock_client_sizing_lifecycle(); + let mut sizing = self.client_sizing.lock().unwrap(); + sizing.surfaces.remove(&surface); + sizing.report_order.retain(|(reported_surface, _), _| *reported_surface != surface); + sizing.policies.remove(&surface); + sizing.terminal_runtime_by_placement.remove(&surface); + sizing.terminal_authorities.retain(|_, authority| authority.placement != surface); + drop(sizing); + self.placement_notifications.lock().unwrap().remove(&surface); + } - fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) { - self.agent_records.lock().unwrap().remove(terminal_id); - self.terminal_notifications.lock().unwrap().remove(terminal_id); - } + fn purge_terminal_side_tables(&self, terminal_id: &TerminalPublicId) { + self.agent_records.lock().unwrap().remove(terminal_id); + self.terminal_notifications.lock().unwrap().remove(terminal_id); + } - fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) { - if let Some(runtime_id) = runtime.terminal_runtime_id() { - self.reserved_in_process_terminals.lock().unwrap().remove(&runtime_id); - let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); - let mut pending = self.pending_cell_pixels.lock().unwrap(); - if let Some(update) = pending.as_mut() { - update.failures.remove(&runtime_id); - if update.failures.is_empty() { - let target = update.target; - *self.cell_pixels.lock().unwrap() = target; - *pending = None; - } + fn purge_terminal_runtime_side_tables(&self, runtime: &Surface) { + if let Some(runtime_id) = runtime.terminal_runtime_id() { + self.reserved_in_process_terminals.lock().unwrap().remove(&runtime_id); + let _cell_pixel_lifecycle = self.cell_pixel_lifecycle.lock().unwrap(); + let mut pending = self.pending_cell_pixels.lock().unwrap(); + if let Some(update) = pending.as_mut() { + update.failures.remove(&runtime_id); + if update.failures.is_empty() { + let target = update.target; + *self.cell_pixels.lock().unwrap() = target; + *pending = None; } } - if let Some(terminal_id) = runtime.terminal_public_id() { - self.purge_terminal_side_tables(terminal_id); - } } + if let Some(terminal_id) = runtime.terminal_public_id() { + self.purge_terminal_side_tables(terminal_id); + } + } + + /// Finish removed runtimes outside the topology lock. One shared deadline + /// bounds an entire pane/workspace close, and failures retain only the + /// minimal process/target owner needed for a later retry. + fn retire_surface_runtime(&self, surface: Arc) { + self.retire_surface_runtimes([surface]); + } - /// Finish removed runtimes outside the topology lock. One shared deadline - /// bounds an entire pane/workspace close, and failures retain only the - /// minimal process/target owner needed for a later retry. - fn retire_surface_runtime(&self, surface: Arc) { - self.retire_surface_runtimes([surface]); + /// Transfer an uninserted runtime from its admission reservation into the + /// retry ledger without opening a capacity-accounting gap. + fn retire_reserved_surface_runtime( + &self, + surface: Arc, + reservation: &mut SurfaceOwnerReservation<'_>, + ) { + surface.release_kitty_image_budget(); + let owner = { + let _state = self.state.lock().unwrap(); + let owner = self.shutdown_owners.stage_surface(&surface); + reservation.release(); + owner + }; + let Some(owner) = owner else { return }; + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { + self.shutdown_owner_reconciler.schedule(); + return; + }; + if self.terminate_shutdown_owners_until(vec![owner], deadline) { + self.shutdown_owner_reconciler.schedule(); } + } - /// Transfer an uninserted runtime from its admission reservation into the - /// retry ledger without opening a capacity-accounting gap. - fn retire_reserved_surface_runtime( - &self, - surface: Arc, - reservation: &mut SurfaceOwnerReservation<'_>, - ) { + fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { + let surfaces = surfaces.into_iter().collect::>(); + for surface in &surfaces { surface.release_kitty_image_budget(); - let owner = { - let _state = self.state.lock().unwrap(); - let owner = self.shutdown_owners.stage_surface(&surface); - reservation.release(); - owner - }; - let Some(owner) = owner else { return }; - let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; - let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { - self.shutdown_owner_reconciler.schedule(); - return; - }; - if self.terminate_shutdown_owners_until(vec![owner], deadline) { - self.shutdown_owner_reconciler.schedule(); - } } - - fn retire_surface_runtimes(&self, surfaces: impl IntoIterator>) { - let surfaces = surfaces.into_iter().collect::>(); - for surface in &surfaces { - surface.release_kitty_image_budget(); - } - let owners = surfaces - .into_iter() - .filter_map(|surface| self.shutdown_owners.stage_surface(&surface)) - .collect::>(); - let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; - let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { - self.shutdown_owner_reconciler.schedule(); - return; - }; - if self.terminate_shutdown_owners_until(owners, deadline) { - self.shutdown_owner_reconciler.schedule(); - } + let owners = surfaces + .into_iter() + .filter_map(|surface| self.shutdown_owners.stage_surface(&surface)) + .collect::>(); + let deadline = Instant::now() + SHUTDOWN_TERMINATION_TIMEOUT; + let Ok(_coordinator) = self.lock_shutdown_coordinator_until(deadline) else { + self.shutdown_owner_reconciler.schedule(); + return; + }; + if self.terminate_shutdown_owners_until(owners, deadline) { + self.shutdown_owner_reconciler.schedule(); } + } - fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) -> bool { - let owners = self.shutdown_owners.snapshot(); - self.terminate_shutdown_owners_until(owners, deadline) - } + fn terminate_staged_shutdown_owners_until(&self, deadline: Instant) -> bool { + let owners = self.shutdown_owners.snapshot(); + self.terminate_shutdown_owners_until(owners, deadline) + } - fn terminate_shutdown_owners_until( - &self, - owners: Vec<(ShutdownOwnerKey, Arc)>, - deadline: Instant, - ) -> bool { - if owners.is_empty() { - return false; + fn terminate_shutdown_owners_until( + &self, + owners: Vec<(ShutdownOwnerKey, Arc)>, + deadline: Instant, + ) -> bool { + if owners.is_empty() { + return false; + } + #[cfg(unix)] + let process_snapshot = crate::process_session::SessionProcessSnapshot::capture( + owners.iter().filter_map(|(_, owner)| owner.local_process_session()), + deadline, + ) + .ok(); + let mut single_work = Vec::new(); + let mut browser_batches = HashMap::< + *const BrowserRuntime, + (Arc, Vec<(ShutdownOwnerKey, Arc)>), + >::new(); + for (key, owner) in &owners { + if let Some(runtime) = owner.browser_runtime() { + browser_batches + .entry(Arc::as_ptr(runtime)) + .or_insert_with(|| (runtime.clone(), Vec::new())) + .1 + .push((key.clone(), owner.clone())); + } else { + single_work.push(ShutdownOwnerWork::Single((key.clone(), owner.clone()))); } - #[cfg(unix)] - let process_snapshot = crate::process_session::SessionProcessSnapshot::capture( - owners.iter().filter_map(|(_, owner)| owner.local_process_session()), - deadline, - ) - .ok(); - let mut single_work = Vec::new(); - let mut browser_batches = HashMap::< - *const BrowserRuntime, - (Arc, Vec<(ShutdownOwnerKey, Arc)>), - >::new(); - for (key, owner) in &owners { - if let Some(runtime) = owner.browser_runtime() { - browser_batches - .entry(Arc::as_ptr(runtime)) - .or_insert_with(|| (runtime.clone(), Vec::new())) - .1 - .push((key.clone(), owner.clone())); - } else { - single_work.push(ShutdownOwnerWork::Single((key.clone(), owner.clone()))); + } + // Start each high-fan-in browser batch before individual process work, + // so a saturated fanout cannot strand every target behind the deadline. + let mut work = browser_batches + .into_values() + .map(|(runtime, owners)| ShutdownOwnerWork::BrowserBatch { runtime, owners }) + .collect::>(); + work.extend(single_work); + + let failed = Mutex::new(HashSet::new()); + let started = bounded_shutdown_fanout(&work, deadline, |work, deadline| match work { + ShutdownOwnerWork::Single((key, owner)) => { + #[cfg(unix)] + let terminated = + owner.terminate_until_in_batch(deadline, process_snapshot.as_ref()); + #[cfg(not(unix))] + let terminated = owner.terminate_until(deadline); + if !terminated { + failed.lock().unwrap().insert(key.clone()); } } - // Start each high-fan-in browser batch before individual process work, - // so a saturated fanout cannot strand every target behind the deadline. - let mut work = browser_batches - .into_values() - .map(|(runtime, owners)| ShutdownOwnerWork::BrowserBatch { runtime, owners }) - .collect::>(); - work.extend(single_work); - - let failed = Mutex::new(HashSet::new()); - let started = bounded_shutdown_fanout(&work, deadline, |work, deadline| match work { - ShutdownOwnerWork::Single((key, owner)) => { - #[cfg(unix)] - let terminated = - owner.terminate_until_in_batch(deadline, process_snapshot.as_ref()); - #[cfg(not(unix))] - let terminated = owner.terminate_until(deadline); - if !terminated { - failed.lock().unwrap().insert(key.clone()); - } - } - ShutdownOwnerWork::BrowserBatch { runtime, owners } => { - let browser_owners = owners - .iter() - .map(|(_, owner)| { - owner - .browser_shutdown_owner() - .expect("browser batch has browser owners") - }) - .collect::>(); - let outcomes = runtime.close_owners_for_shutdown(&browser_owners, deadline); - let mut failed = failed.lock().unwrap(); - for (index, (key, _)) in owners.iter().enumerate() { - if !outcomes.get(index).copied().unwrap_or(false) { - failed.insert(key.clone()); - } + ShutdownOwnerWork::BrowserBatch { runtime, owners } => { + let browser_owners = owners + .iter() + .map(|(_, owner)| { + owner.browser_shutdown_owner().expect("browser batch has browser owners") + }) + .collect::>(); + let outcomes = runtime.close_owners_for_shutdown(&browser_owners, deadline); + let mut failed = failed.lock().unwrap(); + for (index, (key, _)) in owners.iter().enumerate() { + if !outcomes.get(index).copied().unwrap_or(false) { + failed.insert(key.clone()); } } - }); - let mut failed = failed.into_inner().unwrap(); - for work in &work[started..] { - work.record_failed_keys(&mut failed); - } - let confirmed = - owners.into_iter().filter(|(key, _)| !failed.contains(key)).collect::>(); - self.shutdown_owners.remove_confirmed(&confirmed); - !failed.is_empty() - } - - pub fn list_agents( - &self, - surface: Option, - state: Option, - ) -> Vec { - let records = self.agent_records.lock().unwrap().clone(); - let state_snapshot = self.state.lock().unwrap(); - let requested_terminal = surface.and_then(|surface| { - state_snapshot - .surfaces - .get(&surface) - .or_else(|| state_snapshot.terminal_runtime_by_id(surface)) - .and_then(|surface| surface.terminal_public_id().cloned()) - }); - let mut records = records - .into_iter() - .filter_map(|(terminal_id, record)| { - let representative = state_snapshot - .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) - .first() - .copied() - .or_else(|| { - state_snapshot - .terminal_catalog - .get(&terminal_id) - .map(|surface| surface.id) - })?; - Some(AgentRecord { - surface: representative, - terminal_id, - state: record.state, - source: record.source, - session: record.session, - updated_at_ms: record.updated_at_ms, - }) - }) - .collect::>(); - records - .sort_by(|left, right| left.terminal_id.as_str().cmp(right.terminal_id.as_str())); - records - .into_iter() - .filter(|record| { - requested_terminal - .as_ref() - .is_none_or(|terminal_id| &record.terminal_id == terminal_id) - }) - .filter(|record| state.is_none_or(|state| record.state == state)) - .collect() + } + }); + let mut failed = failed.into_inner().unwrap(); + for work in &work[started..] { + work.record_failed_keys(&mut failed); } + let confirmed = + owners.into_iter().filter(|(key, _)| !failed.contains(key)).collect::>(); + self.shutdown_owners.remove_confirmed(&confirmed); + !failed.is_empty() + } - fn shutdown_browser_runtimes_until(&self, deadline: Instant) -> bool { - let retained = self.shutdown_owners.snapshot(); - let current_browser_runtime = self.browser_runtime.take_for_shutdown(); - let mut browser_runtimes = Vec::new(); - for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { - if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { - browser_runtimes.push(runtime.clone()); - } - } - if let Some(runtime) = ¤t_browser_runtime - && !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) - { + pub fn list_agents( + &self, + surface: Option, + state: Option, + ) -> Vec { + let records = self.agent_records.lock().unwrap().clone(); + let state_snapshot = self.state.lock().unwrap(); + let requested_terminal = surface.and_then(|surface| { + state_snapshot + .surfaces + .get(&surface) + .or_else(|| state_snapshot.terminal_runtime_by_id(surface)) + .and_then(|surface| surface.terminal_public_id().cloned()) + }); + let mut records = records + .into_iter() + .filter_map(|(terminal_id, record)| { + let representative = state_snapshot + .placements_of_content(&ContentPublicId::Terminal(terminal_id.clone())) + .first() + .copied() + .or_else(|| { + state_snapshot.terminal_catalog.get(&terminal_id).map(|surface| surface.id) + })?; + Some(AgentRecord { + surface: representative, + terminal_id, + state: record.state, + source: record.source, + session: record.session, + updated_at_ms: record.updated_at_ms, + }) + }) + .collect::>(); + records.sort_by(|left, right| left.terminal_id.as_str().cmp(right.terminal_id.as_str())); + records + .into_iter() + .filter(|record| { + requested_terminal + .as_ref() + .is_none_or(|terminal_id| &record.terminal_id == terminal_id) + }) + .filter(|record| state.is_none_or(|state| record.state == state)) + .collect() + } + + fn shutdown_browser_runtimes_until(&self, deadline: Instant) -> bool { + let retained = self.shutdown_owners.snapshot(); + let current_browser_runtime = self.browser_runtime.take_for_shutdown(); + let mut browser_runtimes = Vec::new(); + for runtime in retained.iter().filter_map(|(_, owner)| owner.browser_runtime()) { + if !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) { browser_runtimes.push(runtime.clone()); } + } + if let Some(runtime) = ¤t_browser_runtime + && !browser_runtimes.iter().any(|candidate| Arc::ptr_eq(candidate, runtime)) + { + browser_runtimes.push(runtime.clone()); + } - let mut failed = false; - for runtime in browser_runtimes { - let is_current = current_browser_runtime - .as_ref() - .is_some_and(|current| Arc::ptr_eq(current, &runtime)); - let surface_failed = retained.iter().any(|(_, owner)| { - owner - .browser_runtime() - .is_some_and(|owner_runtime| Arc::ptr_eq(owner_runtime, &runtime)) - }); - if surface_failed && runtime.source() == BrowserSource::External { - if is_current { - self.browser_runtime.restore_for_shutdown(runtime); - } - continue; + let mut failed = false; + for runtime in browser_runtimes { + let is_current = current_browser_runtime + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &runtime)); + let surface_failed = retained.iter().any(|(_, owner)| { + owner + .browser_runtime() + .is_some_and(|owner_runtime| Arc::ptr_eq(owner_runtime, &runtime)) + }); + if surface_failed && runtime.source() == BrowserSource::External { + if is_current { + self.browser_runtime.restore_for_shutdown(runtime); } - if runtime.shutdown_until(deadline) { - if runtime.source() == BrowserSource::Launched { - self.shutdown_owners.remove_browser_runtime(&runtime); - } - } else { - failed = true; - if is_current { - self.browser_runtime.restore_for_shutdown(runtime); - } + continue; + } + if runtime.shutdown_until(deadline) { + if runtime.source() == BrowserSource::Launched { + self.shutdown_owners.remove_browser_runtime(&runtime); + } + } else { + failed = true; + if is_current { + self.browser_runtime.restore_for_shutdown(runtime); } } - failed } + failed + } - fn shutdown_once_until(&self, deadline: Instant) -> anyhow::Result<()> { - #[cfg(unix)] - crate::process_session::require_stable_process_signaling_until(deadline) - .context("preflight process control for daemon exit")?; - let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; - self.shutting_down.store(true, Ordering::Release); - self.browser_runtime.stop(); - self.journal_kernel.wake_waiters(); - let hook_deadline = deadline.min(Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT); - if !self.journal_hook_runtime.shutdown_until(hook_deadline) { - eprintln!( - "cmux-tui: journal hook workers did not stop before the shutdown deadline" - ); - } - #[cfg(unix)] - self.request_terminal_adoption_stop(); - if !self.surface_creations.stop_and_wait_until(deadline) { - anyhow::bail!("surface creation remains active at the daemon exit deadline"); - } - if !self.async_surface_creations.stop_and_wait_until(deadline) { - anyhow::bail!("browser bootstrap remains active at the daemon exit deadline"); - } - #[cfg(unix)] - if !self.wait_for_terminal_adoption_workers_until(deadline) { - anyhow::bail!("terminal adoption remains active at the daemon exit deadline"); - } - #[cfg(unix)] - crate::process_session::wait_for_unpublished_child_reaper_until(deadline) - .context("drain unpublished child reaper ownership for daemon exit")?; + fn shutdown_once_until(&self, deadline: Instant) -> anyhow::Result<()> { + #[cfg(unix)] + crate::process_session::require_stable_process_signaling_until(deadline) + .context("preflight process control for daemon exit")?; + let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; + self.shutting_down.store(true, Ordering::Release); + self.browser_runtime.stop(); + self.journal_kernel.wake_waiters(); + let hook_deadline = deadline.min(Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT); + if !self.journal_hook_runtime.shutdown_until(hook_deadline) { + eprintln!("cmux-tui: journal hook workers did not stop before the shutdown deadline"); + } + #[cfg(unix)] + self.request_terminal_adoption_stop(); + if !self.surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("surface creation remains active at the daemon exit deadline"); + } + if !self.async_surface_creations.stop_and_wait_until(deadline) { + anyhow::bail!("browser bootstrap remains active at the daemon exit deadline"); + } + #[cfg(unix)] + if !self.wait_for_terminal_adoption_workers_until(deadline) { + anyhow::bail!("terminal adoption remains active at the daemon exit deadline"); + } + #[cfg(unix)] + crate::process_session::wait_for_unpublished_child_reaper_until(deadline) + .context("drain unpublished child reaper ownership for daemon exit")?; - let surfaces = unique_surface_runtimes(&self.state.lock().unwrap()); - for surface in &surfaces { - let _ = self.shutdown_owners.stage_local_process_surface(surface); - } - let terminal_reader_deadline = - deadline.min(Instant::now() + TERMINAL_READER_SHUTDOWN_TIMEOUT); - let mut terminal_gaps = surfaces - .iter() - .filter_map(|surface| surface.shutdown_for_daemon(terminal_reader_deadline)) - .collect::>(); - for surface in surfaces { - terminal_gaps.extend(surface.finish_terminal_reader(terminal_reader_deadline)); - } - for gap in terminal_gaps { - if let Err(error) = self.journal_ingress.send_durable( - crate::journal_ingress::JournalIngressEvent::TerminalOutputGap { - terminal_id: gap.terminal_id, - generation: gap.generation, - occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), - reason: gap.reason, - }, - ) { - eprintln!("cmux-tui: record terminal output gap during shutdown: {error:#}"); - } - } - if let Err(error) = self.flush_terminal_journal() { - eprintln!("cmux-tui: flush terminal journal during shutdown: {error:#}"); - } - if let Err(error) = self.journal_ingress.close_and_join() { - eprintln!("cmux-tui: stop session journal writer during shutdown: {error:#}"); + let surfaces = unique_surface_runtimes(&self.state.lock().unwrap()); + for surface in &surfaces { + let _ = self.shutdown_owners.stage_local_process_surface(surface); + } + let terminal_reader_deadline = + deadline.min(Instant::now() + TERMINAL_READER_SHUTDOWN_TIMEOUT); + let mut terminal_gaps = surfaces + .iter() + .filter_map(|surface| surface.shutdown_for_daemon(terminal_reader_deadline)) + .collect::>(); + for surface in surfaces { + terminal_gaps.extend(surface.finish_terminal_reader(terminal_reader_deadline)); + } + for gap in terminal_gaps { + if let Err(error) = self.journal_ingress.send_durable( + crate::journal_ingress::JournalIngressEvent::TerminalOutputGap { + terminal_id: gap.terminal_id, + generation: gap.generation, + occurred_at_ms: crate::workspace_registry::unix_epoch_ms().unwrap_or(0), + reason: gap.reason, + }, + ) { + eprintln!("cmux-tui: record terminal output gap during shutdown: {error:#}"); } - self.journal_kernel.shutdown(); + } + if let Err(error) = self.flush_terminal_journal() { + eprintln!("cmux-tui: flush terminal journal during shutdown: {error:#}"); + } + if let Err(error) = self.journal_ingress.close_and_join() { + eprintln!("cmux-tui: stop session journal writer during shutdown: {error:#}"); + } + self.journal_kernel.shutdown(); - self.terminate_staged_shutdown_owners_until(deadline); - let browser_runtime_failed = self.shutdown_browser_runtimes_until(deadline); - let retained = self.shutdown_owners.len(); - if retained != 0 || browser_runtime_failed { - anyhow::bail!( - "daemon exit retained {retained} surface owner(s); browser runtime failure: \ + self.terminate_staged_shutdown_owners_until(deadline); + let browser_runtime_failed = self.shutdown_browser_runtimes_until(deadline); + let retained = self.shutdown_owners.len(); + if retained != 0 || browser_runtime_failed { + anyhow::bail!( + "daemon exit retained {retained} surface owner(s); browser runtime failure: \ {browser_runtime_failed}" - ); - } + ); } Ok(()) } @@ -18582,7 +18399,7 @@ fn terminal_host_cleanup_identities( Ok(identities) } -fn remove_terminal_host_cleanup_receipt(result: &mut Value) { +pub(crate) fn remove_terminal_host_cleanup_receipt(result: &mut Value) { if let Some(result) = result.as_object_mut() { result.remove(TERMINAL_HOST_CLEANUP_RECEIPT_FIELD); } diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_router/effects.rs b/cmux-tui/crates/cmux-tui-core/src/resource_router/effects.rs index 84fe2ab63b57..06b6cc1e28c3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_router/effects.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_router/effects.rs @@ -79,7 +79,7 @@ fn resolve_preparation( if operation == "terminal.close" { mux.terminate_workspace_hosts_from_receipt(&value) .map_err(resource_operation_error)?; - crate::remove_terminal_host_cleanup_receipt(&mut value); + crate::mux::remove_terminal_host_cleanup_receipt(&mut value); } mutation_result_for_operation(mux, &operation, value, revision, true) } diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 32df958c1a3d..b1a6e4eeb9a1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -33,8 +33,7 @@ use crate::terminal_host_protocol::{ KITTY_IMAGE_ALIAS_COUNT_LEN, KITTY_IMAGE_ALIAS_ENCODED_LEN, LAUNCH_ACTIVATION_PROTOCOL_VERSION, MAX_FRAME_PAYLOAD, MAX_KITTY_IMAGE_ALIASES, MessageKind, PROTOCOL_VERSION, RESIZE_ACK_CANONICAL_CHANGED, TerminalExit, decode_host_launch_failure, decode_terminal_exit, - encode_host_launch_failure, encode_terminal_exit, read_frame, wait_for_native_child_status, - write_frame, + encode_host_launch_failure, encode_terminal_exit, read_frame, write_frame, }; const HOST_RECORD_VERSION: u32 = 4; From 20f6c2289dfa53c9b84007142728b7741297b6b7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:15:44 -0700 Subject: [PATCH 386/521] refactor(macOS): isolate teardown lifecycle types --- ...TerminalSurfaceRuntimeActiveTeardown.swift | 6 +++++ ...inalSurfaceRuntimeOwnershipAdmission.swift | 2 ++ ...urfaceRuntimeOwnershipAdmissionState.swift | 8 ------- ...nalSurfaceRuntimeTeardownCoordinator.swift | 22 ------------------- ...aceRuntimeTeardownIngressReservation.swift | 9 ++++++++ ...minalSurfaceRuntimeTeardownStartGate.swift | 4 +++- ...inalSurfaceRuntimeTeardownSubmission.swift | 18 +++++++++++++++ 7 files changed, 38 insertions(+), 31 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownIngressReservation.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmission.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift new file mode 100644 index 000000000000..9303590fa226 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift @@ -0,0 +1,6 @@ +import Foundation + +internal struct TerminalSurfaceRuntimeActiveTeardown: Sendable { + let ticketID: UUID + let task: Task +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index a24e9eaf8891..e36b87177bb9 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -8,6 +8,8 @@ typealias TerminalSurfaceRuntimeOwnershipRecovery = /// /// Creation, public teardown, and deinit cannot suspend while transferring a /// native pointer, so this ledger uses one short unfair-lock critical section. +/// An actor would make creation and deinit await while they hold sole pointer +/// custody, so the lock is the explicit synchronous isolation boundary. /// Each ownership reservation also holds one submission-ingress slot until the /// teardown submission is dequeued or the unused ownership is released. /// Native work never runs under the lock. A fully stalled close pool degrades diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index c187f55cd5fc..0b5029123964 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -1,13 +1,5 @@ import Foundation -struct TerminalSurfaceRuntimeTeardownIngressReservation: Sendable { - let id: UUID - - init(id: UUID = UUID()) { - self.id = id - } -} - struct TerminalSurfaceRuntimeOwnershipAdmissionState { var reservationIDs: Set = [] /// Ingress slots held by live owners or by ordered control submissions. diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index dd4ec89aa4b8..9a89a2f5324e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -5,28 +5,6 @@ public import CmuxTerminalCore internal import CMUXDebugLog #endif -private enum TerminalSurfaceRuntimeTeardownSubmission: Sendable { - case enqueue( - request: TerminalSurfaceRuntimeTeardownRequest, - executionLane: TerminalSurfaceRuntimeTeardownExecutionLane, - isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation?, - ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation - ) - case cancel( - ticketID: UUID, - result: AsyncStream.Continuation, - ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation - ) - case cancelAll( - ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation - ) -} - -private struct TerminalSurfaceRuntimeActiveTeardown: Sendable { - let ticketID: UUID - let task: Task -} - /// Coordinates native `ghostty_surface_free` calls off the close/deinit paths. /// /// Close/deinit frees use two independently startable utility slots. Each slot diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownIngressReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownIngressReservation.swift new file mode 100644 index 000000000000..f4e89769c676 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownIngressReservation.swift @@ -0,0 +1,9 @@ +import Foundation + +internal struct TerminalSurfaceRuntimeTeardownIngressReservation: Sendable { + let id: UUID + + init(id: UUID = UUID()) { + self.id = id + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift index 41df806dafb7..c3ee3c86800e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift @@ -2,7 +2,9 @@ internal import os /// Safety: the lock protects every state transition and the stored continuation. /// The single waiter and idempotent start move that continuation out at most -/// once, and resume it only after releasing the lock. +/// once, and resume it only after releasing the lock. The coordinator must +/// store the prepared Task before `start()` releases it; actor suspension at +/// this boundary could break that order. final class TerminalSurfaceRuntimeTeardownStartGate: @unchecked Sendable { private enum State { case pending diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmission.swift new file mode 100644 index 000000000000..750d5558c7f7 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmission.swift @@ -0,0 +1,18 @@ +import Foundation + +internal enum TerminalSurfaceRuntimeTeardownSubmission: Sendable { + case enqueue( + request: TerminalSurfaceRuntimeTeardownRequest, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane, + isolatedHibernationReservation: TerminalSurfaceRuntimeTeardownReservation?, + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation + ) + case cancel( + ticketID: UUID, + result: AsyncStream.Continuation, + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation + ) + case cancelAll( + ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation + ) +} From 5485a1b0e46cfe08397c2eed46fa243f52b69735 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:43:52 -0700 Subject: [PATCH 387/521] test: retain accepted teardown coordinator --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index e4ba8b526d06..2290f27b8eba 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -36,6 +36,49 @@ private final class FreedSurfaceRecorder: @unchecked Sendable { } } +/// Holds preferred-executor jobs until a test opens the gate. +private final class GatedTaskExecutor: TaskExecutor, @unchecked Sendable { + private struct State { + var isOpen = true + var jobs: [UnownedJob] = [] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + func enqueue(_ job: consuming ExecutorJob) { + let job = UnownedJob(job) + let shouldRun = state.withLock { state in + guard !state.isOpen else { return true } + state.jobs.append(job) + return false + } + if shouldRun { + job.runSynchronously(on: asUnownedTaskExecutor()) + } + } + + func close() { + state.withLock { state in + precondition(state.isOpen) + precondition(state.jobs.isEmpty) + state.isOpen = false + } + } + + func open() { + let jobs = state.withLock { state in + guard !state.isOpen else { return [UnownedJob]() } + state.isOpen = true + let jobs = state.jobs + state.jobs.removeAll(keepingCapacity: true) + return jobs + } + for job in jobs { + job.runSynchronously(on: asUnownedTaskExecutor()) + } + } +} + private final class TeardownLifetimeRecorder: @unchecked Sendable { let events: AsyncStream private let continuation: AsyncStream.Continuation @@ -85,6 +128,57 @@ private func requireTeardownTicket( #expect(await ticket.wait(timeout: nil)) } + @Test func acceptedTeardownRetainsCoordinatorUntilTicketCompletion() async throws { + let executor = GatedTaskExecutor() + defer { executor.open() } + let recorder = FreedSurfaceRecorder() + let surface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + defer { surface.deallocate() } + + var accepted = try await withTaskExecutorPreference(executor) { + executor.close() + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() + let runtimeReservation = try #require( + coordinator.reserveRuntimeSurfaceOwnership() + ) + let ticket = try requireTeardownTicket( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.acceptedCoordinatorLifetime", + surface: surface, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: runtimeReservation, + freeSurface: { pointer in + recorder.record(UInt(bitPattern: pointer)) + } + ) + ) + return (coordinator: Optional(coordinator), ticket: ticket) + } + weak var weakCoordinator = accepted.coordinator + + accepted.coordinator = nil + + try #require(weakCoordinator != nil) + executor.open() + await recorder.waitForFreeCount(1) + #expect(await accepted.ticket.wait(timeout: nil)) + #expect(recorder.freed == [UInt(bitPattern: surface)]) + } + + @Test func idleCoordinatorDoesNotRetainItself() { + var coordinator: TerminalSurfaceRuntimeTeardownCoordinator? = + TerminalSurfaceRuntimeTeardownCoordinator() + weak var weakCoordinator = coordinator + + coordinator = nil + + #expect(weakCoordinator == nil) + } + @Test func enqueuedTeardownInvokesInjectedFreeWithTheSamePointer() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let recorder = FreedSurfaceRecorder() From c1d87f092905f95199f5c7614d1cdd7ff349315f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:55:56 -0700 Subject: [PATCH 388/521] fix: retain accepted teardown submissions --- ...nalSurfaceRuntimeTeardownCoordinator.swift | 68 +++++++------------ ...urfaceRuntimeTeardownSubmissionDrain.swift | 34 ++++++++++ 2 files changed, 60 insertions(+), 42 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmissionDrain.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 9a89a2f5324e..a87916e89433 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -28,9 +28,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Largest batch that can own independently startable native-free slots. public static let maximumIsolatedHibernationTeardownCount = 2 - private nonisolated let submissionContinuation: - AsyncStream.Continuation - private var submissionTask: Task? + private nonisolated let submissionDrain: + TerminalSurfaceRuntimeTeardownSubmissionDrain #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in // TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift can report the @@ -61,56 +60,32 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Creates the process's teardown coordinator. public init() { - let submissions = AsyncStream< - TerminalSurfaceRuntimeTeardownSubmission - >.makeStream( - bufferingPolicy: .bufferingOldest( + submissionDrain = TerminalSurfaceRuntimeTeardownSubmissionDrain( + maximumBufferedOperationCount: Self.maximumRuntimeSurfaceOwnerCount - ) ) - submissionContinuation = submissions.continuation - submissionTask = nil runtimeOwnershipAdmission = TerminalSurfaceRuntimeOwnershipAdmission( maximumOwnerCount: Self.maximumRuntimeSurfaceOwnerCount ) availableCloseExecutionSlots = Set( 0...makeStream( - bufferingPolicy: .bufferingOldest( + submissionDrain = TerminalSurfaceRuntimeTeardownSubmissionDrain( + maximumBufferedOperationCount: maximumRuntimeSurfaceOwnerCount - ) ) - submissionContinuation = submissions.continuation - submissionTask = nil runtimeOwnershipAdmission = TerminalSurfaceRuntimeOwnershipAdmission( maximumOwnerCount: maximumRuntimeSurfaceOwnerCount ) availableCloseExecutionSlots = Set( 0...makeStream( bufferingPolicy: .bufferingNewest(1) ) - switch submissionContinuation.yield( - .cancel( - ticketID: ticketID, - result: result.continuation, - ingressReservation: ingressReservation - ) - ) { + let submission = TerminalSurfaceRuntimeTeardownSubmission.cancel( + ticketID: ticketID, + result: result.continuation, + ingressReservation: ingressReservation + ) + let yieldResult = submissionDrain.yield { [self] in + await receive(submission) + } + switch yieldResult { case .enqueued: var iterator = result.stream.makeAsyncIterator() return await iterator.next() ?? false @@ -299,9 +279,13 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { runtimeOwnershipAdmission.reserveControlIngress() else { return } - switch submissionContinuation.yield( - .cancelAll(ingressReservation: ingressReservation) - ) { + let submission = TerminalSurfaceRuntimeTeardownSubmission.cancelAll( + ingressReservation: ingressReservation + ) + let yieldResult = submissionDrain.yield { [self] in + await receive(submission) + } + switch yieldResult { case .enqueued: break case .dropped, .terminated: diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmissionDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmissionDrain.swift new file mode 100644 index 000000000000..872d88fddca9 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownSubmissionDrain.swift @@ -0,0 +1,34 @@ +/// Drains accepted teardown operations in ingress order. +/// +/// The consumer Task captures only the stream. Each buffered operation retains +/// its coordinator until that operation reaches the actor receive boundary. +internal final class TerminalSurfaceRuntimeTeardownSubmissionDrain: Sendable { + internal typealias Operation = @Sendable () async -> Void + + private let continuation: AsyncStream.Continuation + private let consumerTask: Task + + internal init(maximumBufferedOperationCount: Int) { + let submissions = AsyncStream.makeStream( + bufferingPolicy: .bufferingOldest(maximumBufferedOperationCount) + ) + let stream = submissions.stream + continuation = submissions.continuation + consumerTask = Task { + for await operation in stream { + await operation() + } + } + } + + deinit { + continuation.finish() + consumerTask.cancel() + } + + internal func yield( + _ operation: @escaping Operation + ) -> AsyncStream.Continuation.YieldResult { + continuation.yield(operation) + } +} From 16f96ad09545d4965ecc53404679782248df28b5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:09:03 -0700 Subject: [PATCH 389/521] fix(tui): restore merged browser failure owner --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 337ec9c87be2..5286a300c847 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8963,6 +8963,15 @@ impl Mux { } } + fn report_browser_bootstrap_failure(&self, surface: &Arc, error: String) { + if let Surface::Browser(browser) = surface.as_ref() { + browser.mark_failed(error.clone()); + } + self.emit(MuxEvent::Status(format!("browser failed: {error}"))); + self.emit(MuxEvent::TitleChanged { surface: surface.id, title: surface.title().into() }); + self.emit(MuxEvent::SurfaceOutput(surface.id)); + } + pub(crate) fn restart_provider_browser_surface(self: &Arc, surface: Arc) { let Some(identity) = surface.resource_identity() else { return }; if !matches!(identity.content_id, ContentPublicId::Browser(_)) { @@ -9369,7 +9378,10 @@ impl Mux { } } - fn terminate_workspace_hosts_from_receipt(&self, result: &Value) -> anyhow::Result<()> { + pub(crate) fn terminate_workspace_hosts_from_receipt( + &self, + result: &Value, + ) -> anyhow::Result<()> { let identities = terminal_host_cleanup_identities(result)?; let mut runtimes = Vec::new(); let mut removed = Vec::new(); From 443f27405e00774e3a0cb52cb0081353cf2d8f32 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:17:26 -0700 Subject: [PATCH 390/521] test(tui): implement ordered control sink --- cmux-tui/crates/cmux-tui-core/src/server.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 8c66daf899b5..b8b8ecaee1ef 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -19245,6 +19245,14 @@ mod tests { Ok(()) } + fn send_ordered_terminal( + &self, + _text: Arc, + _stream: &OutboundStream, + ) -> std::io::Result<()> { + Ok(()) + } + fn is_open(&self) -> bool { self.open.load(Ordering::Acquire) } From a135cc20e4860eb6e5649f71abec13bf525eb4ad Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:48:23 -0700 Subject: [PATCH 391/521] test: preserve overflowed surface creation --- .../RecordingRestoreSpawnScheduler.swift | 10 + ...nalSurfaceRestoreSpawnSchedulerTests.swift | 243 +++++++++++++++++- 2 files changed, 252 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/RecordingRestoreSpawnScheduler.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/RecordingRestoreSpawnScheduler.swift index c6440896fb4e..627bbe3116ea 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/RecordingRestoreSpawnScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/RecordingRestoreSpawnScheduler.swift @@ -5,13 +5,23 @@ import Foundation final class RecordingRestoreSpawnScheduler: TerminalSurfaceRuntimeSpawnScheduling { private(set) var scheduledSurfaceIds: [UUID] = [] private var scheduledOperations: [@MainActor () -> Void] = [] + private let scheduledEvents = AsyncStream.makeStream() func scheduleRestoredSurfaceSpawn(surfaceId: UUID, operation: @escaping @MainActor () -> Void) { scheduledSurfaceIds.append(surfaceId) scheduledOperations.append(operation) + scheduledEvents.continuation.yield(surfaceId) } func runScheduledOperation(at index: Int = 0) { scheduledOperations[index]() } + + func waitForScheduledCount(_ count: Int) async { + guard scheduledSurfaceIds.count < count else { return } + for await _ in scheduledEvents.stream { + guard scheduledSurfaceIds.count >= count else { continue } + return + } + } } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 1423de920115..3fba4fb8fcd7 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -365,6 +365,185 @@ import CmuxTerminalCore } } + @Test func overflowedCreationRetriesInAdmissionOrder() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let saturation = try saturateRuntimeOwnershipRecovery(coordinator) + defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let fixtures = (0..<2).map { _ in + makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + } + + for (index, fixture) in fixtures.enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + try #require( + fixture.surface.runtimeSurfaceAdmissionDeferredCreationSource + == .scheduledRestore + ) + } + + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + await scheduler.waitForScheduledCount(4) + + #expect( + scheduler.scheduledSurfaceIds == [ + fixtures[0].surface.id, + fixtures[1].surface.id, + fixtures[0].surface.id, + fixtures[1].surface.id, + ] + ) + } + + @Test func overflowStoreSurvivesImmediateRecoveryCapacityRelease() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let saturation = try saturateRuntimeOwnershipRecovery(coordinator) + defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) } + let scheduler = RecordingRestoreSpawnScheduler() + let fixture = makeSurfaceFixture( + registry: FakeSurfaceRegistry(), + scheduler: scheduler, + runtimeTeardown: coordinator + ) + + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation() + try #require( + fixture.surface.runtimeSurfaceAdmissionDeferredCreationSource + == .scheduledRestore + ) + + coordinator.cancelRuntimeSurfaceOwnershipRecovery( + saturation.recoveryIDs[0] + ) + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + await scheduler.waitForScheduledCount(2) + + #expect( + scheduler.scheduledSurfaceIds == [ + fixture.surface.id, + fixture.surface.id, + ] + ) + } + + @Test func repeatedOverflowForOneSurfaceCoalescesAndPromotesSource() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let saturation = try saturateRuntimeOwnershipRecovery(coordinator) + defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let repeated = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + let sentinel = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + + repeated.surface.createSurface(for: repeated.nativeView) + scheduler.runScheduledOperation(at: 0) + repeated.surface.createSurface( + for: repeated.nativeView, + source: .inputDemand + ) + try #require( + repeated.surface.runtimeSurfaceAdmissionDeferredCreationSource + == .inputDemand + ) + sentinel.surface.createSurface(for: sentinel.nativeView) + scheduler.runScheduledOperation(at: 1) + try #require( + sentinel.surface.runtimeSurfaceAdmissionDeferredCreationSource + == .scheduledRestore + ) + + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + await scheduler.waitForScheduledCount(3) + + #expect( + repeated.surface.debugRuntimeSurfaceCreateAttemptCountForTesting() + == 3 + ) + #expect( + scheduler.scheduledSurfaceIds == [ + repeated.surface.id, + sentinel.surface.id, + sentinel.surface.id, + ] + ) + } + + @Test func deinitializedOverflowSurfaceIsNotRetried() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let saturation = try saturateRuntimeOwnershipRecovery(coordinator) + defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + var deadFixture: ( + surface: TerminalSurface?, + nativeView: FakeTerminalSurfaceNativeView + ) = { + let fixture = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + return (Optional(fixture.surface), fixture.nativeView) + }() + let deadSurfaceID = try #require(deadFixture.surface?.id) + weak var weakDeadSurface = deadFixture.surface + + deadFixture.surface?.createSurface(for: deadFixture.nativeView) + scheduler.runScheduledOperation(at: 0) + try #require( + deadFixture.surface?.runtimeSurfaceAdmissionDeferredCreationSource + == .scheduledRestore + ) + deadFixture.surface = nil + try #require(weakDeadSurface == nil) + + let sentinel = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + sentinel.surface.createSurface(for: sentinel.nativeView) + scheduler.runScheduledOperation(at: 1) + try #require( + sentinel.surface.runtimeSurfaceAdmissionDeferredCreationSource + == .scheduledRestore + ) + + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + await scheduler.waitForScheduledCount(3) + + #expect( + scheduler.scheduledSurfaceIds.filter { $0 == deadSurfaceID }.count + == 1 + ) + #expect( + scheduler.scheduledSurfaceIds.last == sentinel.surface.id + ) + } + @Test func configurationReloadDefersAndPromotesRuntimeCreation() { let nativeView = FakeTerminalSurfaceNativeView( frame: NSRect(x: 0, y: 0, width: 800, height: 600) @@ -577,6 +756,7 @@ import CmuxTerminalCore scheduler: RecordingRestoreSpawnScheduler, nativeView: FakeTerminalSurfaceNativeView, paneHost: FakeTerminalSurfacePaneHost, + registry: any TerminalSurfaceRegistering = FakeSurfaceRegistry(), engine: FakeTerminalEngine = FakeTerminalEngine(), runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator = TerminalSurfaceRuntimeTeardownCoordinator(), @@ -592,7 +772,7 @@ import CmuxTerminalCore configTemplate: nil, runtimeSpawnPolicy: runtimeSpawnPolicy, dependencies: TerminalSurfaceRuntimeDependencies( - registry: FakeSurfaceRegistry(), + registry: registry, engine: engine, viewProvider: FakeTerminalSurfaceViewProvider(surfaceView: nativeView, paneHost: paneHost), spawnPolicy: FakeSpawnPolicyProvider(), @@ -608,4 +788,65 @@ import CmuxTerminalCore ) ) } + + private func makeSurfaceFixture( + registry: any TerminalSurfaceRegistering, + scheduler: RecordingRestoreSpawnScheduler, + runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator + ) -> ( + surface: TerminalSurface, + nativeView: FakeTerminalSurfaceNativeView + ) { + let nativeView = FakeTerminalSurfaceNativeView( + frame: NSRect(x: 0, y: 0, width: 800, height: 600) + ) + let paneHost = FakeTerminalSurfacePaneHost(surfaceView: nativeView) + let surface = makeSurface( + scheduler: scheduler, + nativeView: nativeView, + paneHost: paneHost, + registry: registry, + runtimeTeardown: runtimeTeardown + ) + surface.agentCommandShimInstallCompleted = true + return (surface, nativeView) + } + + private func saturateRuntimeOwnershipRecovery( + _ coordinator: TerminalSurfaceRuntimeTeardownCoordinator + ) throws -> ( + owners: [TerminalSurfaceRuntimeOwnershipReservation], + recoveryIDs: [UUID] + ) { + let owners = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let recoveryIDs = (0..<2).map { _ in UUID() } + for recoveryID in recoveryIDs { + #expect( + coordinator.reserveRuntimeSurfaceOwnership( + recoveryID: recoveryID, + onRecovery: { reservation in + coordinator.cancelRuntimeSurfaceOwnership(reservation) + } + ) == .deferred + ) + } + return (owners, recoveryIDs) + } + + private func releaseRuntimeOwnershipSaturation( + _ saturation: ( + owners: [TerminalSurfaceRuntimeOwnershipReservation], + recoveryIDs: [UUID] + ), + from coordinator: TerminalSurfaceRuntimeTeardownCoordinator + ) { + for recoveryID in saturation.recoveryIDs { + coordinator.cancelRuntimeSurfaceOwnershipRecovery(recoveryID) + } + for owner in saturation.owners { + coordinator.cancelRuntimeSurfaceOwnership(owner) + } + } } From 512fea4e9a40e5c2d92d3d2fddf8afee9d52111e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:51:44 -0700 Subject: [PATCH 392/521] test(tui): pass hook payload to wheel fixtures --- cmux-tui/dist/scripts/test_package_pypi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/dist/scripts/test_package_pypi.py b/cmux-tui/dist/scripts/test_package_pypi.py index 68b4e4733bc5..eeb762d472a0 100644 --- a/cmux-tui/dist/scripts/test_package_pypi.py +++ b/cmux-tui/dist/scripts/test_package_pypi.py @@ -12,7 +12,7 @@ def launcher_command(version: str, argv0: str) -> str: - files = wheel_bytes(version, "test-platform", b"binary") + files = wheel_bytes(version, "test-platform", b"binary", b"hook-binary") source = next(data for name, data, _ in files if name == "cmux_tui/_main.py") namespace: dict[str, object] = {} exec(compile(source, "cmux_tui/_main.py", "exec"), namespace) @@ -21,7 +21,7 @@ def launcher_command(version: str, argv0: str) -> str: class LauncherCommandTests(unittest.TestCase): def test_launcher_remains_compatible_with_python_3_8(self) -> None: - files = wheel_bytes("1.2.3", "test-platform", b"binary") + files = wheel_bytes("1.2.3", "test-platform", b"binary", b"hook-binary") source = next(data for name, data, _ in files if name == "cmux_tui/_main.py") self.assertNotIn(b".removeprefix(", source) From 26ad7cbedd2764bf69a4b46986d6271cb18845e3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:53:44 -0700 Subject: [PATCH 393/521] fix: retry overflowed surface creation --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 159 +++++++++++++++--- ...urfaceRuntimeOwnershipAdmissionState.swift | 2 + ...OwnershipRecoveryCapacityReservation.swift | 8 + ...timeOwnershipRecoveryRescanScheduler.swift | 115 +++++++++++++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 43 ++++- .../TerminalSurface+RuntimeLifecycle.swift | 85 ++++++++-- .../Surface/TerminalSurface.swift | 1 + 7 files changed, 369 insertions(+), 44 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation.swift create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index e36b87177bb9..f456edb5e523 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -17,17 +17,25 @@ typealias TerminalSurfaceRuntimeOwnershipRecovery = /// from growing an unbounded retained teardown backlog. final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { private let maximumOwnerCount: Int + private let recoveryRescanScheduler: + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler private let state = OSAllocatedUnfairLock( initialState: TerminalSurfaceRuntimeOwnershipAdmissionState() ) - init(maximumOwnerCount: Int) { + init( + maximumOwnerCount: Int, + recoveryRescanScheduler: + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler = + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler() + ) { precondition( maximumOwnerCount >= TerminalSurfaceRuntimeTeardownCoordinator .maximumConcurrentCloseTeardownCount, "native surface ownership must cover every close worker" ) self.maximumOwnerCount = maximumOwnerCount + self.recoveryRescanScheduler = recoveryRescanScheduler } func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { @@ -43,29 +51,75 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func reserve( recoveryID: UUID, - onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery + onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + capacityReservation: + TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? = nil ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { - state.withLock { state in + let output: ( + result: TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult, + requestRescan: Bool + ) = state.withLock { state in + let claimedCapacity: Bool + if let capacityReservation { + guard state.recoveryCapacityReservationIDs.remove( + capacityReservation.id + ) != nil else { + return (.rejected, false) + } + claimedCapacity = true + } else { + claimedCapacity = false + } if !state.closeTeardownDegraded, state.reservationIDs.count < maximumOwnerCount, state.ingressReservationIDs.count < maximumOwnerCount { _ = removeRecoveryAction(recoveryID, from: &state) - return .reserved(reserveOwnershipAndIngress(in: &state)) + return ( + .reserved(reserveOwnershipAndIngress(in: &state)), + takeRecoveryRescanRequest(from: &state) || claimedCapacity + ) } if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery - return .deferred + return (.deferred, claimedCapacity) } - guard state.recoveryEntriesByID.count < maximumOwnerCount else { - return .rejected + guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { + return (.rejected, false) } enqueueRecoveryAction( onRecovery, id: recoveryID, in: &state ) - return .deferred + return (.deferred, false) + } + if output.requestRescan { + recoveryRescanScheduler.requestRescan() + } + return output.result + } + + func claimRecoveryCapacity() + -> TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? { + state.withLock { state in + guard recoveryCapacityIsOpen(in: state) else { return nil } + let reservation = + TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation() + state.recoveryCapacityReservationIDs.insert(reservation.id) + return reservation + } + } + + func releaseRecoveryCapacity( + _ reservation: + TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation + ) { + let released = state.withLock { state in + state.recoveryCapacityReservationIDs.remove(reservation.id) != nil + } + if released { + recoveryRescanScheduler.requestRescan() } } @@ -93,33 +147,45 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { -> TerminalSurfaceRuntimeTeardownIngressReservation? { let result: ( TerminalSurfaceRuntimeTeardownIngressReservation?, - TerminalSurfaceRuntimeOwnershipRecoveryGrant? + TerminalSurfaceRuntimeOwnershipRecoveryGrant?, + Bool ) = state.withLock { state in if let recoveryGrant = takeNextRecoveryGrant(from: &state) { - return (nil, recoveryGrant) + return ( + nil, + recoveryGrant, + takeRecoveryRescanRequest(from: &state) + ) } guard state.ingressReservationIDs.count < maximumOwnerCount else { - return (nil, nil) + return (nil, nil, false) } let reservation = TerminalSurfaceRuntimeTeardownIngressReservation() state.ingressReservationIDs.insert(reservation.id) - return (reservation, nil) + return (reservation, nil, false) } schedule(result.1) + if result.2 { + recoveryRescanScheduler.requestRescan() + } return result.0 } func releaseIngress( _ reservation: TerminalSurfaceRuntimeTeardownIngressReservation ) { - let recoveryGrant = state.withLock { state in + let output = state.withLock { state in _ = state.ingressReservationIDs.remove(reservation.id) _ = state.unclaimedOwnershipIngressReservationIDs.remove( reservation.id ) - return takeNextRecoveryGrant(from: &state) + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() } - schedule(recoveryGrant) } func releaseFailedSubmission( @@ -127,41 +193,57 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ingress ingressReservation: TerminalSurfaceRuntimeTeardownIngressReservation ) { - let recoveryGrant = state.withLock { state in + let output = state.withLock { state in _ = state.reservationIDs.remove(reservation.id) _ = state.ingressReservationIDs.remove(ingressReservation.id) _ = state.unclaimedOwnershipIngressReservationIDs.remove( reservation.id ) - return takeNextRecoveryGrant(from: &state) + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() } - schedule(recoveryGrant) } func release(_ reservation: TerminalSurfaceRuntimeOwnershipReservation) { - let recoveryGrant = state.withLock { state in + let output = state.withLock { state in _ = state.reservationIDs.remove(reservation.id) if state.unclaimedOwnershipIngressReservationIDs.remove( reservation.id ) != nil { _ = state.ingressReservationIDs.remove(reservation.id) } - return takeNextRecoveryGrant(from: &state) + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() } - schedule(recoveryGrant) } func setCloseTeardownDegraded(_ degraded: Bool) { - let recoveryGrant = state.withLock { state in + let output = state.withLock { state in state.closeTeardownDegraded = degraded - return takeNextRecoveryGrant(from: &state) + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() } - schedule(recoveryGrant) } func cancelRecovery(_ recoveryID: UUID) { - state.withLock { state in + let requestRescan = state.withLock { state in _ = removeRecoveryAction(recoveryID, from: &state) + return takeRecoveryRescanRequest(from: &state) + } + if requestRescan { + recoveryRescanScheduler.requestRescan() } } @@ -174,6 +256,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) else { return nil } + state.recoveryRescanRequested = true if let previousID = entry.previousID { state.recoveryEntriesByID[previousID]?.nextID = entry.nextID } else { @@ -207,6 +290,22 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.recoveryTailID = recoveryID } + private func recoveryCapacityIsOpen( + in state: TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> Bool { + state.recoveryEntriesByID.count + + state.recoveryCapacityReservationIDs.count + < maximumOwnerCount + } + + private func takeRecoveryRescanRequest( + from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> Bool { + let requested = state.recoveryRescanRequested + state.recoveryRescanRequested = false + return requested + } + private func reserveOwnershipAndIngress( in state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) -> TerminalSurfaceRuntimeOwnershipReservation { @@ -250,11 +349,15 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } private func recoveryGrantDidComplete() { - let recoveryGrant = state.withLock { state in + let output = state.withLock { state in state.recoveryGrantIsScheduled = false - return takeNextRecoveryGrant(from: &state) + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() } - schedule(recoveryGrant) } #if DEBUG diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index 0b5029123964..0b863b4ccf85 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -9,7 +9,9 @@ struct TerminalSurfaceRuntimeOwnershipAdmissionState { var closeTeardownDegraded = false var recoveryEntriesByID: [UUID: TerminalSurfaceRuntimeOwnershipRecoveryEntry] = [:] + var recoveryCapacityReservationIDs: Set = [] var recoveryHeadID: UUID? var recoveryTailID: UUID? var recoveryGrantIsScheduled = false + var recoveryRescanRequested = false } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation.swift new file mode 100644 index 000000000000..d21c3a1e9bca --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation.swift @@ -0,0 +1,8 @@ +import Foundation + +/// Atomically claims one open recovery-list slot for an overflow rescan. +nonisolated struct TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation: + Sendable +{ + let id = UUID() +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift new file mode 100644 index 000000000000..d879fb9dd89c --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -0,0 +1,115 @@ +internal import CmuxTerminalCore +internal import os + +/// Coalesces MainActor scans for surface-owned overflow recovery intents. +/// +/// The registry stores weak surface entries. This scheduler stores no surface +/// or recovery closure; the strong array exists only for one synchronous scan. +internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: + @unchecked Sendable +{ + private struct State { + var nextSequence: UInt64 = 0 + var registry: (any TerminalSurfaceRegistering)? + var rescanRequested = false + var rescanTask: Task? + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + internal func registerOverflow( + registry: any TerminalSurfaceRegistering + ) -> UInt64 { + var startGate: TerminalSurfaceRuntimeTeardownStartGate? + let sequence = state.withLock { state in + precondition(state.nextSequence < UInt64.max) + state.nextSequence += 1 + if let registeredRegistry = state.registry { + precondition(registeredRegistry === registry) + } else { + state.registry = registry + } + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + return state.nextSequence + } + startGate?.start() + return sequence + } + + internal func requestRescan() { + var startGate: TerminalSurfaceRuntimeTeardownStartGate? + state.withLock { state in + guard state.registry != nil else { return } + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + } + startGate?.start() + } + + deinit { + state.withLock { state in + state.rescanTask?.cancel() + state.rescanTask = nil + } + } + + private func prepareRescanTaskIfNeeded( + state: inout State, + startGate: inout TerminalSurfaceRuntimeTeardownStartGate? + ) { + guard state.rescanTask == nil else { return } + let gate = TerminalSurfaceRuntimeTeardownStartGate() + state.rescanTask = Task { @MainActor [weak self] in + await gate.wait() + self?.drainRequestedRescans() + } + startGate = gate + } + + @MainActor + private func drainRequestedRescans() { + while true { + let registry = state.withLock { state in + state.rescanRequested = false + return state.registry + } + if let registry { + rescan(registry: registry) + } + let shouldContinue = state.withLock { state in + guard state.rescanRequested else { + state.rescanTask = nil + return false + } + return true + } + guard shouldContinue else { return } + } + } + + @MainActor + private func rescan(registry: any TerminalSurfaceRegistering) { + let surfaces = registry.allSurfaces() + .compactMap { $0 as? TerminalSurface } + .compactMap { surface in + surface.runtimeSurfaceAdmissionOverflowSequence.map { + (sequence: $0, surface: surface) + } + } + .sorted { lhs, rhs in lhs.sequence < rhs.sequence } + + for entry in surfaces { + guard + let capacityReservation = + entry.surface.runtimeTeardown + .claimRuntimeSurfaceOwnershipRecoveryCapacity() + else { + return + } + entry.surface.retryRuntimeSurfaceCreationAfterAdmissionOverflow( + capacityReservation: capacityReservation + ) + } + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index a87916e89433..83c8b4152a1e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -30,6 +30,8 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { private nonisolated let submissionDrain: TerminalSurfaceRuntimeTeardownSubmissionDrain + private nonisolated let recoveryRescanScheduler: + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in // TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift can report the @@ -60,26 +62,34 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Creates the process's teardown coordinator. public init() { + let recoveryRescanScheduler = + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler() submissionDrain = TerminalSurfaceRuntimeTeardownSubmissionDrain( maximumBufferedOperationCount: Self.maximumRuntimeSurfaceOwnerCount ) runtimeOwnershipAdmission = TerminalSurfaceRuntimeOwnershipAdmission( - maximumOwnerCount: Self.maximumRuntimeSurfaceOwnerCount + maximumOwnerCount: Self.maximumRuntimeSurfaceOwnerCount, + recoveryRescanScheduler: recoveryRescanScheduler ) + self.recoveryRescanScheduler = recoveryRescanScheduler availableCloseExecutionSlots = Set( 0.. TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { runtimeOwnershipAdmission.reserve( recoveryID: recoveryID, - onRecovery: onRecovery + onRecovery: onRecovery, + capacityReservation: capacityReservation ) } + nonisolated func registerRuntimeSurfaceOwnershipRecoveryOverflow( + registry: any TerminalSurfaceRegistering + ) -> UInt64 { + recoveryRescanScheduler.registerOverflow(registry: registry) + } + + nonisolated func requestRuntimeSurfaceOwnershipRecoveryRescan() { + recoveryRescanScheduler.requestRescan() + } + + nonisolated func claimRuntimeSurfaceOwnershipRecoveryCapacity() + -> TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? { + runtimeOwnershipAdmission.claimRecoveryCapacity() + } + + nonisolated func cancelRuntimeSurfaceOwnershipRecoveryCapacity( + _ reservation: + TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation + ) { + runtimeOwnershipAdmission.releaseRecoveryCapacity(reservation) + } + nonisolated func cancelRuntimeSurfaceOwnershipRecovery(_ recoveryID: UUID) { runtimeOwnershipAdmission.cancelRecovery(recoveryID) } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index fa8f826de08b..7cee10f088e6 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -744,6 +744,23 @@ extension TerminalSurface { createSurface(for: view, source: source) } + @MainActor + private func runtimeSurfaceOwnershipRecovery( + teardownCoordinator: TerminalSurfaceRuntimeTeardownCoordinator + ) -> TerminalSurfaceRuntimeOwnershipRecovery { + { [weak self, teardownCoordinator] reservation in + guard let self else { + teardownCoordinator.cancelRuntimeSurfaceOwnership( + reservation + ) + return + } + self.resumeRuntimeSurfaceCreationAfterAdmissionRecovery( + reservation: reservation + ) + } + } + @MainActor private func reserveRuntimeSurfaceOwnershipForCreation( view: any TerminalSurfaceNativeViewing, @@ -759,29 +776,72 @@ extension TerminalSurface { let admissionResult = teardownCoordinator.reserveRuntimeSurfaceOwnership( recoveryID: id, - onRecovery: { [weak self, teardownCoordinator] reservation in - guard let self else { - teardownCoordinator.cancelRuntimeSurfaceOwnership( - reservation - ) - return - } - self.resumeRuntimeSurfaceCreationAfterAdmissionRecovery( - reservation: reservation - ) - } + onRecovery: runtimeSurfaceOwnershipRecovery( + teardownCoordinator: teardownCoordinator + ) ) switch admissionResult { case .reserved(let reservation): + runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil return reservation case .deferred: + runtimeSurfaceAdmissionOverflowSequence = nil return nil case .rejected: + if runtimeSurfaceAdmissionOverflowSequence == nil { + runtimeSurfaceAdmissionOverflowSequence = + teardownCoordinator + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + registry: registry + ) + } else { + teardownCoordinator + .requestRuntimeSurfaceOwnershipRecoveryRescan() + } + return nil + } + } + + @MainActor + func retryRuntimeSurfaceCreationAfterAdmissionOverflow( + capacityReservation: + TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation + ) { + guard runtimeSurfaceAdmissionOverflowSequence != nil, + allowsRuntimeSurfaceCreation(), + surface == nil, + runtimeSurfaceAdmissionDeferredCreationSource != nil else { + runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil - return nil + runtimeTeardown + .cancelRuntimeSurfaceOwnershipRecoveryCapacity( + capacityReservation + ) + return + } + let teardownCoordinator = runtimeTeardown + let admissionResult = + teardownCoordinator.reserveRuntimeSurfaceOwnership( + recoveryID: id, + onRecovery: runtimeSurfaceOwnershipRecovery( + teardownCoordinator: teardownCoordinator + ), + capacityReservation: capacityReservation + ) + switch admissionResult { + case .reserved(let reservation): + runtimeSurfaceAdmissionOverflowSequence = nil + resumeRuntimeSurfaceCreationAfterAdmissionRecovery( + reservation: reservation + ) + case .deferred: + runtimeSurfaceAdmissionOverflowSequence = nil + case .rejected: + teardownCoordinator + .requestRuntimeSurfaceOwnershipRecoveryRescan() } } @@ -815,6 +875,7 @@ extension TerminalSurface { private func cancelRuntimeSurfaceCreationAfterAdmissionRecovery() { runtimeTeardown.cancelRuntimeSurfaceOwnershipRecovery(id) + runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index cf2995ae2282..84144b22f572 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -290,6 +290,7 @@ public final class TerminalSurface: Identifiable, ObservableObject { RuntimeSurfaceCreationSource? weak var runtimeSurfaceAdmissionDeferredCreationView: (any TerminalSurfaceNativeViewing)? + var runtimeSurfaceAdmissionOverflowSequence: UInt64? var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false /// Bounded ownership of the live native surface. Transferred to the From b53184d8e656583c644cc49ac759ee300a9fb134 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:54:52 -0700 Subject: [PATCH 394/521] fix(tui): reconcile merged shutdown contracts --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 6 ------ cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs | 12 ++++++------ .../cmux-tui-core/src/mux/resource_topology.rs | 2 +- cmux-tui/crates/cmux-tui-core/src/server.rs | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 714b36adfc54..d8128c8da744 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -895,11 +895,6 @@ impl BrowserRuntime { anyhow::bail!("CDP browser connection is closed"); } let (target_id, normalized_url) = match bootstrap { - BrowserBootstrap::Create { url } => { - let normalized_url = normalize_url(&url); - let target_id = self.client.create_target(&normalized_url)?; - (target_id, normalized_url) - } BrowserBootstrap::ExistingTarget { target_id, url } => (target_id, normalize_url(&url)), BrowserBootstrap::Provider { .. } => { anyhow::bail!("browser provider target was not resolved before CDP bootstrap") @@ -1175,7 +1170,6 @@ impl BrowserRuntime { } pub(crate) enum BrowserBootstrap { - Create { url: String }, ExistingTarget { target_id: String, url: String }, Provider { tab_id: crate::resource::TabPublicId, url: String }, } diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs index 2d02d28ff010..6fd0dd10f60a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_ingress.rs @@ -1278,7 +1278,7 @@ mod tests { let shutdown_mux = mux.clone(); let (completed, completion) = sync_channel(1); let shutdown = std::thread::spawn(move || { - shutdown_mux.shutdown(); + shutdown_mux.shutdown().unwrap(); completed.send(()).unwrap(); }); assert!( @@ -1331,7 +1331,7 @@ mod tests { ); let started = Instant::now(); - mux.shutdown(); + mux.shutdown().unwrap(); assert!( started.elapsed() < JOURNAL_DURABLE_WAIT + Duration::from_secs(2), @@ -1719,7 +1719,7 @@ mod tests { let shutdown_mux = mux.clone(); let (shutdown_completed, shutdown_completion) = sync_channel(1); let shutdown = std::thread::spawn(move || { - shutdown_mux.shutdown(); + shutdown_mux.shutdown().unwrap(); shutdown_completed.send(()).unwrap(); }); let returned = shutdown_completion.recv_timeout(Duration::from_secs(5)); @@ -1773,7 +1773,7 @@ mod tests { let shutdown_mux = mux.clone(); let (completed, completion) = sync_channel(1); let shutdown = std::thread::spawn(move || { - shutdown_mux.shutdown(); + shutdown_mux.shutdown().unwrap(); completed.send(()).unwrap(); }); assert!( @@ -1891,7 +1891,7 @@ mod tests { } let started = Instant::now(); - mux.shutdown(); + mux.shutdown().unwrap(); assert!( started.elapsed() < Duration::from_secs(3), @@ -1975,7 +1975,7 @@ mod tests { .unwrap(); let started = Instant::now(); - mux.shutdown(); + mux.shutdown().unwrap(); assert!( started.elapsed() < Duration::from_secs(3), "shutdown waited without a bound for a descendant-held PTY" diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 8070f7f32349..7c25f5c3cc58 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -6185,7 +6185,7 @@ mod creation_recovery_tests { ) .unwrap_err(); assert_eq!(error.to_string(), "resource revision conflict: expected 0, current 1"); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index b8b8ecaee1ef..ef1fd7f8c1f2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -19763,7 +19763,7 @@ mod tests { 1 ); mux.close_surface(surface).unwrap(); - mux.shutdown(); + mux.shutdown().unwrap(); } #[test] From ad3f370987510adc1eefc82f6677a7ad177c7ea5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:12:33 -0700 Subject: [PATCH 395/521] test(macOS): bound overflow recovery scans --- .../FakeSurfaceRegistry.swift | 9 ++ ...nalSurfaceRestoreSpawnSchedulerTests.swift | 99 ++++++++++++++++++- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift index aa8ad0c33ed2..5d2758984ac9 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift @@ -1,10 +1,18 @@ import Foundation import GhosttyKit import CmuxTerminalCore +import os @testable import CmuxTerminal final class FakeSurfaceRegistry: @unchecked Sendable, TerminalSurfaceRegistering { private let backing = TerminalSurfaceRegistry() + private let allSurfacesCallCountLock = OSAllocatedUnfairLock( + initialState: 0 + ) + + var allSurfacesCallCount: Int { + allSurfacesCallCountLock.withLock { $0 } + } var topologyGeneration: UInt64 { backing.topologyGeneration } func register( @@ -72,6 +80,7 @@ final class FakeSurfaceRegistry: @unchecked Sendable, TerminalSurfaceRegistering backing.updateFocusPlacement(id: id, placement) } func allSurfaces() -> [any TerminalSurfacing] { + allSurfacesCallCountLock.withLock { $0 += 1 } backing.allSurfaces() } } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 3fba4fb8fcd7..4034935932d9 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -401,6 +401,56 @@ import CmuxTerminalCore fixtures[1].surface.id, ] ) + #expect( + registry.allSurfacesCallCount == 0, + "overflow recovery must not scan the full surface registry" + ) + } + + @Test func overflowRecoveryContinuesInFixedMainActorBatches() async throws { + let batchCount = 32 + let overflowCount = batchCount + 1 + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: overflowCount + ) + let saturation = try saturateRuntimeOwnershipRecovery( + coordinator, + count: overflowCount + ) + defer { + releaseRuntimeOwnershipSaturation(saturation, from: coordinator) + } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let fixtures = (0.. ( owners: [TerminalSurfaceRuntimeOwnershipReservation], recoveryIDs: [UUID] ) { - let owners = try (0..<2).map { _ in + let owners = try (0.. Date: Mon, 10 Aug 2026 13:17:22 -0700 Subject: [PATCH 396/521] fix(macOS): bound overflow recovery work --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 4 + ...timeOwnershipRecoveryRescanScheduler.swift | 195 +++++++++++++----- ...nalSurfaceRuntimeTeardownCoordinator.swift | 18 +- .../TerminalSurface+RuntimeLifecycle.swift | 26 ++- 4 files changed, 185 insertions(+), 58 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index f456edb5e523..7d0e05754e56 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -111,6 +111,10 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } } + func recoveryCapacityIsOpen() -> Bool { + state.withLock { recoveryCapacityIsOpen(in: $0) } + } + func releaseRecoveryCapacity( _ reservation: TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index d879fb9dd89c..f584f3f1285d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -1,16 +1,40 @@ -internal import CmuxTerminalCore internal import os -/// Coalesces MainActor scans for surface-owned overflow recovery intents. +/// Coalesces bounded MainActor batches for surface-owned overflow recovery. /// -/// The registry stores weak surface entries. This scheduler stores no surface -/// or recovery closure; the strong array exists only for one synchronous scan. +/// The FIFO stores one weak surface per public surface id. It does not retain +/// native-creation closures or surfaces, and each task processes a fixed batch +/// before yielding the main actor. internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: @unchecked Sendable { + private static let maximumBatchCount = 32 + + private final class OverflowEntry { + let surfaceID: UUID + let sequence: UInt64 + weak var surface: TerminalSurface? + var previousID: UUID? + var nextID: UUID? + + init( + surfaceID: UUID, + sequence: UInt64, + surface: TerminalSurface, + previousID: UUID? + ) { + self.surfaceID = surfaceID + self.sequence = sequence + self.surface = surface + self.previousID = previousID + } + } + private struct State { var nextSequence: UInt64 = 0 - var registry: (any TerminalSurfaceRegistering)? + var entriesByID: [UUID: OverflowEntry] = [:] + var headID: UUID? + var tailID: UUID? var rescanRequested = false var rescanTask: Task? } @@ -18,29 +42,55 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: private let state = OSAllocatedUnfairLock(initialState: State()) internal func registerOverflow( - registry: any TerminalSurfaceRegistering + surfaceID: UUID, + surface: TerminalSurface ) -> UInt64 { var startGate: TerminalSurfaceRuntimeTeardownStartGate? let sequence = state.withLock { state in + if let entry = state.entriesByID[surfaceID] { + entry.surface = surface + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + return entry.sequence + } + precondition(state.nextSequence < UInt64.max) state.nextSequence += 1 - if let registeredRegistry = state.registry { - precondition(registeredRegistry === registry) + let previousID = state.tailID + let entry = OverflowEntry( + surfaceID: surfaceID, + sequence: state.nextSequence, + surface: surface, + previousID: previousID + ) + state.entriesByID[surfaceID] = entry + if let previousID { + state.entriesByID[previousID]?.nextID = surfaceID } else { - state.registry = registry + state.headID = surfaceID } + state.tailID = surfaceID state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) - return state.nextSequence + return entry.sequence } startGate?.start() return sequence } + internal func cancelOverflow(surfaceID: UUID) { + let removed = state.withLock { state in + removeOverflow(surfaceID: surfaceID, from: &state) != nil + } + if removed { + requestRescan() + } + } + internal func requestRescan() { var startGate: TerminalSurfaceRuntimeTeardownStartGate? state.withLock { state in - guard state.registry != nil else { return } + guard state.headID != nil else { return } state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) } @@ -62,54 +112,105 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: let gate = TerminalSurfaceRuntimeTeardownStartGate() state.rescanTask = Task { @MainActor [weak self] in await gate.wait() - self?.drainRequestedRescans() + self?.runScheduledBatch() } startGate = gate } @MainActor - private func drainRequestedRescans() { - while true { - let registry = state.withLock { state in - state.rescanRequested = false - return state.registry - } - if let registry { - rescan(registry: registry) - } - let shouldContinue = state.withLock { state in - guard state.rescanRequested else { - state.rescanTask = nil - return false - } - return true - } - guard shouldContinue else { return } + private func runScheduledBatch() { + let shouldRun = state.withLock { state in + state.rescanRequested = false + return state.headID != nil } - } + var processedCount = 0 + if shouldRun { + while processedCount < Self.maximumBatchCount { + guard + let entry = state.withLock({ state in + state.headID.flatMap { state.entriesByID[$0] } + }) + else { + break + } + guard let surface = entry.surface else { + state.withLock { state in + _ = removeOverflow( + surfaceID: entry.surfaceID, + from: &state + ) + } + processedCount += 1 + continue + } + guard + let capacityReservation = + surface.runtimeTeardown + .claimRuntimeSurfaceOwnershipRecoveryCapacity() + else { + break + } + surface.retryRuntimeSurfaceCreationAfterAdmissionOverflow( + capacityReservation: capacityReservation + ) + processedCount += 1 - @MainActor - private func rescan(registry: any TerminalSurfaceRegistering) { - let surfaces = registry.allSurfaces() - .compactMap { $0 as? TerminalSurface } - .compactMap { surface in - surface.runtimeSurfaceAdmissionOverflowSequence.map { - (sequence: $0, surface: surface) + let madeProgress = state.withLock { state in + state.entriesByID[entry.surfaceID] == nil } + guard madeProgress else { break } } - .sorted { lhs, rhs in lhs.sequence < rhs.sequence } + } - for entry in surfaces { - guard - let capacityReservation = - entry.surface.runtimeTeardown - .claimRuntimeSurfaceOwnershipRecoveryCapacity() + let followUp = state.withLock { state -> (OverflowEntry, Bool)? in + state.rescanTask = nil + let externallyRequested = state.rescanRequested + state.rescanRequested = false + guard let headID = state.headID, + let entry = state.entriesByID[headID] else { - return + return nil } - entry.surface.retryRuntimeSurfaceCreationAfterAdmissionOverflow( - capacityReservation: capacityReservation - ) + return (entry, externallyRequested) + } + guard let (entry, externallyRequested) = followUp else { return } + let batchWasExhausted = processedCount == Self.maximumBatchCount + let capacityRemains: Bool + if let surface = entry.surface { + capacityRemains = + surface.runtimeTeardown + .runtimeSurfaceOwnershipRecoveryCapacityIsOpen() + } else { + capacityRemains = true + } + if capacityRemains && (batchWasExhausted || externallyRequested) { + requestRescan() + } + } + + @discardableResult + private func removeOverflow( + surfaceID: UUID, + from state: inout State + ) -> OverflowEntry? { + guard let entry = state.entriesByID.removeValue(forKey: surfaceID) else { + return nil + } + if let previousID = entry.previousID { + state.entriesByID[previousID]?.nextID = entry.nextID + } else { + state.headID = entry.nextID + } + if let nextID = entry.nextID { + state.entriesByID[nextID]?.previousID = entry.previousID + } else { + state.tailID = entry.previousID + } + if state.entriesByID.isEmpty { + state.headID = nil + state.tailID = nil + state.rescanRequested = false } + return entry } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 83c8b4152a1e..479a0d38f550 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -123,9 +123,19 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { } nonisolated func registerRuntimeSurfaceOwnershipRecoveryOverflow( - registry: any TerminalSurfaceRegistering + surfaceID: UUID, + surface: TerminalSurface ) -> UInt64 { - recoveryRescanScheduler.registerOverflow(registry: registry) + recoveryRescanScheduler.registerOverflow( + surfaceID: surfaceID, + surface: surface + ) + } + + nonisolated func cancelRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: UUID + ) { + recoveryRescanScheduler.cancelOverflow(surfaceID: surfaceID) } nonisolated func requestRuntimeSurfaceOwnershipRecoveryRescan() { @@ -137,6 +147,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { runtimeOwnershipAdmission.claimRecoveryCapacity() } + nonisolated func runtimeSurfaceOwnershipRecoveryCapacityIsOpen() -> Bool { + runtimeOwnershipAdmission.recoveryCapacityIsOpen() + } + nonisolated func cancelRuntimeSurfaceOwnershipRecoveryCapacity( _ reservation: TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 7cee10f088e6..7ba214756f07 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -782,24 +782,24 @@ extension TerminalSurface { ) switch admissionResult { case .reserved(let reservation): + teardownCoordinator + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil return reservation case .deferred: + teardownCoordinator + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil return nil case .rejected: - if runtimeSurfaceAdmissionOverflowSequence == nil { - runtimeSurfaceAdmissionOverflowSequence = - teardownCoordinator - .registerRuntimeSurfaceOwnershipRecoveryOverflow( - registry: registry - ) - } else { + runtimeSurfaceAdmissionOverflowSequence = teardownCoordinator - .requestRuntimeSurfaceOwnershipRecoveryRescan() - } + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: id, + surface: self + ) return nil } } @@ -813,6 +813,8 @@ extension TerminalSurface { allowsRuntimeSurfaceCreation(), surface == nil, runtimeSurfaceAdmissionDeferredCreationSource != nil else { + runtimeTeardown + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil @@ -833,11 +835,15 @@ extension TerminalSurface { ) switch admissionResult { case .reserved(let reservation): + teardownCoordinator + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil resumeRuntimeSurfaceCreationAfterAdmissionRecovery( reservation: reservation ) case .deferred: + teardownCoordinator + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil case .rejected: teardownCoordinator @@ -875,6 +881,8 @@ extension TerminalSurface { private func cancelRuntimeSurfaceCreationAfterAdmissionRecovery() { runtimeTeardown.cancelRuntimeSurfaceOwnershipRecovery(id) + runtimeTeardown + .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil From d908b30a9013a4cdf4adf6256e8c8b95d287679c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:30:44 -0700 Subject: [PATCH 397/521] fix(tui): satisfy hosted clippy cleanup checks --- cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs | 4 ++-- cmux-tui/crates/cmux-tui/src/cli/wire.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 108357acb702..486f1fbcd3db 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -1782,7 +1782,7 @@ fn provider_disconnect_reconnects_without_closing_canonical_browser_topology() { mux.close_surface(surface.id).unwrap(); drop(provider); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); first_server.join().unwrap(); second_server.join().unwrap(); @@ -1903,7 +1903,7 @@ fn provider_target_revision_reattaches_on_the_same_browser_connection() { mux.close_surface(surface.id).unwrap(); assert_eq!(detached_rx.recv_timeout(Duration::from_secs(10)).unwrap(), "session-second"); drop(provider); - mux.shutdown(); + mux.shutdown().unwrap(); server::cleanup(&socket_path); cdp_server.join().unwrap(); } diff --git a/cmux-tui/crates/cmux-tui/src/cli/wire.rs b/cmux-tui/crates/cmux-tui/src/cli/wire.rs index 79829e45f669..8e37b18e928e 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/wire.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/wire.rs @@ -1,7 +1,6 @@ use std::io::{self, BufRead, BufReader, Read, Write}; #[cfg(unix)] use std::net::Shutdown; -use std::path::PathBuf; use std::time::Duration; use cmux_tui_core::platform::transport; From 3d191bb1262d6595f4ed5b1c859967189a8e44af Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:49:33 -0700 Subject: [PATCH 398/521] test(macOS): cover overflow recovery cancellation --- ...timeOwnershipRecoveryRescanScheduler.swift | 27 +++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 11 + ...nalSurfaceRestoreSpawnSchedulerTests.swift | 209 ++++++++++++++++++ 3 files changed, 247 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index f584f3f1285d..98881c20fb9d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -32,6 +32,8 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: private struct State { var nextSequence: UInt64 = 0 + // Every indexed entry is one linked FIFO node. Synchronous cancellation + // removes both views under this lock, so no weak tombstone survives. var entriesByID: [UUID: OverflowEntry] = [:] var headID: UUID? var tailID: UUID? @@ -41,6 +43,31 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: private let state = OSAllocatedUnfairLock(initialState: State()) + #if DEBUG + internal var debugSnapshot: + ( + entryCount: Int, + linkedNodeCount: Int, + headID: UUID?, + tailID: UUID? + ) + { + state.withLock { state in + var linkedIDs = Set() + var currentID = state.headID + while let nodeID = currentID, linkedIDs.insert(nodeID).inserted { + currentID = state.entriesByID[nodeID]?.nextID + } + return ( + entryCount: state.entriesByID.count, + linkedNodeCount: linkedIDs.count, + headID: state.headID, + tailID: state.tailID + ) + } + } + #endif + internal func registerOverflow( surfaceID: UUID, surface: TerminalSurface diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 479a0d38f550..59b10d7ca6f7 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -142,6 +142,17 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { recoveryRescanScheduler.requestRescan() } +#if DEBUG + nonisolated var debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot: ( + entryCount: Int, + linkedNodeCount: Int, + headID: UUID?, + tailID: UUID? + ) { + recoveryRescanScheduler.debugSnapshot + } +#endif + nonisolated func claimRuntimeSurfaceOwnershipRecoveryCapacity() -> TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? { runtimeOwnershipAdmission.claimRecoveryCapacity() diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 4034935932d9..74a524bf36f3 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -453,6 +453,204 @@ import CmuxTerminalCore #expect(registry.allSurfacesCallCount == 0) } + @Test func overflowCancellationChurnEmptiesIndexAndLinkedOrder() { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 1 + ) + let fixture = makeSurfaceFixture( + registry: FakeSurfaceRegistry(), + scheduler: RecordingRestoreSpawnScheduler(), + runtimeTeardown: coordinator + ) + var previousSequence: UInt64 = 0 + + for _ in 0..<(32 * 2 + 1) { + let surfaceID = UUID() + let sequence = coordinator + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: surfaceID, + surface: fixture.surface + ) + #expect(sequence > previousSequence) + previousSequence = sequence + coordinator.cancelRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: surfaceID + ) + } + + let emptySnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(emptySnapshot.entryCount == 0) + #expect(emptySnapshot.linkedNodeCount == 0) + #expect(emptySnapshot.headID == nil) + #expect(emptySnapshot.tailID == nil) + + let repeatedID = UUID() + let firstSequence = coordinator + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: repeatedID, + surface: fixture.surface + ) + let updatedSequence = coordinator + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: repeatedID, + surface: fixture.surface + ) + let updatedSnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(updatedSequence == firstSequence) + #expect(updatedSnapshot.entryCount == 1) + #expect(updatedSnapshot.linkedNodeCount == 1) + #expect(updatedSnapshot.headID == repeatedID) + #expect(updatedSnapshot.tailID == repeatedID) + + coordinator.cancelRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: repeatedID + ) + let replacementSequence = coordinator + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: repeatedID, + surface: fixture.surface + ) + #expect(replacementSequence > firstSequence) + coordinator.cancelRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: repeatedID + ) + + let finalSnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(finalSnapshot.entryCount == 0) + #expect(finalSnapshot.linkedNodeCount == 0) + #expect(finalSnapshot.headID == nil) + #expect(finalSnapshot.tailID == nil) + } + + @Test func lifecycleCancellationSynchronouslyRemovesOverflowEntries() throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 1 + ) + let saturation = try saturateRuntimeOwnershipRecovery( + coordinator, + count: 1 + ) + defer { + releaseRuntimeOwnershipSaturation(saturation, from: coordinator) + } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + + let closing = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + closing.surface.createSurface(for: closing.nativeView) + scheduler.runScheduledOperation(at: 0) + #expect( + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + .entryCount == 1 + ) + closing.surface.beginPortalCloseLifecycle(reason: "test.overflowClose") + assertOverflowStorageIsEmpty(coordinator) + + let hibernating = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + hibernating.surface.createSurface(for: hibernating.nativeView) + scheduler.runScheduledOperation(at: 1) + #expect( + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + .entryCount == 1 + ) + #expect( + hibernating.surface.suspendRuntimeSurfaceForAgentHibernation( + reason: "test.overflowHibernate" + ) + ) + assertOverflowStorageIsEmpty(coordinator) + + let deinitView = FakeTerminalSurfaceNativeView( + frame: NSRect(x: 0, y: 0, width: 800, height: 600) + ) + var deinitializing: TerminalSurface? = makeSurface( + scheduler: scheduler, + nativeView: deinitView, + paneHost: FakeTerminalSurfacePaneHost(surfaceView: deinitView), + registry: registry, + runtimeTeardown: coordinator + ) + deinitializing?.agentCommandShimInstallCompleted = true + deinitializing?.createSurface(for: deinitView) + scheduler.runScheduledOperation(at: 2) + #expect( + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + .entryCount == 1 + ) + weak var weakDeinitializing = deinitializing + deinitializing = nil + #expect(weakDeinitializing == nil) + assertOverflowStorageIsEmpty(coordinator) + } + + @Test func overflowCancelAfterCapacitySignalLeavesOnlyLiveFIFOHead() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 1 + ) + let saturation = try saturateRuntimeOwnershipRecovery( + coordinator, + count: 1 + ) + defer { + releaseRuntimeOwnershipSaturation(saturation, from: coordinator) + } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let cancelled = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + let sentinel = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + + for (index, fixture) in [cancelled, sentinel].enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + } + let queuedSnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(queuedSnapshot.entryCount == 2) + #expect(queuedSnapshot.linkedNodeCount == 2) + #expect(queuedSnapshot.headID == cancelled.surface.id) + #expect(queuedSnapshot.tailID == sentinel.surface.id) + + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + cancelled.surface.beginPortalCloseLifecycle( + reason: "test.cancelAfterCapacitySignal" + ) + let cancelledSnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(cancelledSnapshot.entryCount == 1) + #expect(cancelledSnapshot.linkedNodeCount == 1) + #expect(cancelledSnapshot.headID == sentinel.surface.id) + #expect(cancelledSnapshot.tailID == sentinel.surface.id) + + await scheduler.waitForScheduledCount(3) + #expect( + scheduler.scheduledSurfaceIds == [ + cancelled.surface.id, + sentinel.surface.id, + sentinel.surface.id, + ] + ) + assertOverflowStorageIsEmpty(coordinator) + } + @Test func overflowStoreSurvivesImmediateRecoveryCapacityRelease() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 @@ -942,4 +1140,15 @@ import CmuxTerminalCore coordinator.cancelRuntimeSurfaceOwnership(owner) } } + + private func assertOverflowStorageIsEmpty( + _ coordinator: TerminalSurfaceRuntimeTeardownCoordinator + ) { + let snapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(snapshot.entryCount == 0) + #expect(snapshot.linkedNodeCount == 0) + #expect(snapshot.headID == nil) + #expect(snapshot.tailID == nil) + } } From d8004c08fe1d5dfea27a1533e8ad99c69d6e74b7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:21:11 -0700 Subject: [PATCH 399/521] test(tui): bound browser transport shutdown --- cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs index 486f1fbcd3db..814ef368d6fd 100644 --- a/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/tests/browser_runtime.rs @@ -1458,6 +1458,7 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let (seen_tx, seen_rx) = mpsc::channel(); + let (transport_closed_tx, transport_closed_rx) = mpsc::sync_channel(1); let server = thread::spawn(move || { let (stream, _) = listener.accept().unwrap(); @@ -1509,6 +1510,7 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { method => panic!("unexpected CDP method {method}"), } } + transport_closed_tx.send(()).unwrap(); }); let opts = SurfaceOptions { browser_max_capture_megapixels: 0.01, ..Default::default() }; @@ -1525,6 +1527,7 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { let surface = mux .new_browser_tab("example.test".to_string(), None, Some((100, 100))) .expect("browser tab"); + let retained_surface_id = surface.id; let _provider = browser_provider( &socket_path, format!("ws://{addr}/devtools/browser/fake"), @@ -1571,6 +1574,10 @@ fn browser_capture_scale_applies_to_metrics_screencast_and_input() { mux.shutdown().unwrap(); server::cleanup(&socket_path); + transport_closed_rx + .recv_timeout(test_duration(Duration::from_secs(10))) + .expect("CDP transport remained open after browser runtime shutdown"); + assert_eq!(surface.id, retained_surface_id); server.join().unwrap(); } From 357114efc1dbe23727ec47946051b2a6a5ef62a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:23:17 -0700 Subject: [PATCH 400/521] fix(tui): close browser transport during shutdown --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 85 ++++++++++++++++++-- cmux-tui/crates/cmux-tui-core/src/browser.rs | 3 +- 2 files changed, 81 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index 72396d74f7ce..e32d50bf98a5 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -5,7 +5,8 @@ use std::net::{IpAddr, SocketAddr, TcpStream, ToSocketAddrs}; use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{ - Receiver, Sender, SyncSender, TryRecvError, TrySendError, channel, sync_channel, + Receiver, RecvTimeoutError, Sender, SyncSender, TryRecvError, TrySendError, channel, + sync_channel, }; use std::sync::{Arc, Condvar, Mutex, Weak}; use std::time::{Duration, Instant}; @@ -851,6 +852,7 @@ pub struct CdpClient { struct Inner { outbound: Sender, + transport_close: Mutex, pending: Mutex>, events: Arc, frame_epochs: Mutex>, @@ -1110,6 +1112,13 @@ impl Drop for Inner { enum Outbound { Message(String), Flush(Sender<()>), + Close(SyncSender<()>), +} + +enum TransportClose { + Open, + Closing(Receiver<()>), + Closed, } struct DeadlineTcpStream { @@ -1342,6 +1351,7 @@ impl CdpClient { let client = CdpClient { inner: Arc::new(Inner { outbound: outbound_tx, + transport_close: Mutex::new(TransportClose::Open), pending: Mutex::new(HashMap::new()), events: event_queue, frame_epochs: Mutex::new(HashMap::new()), @@ -1613,14 +1623,54 @@ impl CdpClient { if self.inner.closed.load(Ordering::Acquire) { anyhow::bail!("CDP connection is closed"); } + let close = self.inner.transport_close.lock().unwrap(); + if !matches!(*close, TransportClose::Open) { + anyhow::bail!("CDP connection is closing"); + } let (tx, rx) = channel(); self.inner .outbound .send(Outbound::Flush(tx)) .map_err(|_| anyhow::anyhow!("CDP connection is closed"))?; + drop(close); rx.recv_timeout(timeout).map_err(|_| anyhow::anyhow!("timed out flushing CDP commands")) } + /// Close the WebSocket transport once and retain its completion receipt + /// across deadline expiry so a retry cannot publish a second close owner. + pub fn close_until(&self, deadline: Instant) -> bool { + let mut close = self.inner.transport_close.lock().unwrap(); + loop { + match &mut *close { + TransportClose::Open => { + if self.inner.closed.load(Ordering::Acquire) { + *close = TransportClose::Closed; + return true; + } + let (receipt, receiver) = sync_channel(1); + if self.inner.outbound.send(Outbound::Close(receipt)).is_err() { + *close = TransportClose::Closed; + return true; + } + *close = TransportClose::Closing(receiver); + } + TransportClose::Closing(receiver) => { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + match receiver.recv_timeout(remaining) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => { + *close = TransportClose::Closed; + return true; + } + Err(RecvTimeoutError::Timeout) => return false, + } + } + TransportClose::Closed => return true, + } + } + } + pub fn page_enable(&self, session_id: &str) -> anyhow::Result<()> { self.call("Page.enable", json!({}), Some(session_id)).map(|_| ()) } @@ -2207,6 +2257,10 @@ impl CdpClient { if self.inner.closed.load(Ordering::Acquire) { anyhow::bail!("CDP connection is closed"); } + let close = self.inner.transport_close.lock().unwrap(); + if !matches!(*close, TransportClose::Open) { + anyhow::bail!("CDP connection is closing"); + } let text = serde_json::to_string(value)?; if cdp_debug() { eprintln!("cdp-> {text}"); @@ -2215,6 +2269,7 @@ impl CdpClient { .outbound .send(Outbound::Message(text)) .map_err(|_| anyhow::anyhow!("CDP connection is closed"))?; + drop(close); Ok(()) } } @@ -2258,9 +2313,19 @@ fn reader_loop( if inner.closed.load(Ordering::Acquire) { break; } - if let Err(err) = drain_outbound(&mut ws, outbound) { - close_inner(&inner, &format!("CDP socket error: {err}")); - break; + match drain_outbound(&mut ws, outbound) { + Ok(OutboundDrain::Continue) => {} + Ok(OutboundDrain::Close(receipt)) => { + let _ = ws.close(None); + let _ = ws.flush(); + close_inner(&inner, "CDP client shut down"); + let _ = receipt.send(()); + break; + } + Err(err) => { + close_inner(&inner, &format!("CDP socket error: {err}")); + break; + } } let message = ws.read(); match message { @@ -2296,18 +2361,26 @@ fn reader_loop( fn drain_outbound( ws: &mut WebSocket, outbound: &Receiver, -) -> anyhow::Result<()> { +) -> anyhow::Result { loop { match outbound.try_recv() { Ok(Outbound::Message(text)) => ws.send(Message::Text(text.into()))?, Ok(Outbound::Flush(done)) => { let _ = done.send(()); } - Err(TryRecvError::Empty | TryRecvError::Disconnected) => return Ok(()), + Ok(Outbound::Close(receipt)) => return Ok(OutboundDrain::Close(receipt)), + Err(TryRecvError::Empty | TryRecvError::Disconnected) => { + return Ok(OutboundDrain::Continue); + } } } } +enum OutboundDrain { + Continue, + Close(SyncSender<()>), +} + fn handle_text(inner: &Arc, text: &str) { if cdp_debug() { eprintln!("cdp<- {}", &text[..text.len().min(300)]); diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index d8128c8da744..7b8cf40141bc 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -1164,8 +1164,9 @@ impl BrowserRuntime { true }; close_browser_runtime(self, "browser runtime shut down".to_string()); + let transport_closed = self.client.close_until(deadline); let browser_stopped = self.chrome.as_ref().is_none_or(|chrome| chrome.kill_until(deadline)); - outbound_flushed && browser_stopped + outbound_flushed && transport_closed && browser_stopped } } From 9ae8289c93d2d803ada31ef0504c17a3188ed062 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:24:36 -0700 Subject: [PATCH 401/521] test(tui): wait for fixture-owned readiness --- cmux-tui/crates/cmux-tui/src/app.rs | 6 +++++ cmux-tui/crates/cmux-tui/tests/cli.rs | 34 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index 6f921ae2fa40..ca33e88c6689 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -7796,6 +7796,12 @@ fn run_with_machine_updates_inner( ensure_initial_for_machine_ui(&session, initial_size, machine_ui.as_ref()), machine_ui.as_ref(), )?; + #[cfg(debug_assertions)] + if initial_workspace_error.is_none() + && let Some(signal) = std::env::var_os("CMUX_TUI_TEST_INITIAL_WORKSPACE_READY_SIGNAL") + { + std::fs::write(signal, b"1")?; + } let encoder = KeyEncoder::new()?; let (tx, rx) = sync_channel::(APP_EVENT_CAPACITY); let host_input = HostInputRuntime::new(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index b011727e5d38..2180efad6cea 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -2546,14 +2546,14 @@ exit 0 #[test] fn plain_launch_attaches_to_existing_local_session() { let server = HeadlessServer::start("plain-launch-attach"); - let registered_path = server.dir.join("tui-client-registered"); - let mut registered = TestFifoSignal::create(®istered_path); + let ready_path = server.dir.join("tui-initial-workspace-ready"); + let mut ready = TestFifoSignal::create(&ready_path); let mut tui = PtyChild::start_with_env( &["--socket", server.socket.to_str().unwrap()], - &[("CMUX_TUI_TEST_CLIENT_REGISTERED_SIGNAL", registered_path.as_os_str())], + &[("CMUX_TUI_TEST_INITIAL_WORKSPACE_READY_SIGNAL", ready_path.as_os_str())], ); - registered.receive(Duration::from_secs(10)); + ready.receive(Duration::from_secs(10)); if let Some(status) = tui.child.try_wait().unwrap() { panic!("plain launch exited instead of attaching: {status}"); } @@ -2565,7 +2565,7 @@ fn plain_launch_attaches_to_existing_local_session() { .unwrap() .iter() .any(|client| client["client_kind"].as_str() == Some("tui")), - "plain launch registration signal preceded server client state" + "plain launch readiness signal preceded server client state" ); } @@ -3036,6 +3036,8 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ fs::create_dir_all(&dir).unwrap(); let socket = dir.join("mux.sock"); let pid_file = dir.join("local-pty.pid"); + let pid_published_path = dir.join("local-pty-published"); + let mut pid_published = TestFifoSignal::create(&pid_published_path); let mut server = PtyChild::start_with_env( &["--socket", socket.to_str().unwrap(), "--ephemeral"], &[ @@ -3044,8 +3046,11 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ ], ); wait_for_socket_path(&socket); - let command = - format!("trap '' HUP TERM; echo $$ > {}; while :; do sleep 1; done", pid_file.display()); + let command = format!( + "trap '' HUP TERM; echo $$ > {}; printf 1 > {}; while :; do sleep 1; done", + pid_file.display(), + pid_published_path.display() + ); let run = raw_cli_command( &socket, serde_json::json!({ @@ -3058,7 +3063,8 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ .output() .unwrap(); assert_success(&run); - let local_pid = wait_for_pid_file(&pid_file, Duration::from_secs(5)); + pid_published.receive(Duration::from_secs(5)); + let local_pid = fs::read_to_string(&pid_file).unwrap().trim().parse().unwrap(); let identity = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "identify"})); let response = json_socket_request( @@ -3206,10 +3212,13 @@ fn cancelled_published_host_is_terminated_through_its_record() { ); let direct_pid_file = server.dir.join("published-direct.pid"); let descendant_pid_file = server.dir.join("published-descendant.pid"); + let pids_published_path = server.dir.join("published-pids-ready"); + let mut pids_published = TestFifoSignal::create(&pids_published_path); let command = format!( - "echo $$ > {}; trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; wait", + "echo $$ > {}; trap '' HUP; (trap '' HUP; while :; do sleep 1; done) & echo $! > {}; printf 1 > {}; wait", direct_pid_file.display(), - descendant_pid_file.display() + descendant_pid_file.display(), + pids_published_path.display() ); let mut create = raw_cli_command( &server.socket, @@ -3226,8 +3235,9 @@ fn cancelled_published_host_is_terminated_through_its_record() { .unwrap(); let host_root = cmux_tui_core::terminal_host_runtime::terminal_host_root(&server.state, "main"); published.receive(Duration::from_secs(5)); - let direct_pid = wait_for_pid_file(&direct_pid_file, Duration::from_secs(5)); - let descendant_pid = wait_for_pid_file(&descendant_pid_file, Duration::from_secs(5)); + pids_published.receive(Duration::from_secs(5)); + let direct_pid = fs::read_to_string(&direct_pid_file).unwrap().trim().parse().unwrap(); + let descendant_pid = fs::read_to_string(&descendant_pid_file).unwrap().trim().parse().unwrap(); assert!(!terminal_host_pids(&host_root).is_empty()); let stop = cli(&server, &["--json", "server", "stop"]); From cbdbe69f9a4c7bd1de1c37024e45fea6b2b2160c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:12:32 -0700 Subject: [PATCH 402/521] fix(tui): complete shutdown owner adaptations --- ...aceRuntimeOwnershipRecoveryRescanScheduler.swift | 1 + .../TerminalSurfaceRestoreSpawnSchedulerTests.swift | 10 +++++----- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 13 +++++++------ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 98881c20fb9d..eedeec4a381d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -1,3 +1,4 @@ +internal import Foundation internal import os /// Coalesces bounded MainActor batches for surface-owned overflow recovery. diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 74a524bf36f3..35d3b6ce2a4d 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -455,7 +455,7 @@ import CmuxTerminalCore @Test func overflowCancellationChurnEmptiesIndexAndLinkedOrder() { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( - maximumRuntimeSurfaceOwnerCount: 1 + maximumRuntimeSurfaceOwnerCount: 2 ) let fixture = makeSurfaceFixture( registry: FakeSurfaceRegistry(), @@ -527,11 +527,11 @@ import CmuxTerminalCore @Test func lifecycleCancellationSynchronouslyRemovesOverflowEntries() throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( - maximumRuntimeSurfaceOwnerCount: 1 + maximumRuntimeSurfaceOwnerCount: 2 ) let saturation = try saturateRuntimeOwnershipRecovery( coordinator, - count: 1 + count: 2 ) defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) @@ -596,11 +596,11 @@ import CmuxTerminalCore @Test func overflowCancelAfterCapacitySignalLeavesOnlyLiveFIFOHead() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( - maximumRuntimeSurfaceOwnerCount: 1 + maximumRuntimeSurfaceOwnerCount: 2 ) let saturation = try saturateRuntimeOwnershipRecovery( coordinator, - count: 1 + count: 2 ) defer { releaseRuntimeOwnershipSaturation(saturation, from: coordinator) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index e32d50bf98a5..a0138545f363 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -328,14 +328,14 @@ fn resolve_host_addresses_until( handoff_resolver_child(child, reaper); return Err(error); } - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + Err(RecvTimeoutError::Timeout) => { handoff_resolver_child(child, reaper); return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "CDP address resolution deadline expired", )); } - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err(RecvTimeoutError::Disconnected) => { handoff_resolver_child(child, reaper); return Err(std::io::Error::other("host resolver output reader stopped")); } @@ -473,8 +473,8 @@ fn run_resolver_child_reaper( } else { match receiver.recv_timeout(wait) { Ok(request) => pending.push(request), - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {} - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => { disconnected = true; } } @@ -1241,10 +1241,10 @@ fn resolve_socket_addr_until( let remaining = remaining_until(deadline, "CDP address resolution deadline expired")?; let addresses = match receiver.recv_timeout(remaining) { Ok(result) => result?, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + Err(RecvTimeoutError::Timeout) => { anyhow::bail!("CDP address resolution deadline expired") } - Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + Err(RecvTimeoutError::Disconnected) => { anyhow::bail!("CDP resolver worker stopped") } }; @@ -3054,6 +3054,7 @@ mod tests { ( Arc::new(Inner { outbound, + transport_close: Mutex::new(TransportClose::Open), pending: Mutex::new(HashMap::new()), events: Arc::new(EventQueue::new()), frame_epochs: Mutex::new(HashMap::new()), From ba68f2d6da045d49d86fd5e64f9ae7e6d8e6b34f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:28:10 -0700 Subject: [PATCH 403/521] test(macOS): bound surface recovery overflow --- .../FakeTerminalSurfacePaneHost.swift | 5 + ...nalSurfaceRestoreSpawnSchedulerTests.swift | 96 ++++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift index c357104def55..eb30bd32bd20 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift @@ -7,6 +7,7 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { private let attachesThroughSurfaceModel: Bool private let onAttach: (() -> Void)? private(set) var explicitInputCount = 0 + private(set) var runtimeSurfaceCreationFailureMessages: [String] = [] init( surfaceView: FakeTerminalSurfaceNativeView, @@ -39,6 +40,10 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { func syncKeyStateIndicator(text: String?) {} func setMobileViewportBorder(size: CGSize?, drawRight: Bool, drawBottom: Bool) {} + func showRuntimeSurfaceCreationFailure(message: String) { + runtimeSurfaceCreationFailureMessages.append(message) + } + func terminalSurfaceDidReceiveExplicitInput() { explicitInputCount += 1 } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 35d3b6ce2a4d..9899a101d472 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -525,6 +525,97 @@ import CmuxTerminalCore #expect(finalSnapshot.tailID == nil) } + @Test func overflowStoreIsBoundedAndSameIDUpdatesStayCapacityNeutral() { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let fixture = makeSurfaceFixture( + registry: FakeSurfaceRegistry(), + scheduler: RecordingRestoreSpawnScheduler(), + runtimeTeardown: coordinator + ) + let firstID = UUID() + let secondID = UUID() + + _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: firstID, + surface: fixture.surface + ) + _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: secondID, + surface: fixture.surface + ) + _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: secondID, + surface: fixture.surface + ) + _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: UUID(), + surface: fixture.surface + ) + + let snapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(snapshot.entryCount == 2) + #expect(snapshot.linkedNodeCount == 2) + #expect(snapshot.headID == firstID) + #expect(snapshot.tailID == secondID) + } + + @Test func fullOverflowStoreReportsFailureAndKeepsAdmittedFIFOService() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let saturation = try saturateRuntimeOwnershipRecovery( + coordinator, + count: 2 + ) + defer { + releaseRuntimeOwnershipSaturation(saturation, from: coordinator) + } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let fixtures = (0..<3).map { _ in + makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + } + + for (index, fixture) in fixtures.enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + } + + let expectedMessage = String( + localized: "terminal.surface.runtimeCreation.capacityExceeded", + defaultValue: + "Unable to start this terminal because too many terminal sessions are still closing." + ) + #expect( + fixtures[2].surface + .runtimeSurfaceAdmissionDeferredCreationSource == nil + ) + #expect( + fixtures[2].paneHost.runtimeSurfaceCreationFailureMessages + == [expectedMessage] + ) + let fullSnapshot = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(fullSnapshot.entryCount == 2) + #expect(fullSnapshot.headID == fixtures[0].surface.id) + #expect(fullSnapshot.tailID == fixtures[1].surface.id) + + coordinator.cancelRuntimeSurfaceOwnershipRecovery( + saturation.recoveryIDs[0] + ) + coordinator.cancelRuntimeSurfaceOwnership(saturation.owners[0]) + await scheduler.waitForScheduledCount(4) + + #expect(scheduler.scheduledSurfaceIds.last == fixtures[0].surface.id) + } + @Test func lifecycleCancellationSynchronouslyRemovesOverflowEntries() throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 @@ -1085,7 +1176,8 @@ import CmuxTerminalCore runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator ) -> ( surface: TerminalSurface, - nativeView: FakeTerminalSurfaceNativeView + nativeView: FakeTerminalSurfaceNativeView, + paneHost: FakeTerminalSurfacePaneHost ) { let nativeView = FakeTerminalSurfaceNativeView( frame: NSRect(x: 0, y: 0, width: 800, height: 600) @@ -1099,7 +1191,7 @@ import CmuxTerminalCore runtimeTeardown: runtimeTeardown ) surface.agentCommandShimInstallCompleted = true - return (surface, nativeView) + return (surface, nativeView, paneHost) } private func saturateRuntimeOwnershipRecovery( From 5de6adc51770b9c4ceae11bcee0bf954d5ab7d61 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:39:43 -0700 Subject: [PATCH 404/521] fix(macOS): bound surface recovery overflow --- .../Hosting/TerminalSurfacePaneHosting.swift | 3 ++ ...inalSurfaceRuntimeOwnershipAdmission.swift | 6 ++-- ...wnershipRecoveryOverflowRegistration.swift | 17 +++++++++ ...timeOwnershipRecoveryRescanScheduler.swift | 21 ++++++++--- ...nalSurfaceRuntimeTeardownCoordinator.swift | 10 ++++-- .../TerminalSurface+RuntimeLifecycle.swift | 28 ++++++++++++--- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 35 ++++++++++++++----- Resources/Localizable.xcstrings | 17 +++++++++ Sources/GhosttyTerminalView.swift | 30 +++++++++++++++- 9 files changed, 142 insertions(+), 25 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift index c0aa51bac652..b5ecb23e6156 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift @@ -26,6 +26,9 @@ public protocol TerminalSurfacePaneHosting: NSView { /// indicator text. func syncKeyStateIndicator(text: String?) + /// Shows a permanent failure message when the native terminal cannot start. + func showRuntimeSurfaceCreationFailure(message: String) + /// Notifies the pane that explicit terminal input is about to be sent. func terminalSurfaceDidReceiveExplicitInput() diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 7d0e05754e56..b4f9dad28217 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -26,8 +26,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { init( maximumOwnerCount: Int, recoveryRescanScheduler: - TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler = - TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler() + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler? = nil ) { precondition( maximumOwnerCount >= TerminalSurfaceRuntimeTeardownCoordinator @@ -36,6 +35,9 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) self.maximumOwnerCount = maximumOwnerCount self.recoveryRescanScheduler = recoveryRescanScheduler + ?? TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( + maximumEntryCount: maximumOwnerCount + ) } func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration.swift new file mode 100644 index 000000000000..f772fde41835 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration.swift @@ -0,0 +1,17 @@ +/// Result of registering one surface with the bounded overflow recovery FIFO. +internal enum TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration: + Equatable, Sendable +{ + case registered(sequence: UInt64) + case updated(sequence: UInt64) + case rejected + + internal var sequence: UInt64? { + switch self { + case .registered(let sequence), .updated(let sequence): + sequence + case .rejected: + nil + } + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index eedeec4a381d..b8a2840b2f6e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -11,6 +11,8 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: { private static let maximumBatchCount = 32 + private let maximumEntryCount: Int + private final class OverflowEntry { let surfaceID: UUID let sequence: UInt64 @@ -44,6 +46,11 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: private let state = OSAllocatedUnfairLock(initialState: State()) + internal init(maximumEntryCount: Int) { + precondition(maximumEntryCount > 0) + self.maximumEntryCount = maximumEntryCount + } + #if DEBUG internal var debugSnapshot: ( @@ -72,14 +79,18 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: internal func registerOverflow( surfaceID: UUID, surface: TerminalSurface - ) -> UInt64 { + ) -> TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration { var startGate: TerminalSurfaceRuntimeTeardownStartGate? - let sequence = state.withLock { state in + let registration = state.withLock { state in if let entry = state.entriesByID[surfaceID] { entry.surface = surface state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) - return entry.sequence + return .updated(sequence: entry.sequence) + } + + guard state.entriesByID.count < maximumEntryCount else { + return .rejected } precondition(state.nextSequence < UInt64.max) @@ -100,10 +111,10 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: state.tailID = surfaceID state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) - return entry.sequence + return .registered(sequence: entry.sequence) } startGate?.start() - return sequence + return registration } internal func cancelOverflow(surfaceID: UUID) { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 59b10d7ca6f7..838d5cef4471 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -63,7 +63,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Creates the process's teardown coordinator. public init() { let recoveryRescanScheduler = - TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler() + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( + maximumEntryCount: Self.maximumRuntimeSurfaceOwnerCount + ) submissionDrain = TerminalSurfaceRuntimeTeardownSubmissionDrain( maximumBufferedOperationCount: Self.maximumRuntimeSurfaceOwnerCount @@ -80,7 +82,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { init(maximumRuntimeSurfaceOwnerCount: Int) { let recoveryRescanScheduler = - TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler() + TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( + maximumEntryCount: maximumRuntimeSurfaceOwnerCount + ) submissionDrain = TerminalSurfaceRuntimeTeardownSubmissionDrain( maximumBufferedOperationCount: maximumRuntimeSurfaceOwnerCount @@ -125,7 +129,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { nonisolated func registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: UUID, surface: TerminalSurface - ) -> UInt64 { + ) -> TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration { recoveryRescanScheduler.registerOverflow( surfaceID: surfaceID, surface: surface diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 7ba214756f07..66f77bfaf6d1 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -794,12 +794,18 @@ extension TerminalSurface { runtimeSurfaceAdmissionOverflowSequence = nil return nil case .rejected: - runtimeSurfaceAdmissionOverflowSequence = + let overflowRegistration = teardownCoordinator - .registerRuntimeSurfaceOwnershipRecoveryOverflow( - surfaceID: id, - surface: self - ) + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: id, + surface: self + ) + switch overflowRegistration { + case .registered(let sequence), .updated(let sequence): + runtimeSurfaceAdmissionOverflowSequence = sequence + case .rejected: + failRuntimeSurfaceCreationForOverflowCapacity() + } return nil } } @@ -888,6 +894,18 @@ extension TerminalSurface { runtimeSurfaceAdmissionDeferredCreationView = nil } + @MainActor + private func failRuntimeSurfaceCreationForOverflowCapacity() { + cancelRuntimeSurfaceCreationAfterAdmissionRecovery() + paneHost.showRuntimeSurfaceCreationFailure( + message: String( + localized: "terminal.surface.runtimeCreation.capacityExceeded", + defaultValue: + "Unable to start this terminal because too many terminal sessions are still closing." + ) + ) + } + @MainActor func createSurface(for view: any TerminalSurfaceNativeViewing, source: RuntimeSurfaceCreationSource) { createSurface( diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 9899a101d472..d4be47dd7de4 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -466,13 +466,14 @@ import CmuxTerminalCore for _ in 0..<(32 * 2 + 1) { let surfaceID = UUID() - let sequence = coordinator + let registration = coordinator .registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: surfaceID, surface: fixture.surface ) - #expect(sequence > previousSequence) - previousSequence = sequence + let expectedSequence = previousSequence + 1 + #expect(registration == .registered(sequence: expectedSequence)) + previousSequence = registration.sequence ?? previousSequence coordinator.cancelRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: surfaceID ) @@ -498,7 +499,16 @@ import CmuxTerminalCore ) let updatedSnapshot = coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot - #expect(updatedSequence == firstSequence) + let firstSequenceValue = firstSequence.sequence + #expect(firstSequenceValue != nil) + #expect( + firstSequence + == .registered(sequence: firstSequenceValue ?? 0) + ) + #expect( + updatedSequence + == .updated(sequence: firstSequenceValue ?? 0) + ) #expect(updatedSnapshot.entryCount == 1) #expect(updatedSnapshot.linkedNodeCount == 1) #expect(updatedSnapshot.headID == repeatedID) @@ -512,7 +522,10 @@ import CmuxTerminalCore surfaceID: repeatedID, surface: fixture.surface ) - #expect(replacementSequence > firstSequence) + #expect( + replacementSequence + == .registered(sequence: (firstSequenceValue ?? 0) + 1) + ) coordinator.cancelRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: repeatedID ) @@ -537,23 +550,27 @@ import CmuxTerminalCore let firstID = UUID() let secondID = UUID() - _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + let first = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: firstID, surface: fixture.surface ) - _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + let second = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: secondID, surface: fixture.surface ) - _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + let updated = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: secondID, surface: fixture.surface ) - _ = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( + let rejected = coordinator.registerRuntimeSurfaceOwnershipRecoveryOverflow( surfaceID: UUID(), surface: fixture.surface ) + #expect(first == .registered(sequence: 1)) + #expect(second == .registered(sequence: 2)) + #expect(updated == .updated(sequence: 2)) + #expect(rejected == .rejected) let snapshot = coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot #expect(snapshot.entryCount == 2) diff --git a/Resources/Localizable.xcstrings b/Resources/Localizable.xcstrings index 3258c129ad65..23b143856e0c 100644 --- a/Resources/Localizable.xcstrings +++ b/Resources/Localizable.xcstrings @@ -241767,6 +241767,23 @@ } } }, + "terminal.surface.runtimeCreation.capacityExceeded": { + "extractionState": "manual", + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Unable to start this terminal because too many terminal sessions are still closing." + } + }, + "ja": { + "stringUnit": { + "state": "translated", + "value": "終了処理中のターミナルセッションが多すぎるため、このターミナルを開始できません。" + } + } + } + }, "terminal.shortcut.name.ctrlL": { "extractionState": "manual", "localizations": { diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index 74a08c1c5e61..dc71141033ec 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -294,7 +294,12 @@ extension GhosttySurfaceCallbackContext { // The surface model drives its views through the CmuxTerminal hosting seams; // the concrete view classes conform here. extension GhosttyNSView: TerminalSurfaceNativeViewing {} -extension GhosttySurfaceScrollView: TerminalSurfacePaneHosting {} +extension GhosttySurfaceScrollView: TerminalSurfacePaneHosting { + func showRuntimeSurfaceCreationFailure(message: String) { + runtimeSurfaceCreationFailureLabel.stringValue = message + runtimeSurfaceCreationFailureLabel.isHidden = false + } +} extension TerminalSurface { /// Concrete-typed convenience over ``TerminalSurface/paneHost`` for app @@ -8228,6 +8233,7 @@ final class GhosttySurfaceScrollView: NSView { private let flashOverlayView: GhosttyFlashOverlayView private let flashLayer: CAShapeLayer private var cloudTerminalReconnectOverlayView: CloudTerminalReconnectOverlayView? + private let runtimeSurfaceCreationFailureLabel = NSTextField(wrappingLabelWithString: "") private var hasVisibilityRevealRefreshScheduled = false var isRightSidebarDockSurface: Bool { surfaceView.terminalSurface?.focusPlacement == .rightSidebarDock @@ -8531,6 +8537,28 @@ final class GhosttySurfaceScrollView: NSView { addSubview(mobileViewportBorderOverlayView, positioned: .above, relativeTo: scrollView) paneDropTargetView.hostedView = self addSubview(paneDropTargetView, positioned: .above, relativeTo: nil) + runtimeSurfaceCreationFailureLabel.translatesAutoresizingMaskIntoConstraints = false + runtimeSurfaceCreationFailureLabel.alignment = .center + runtimeSurfaceCreationFailureLabel.textColor = .labelColor + runtimeSurfaceCreationFailureLabel.drawsBackground = true + runtimeSurfaceCreationFailureLabel.backgroundColor = .windowBackgroundColor.withAlphaComponent(0.94) + runtimeSurfaceCreationFailureLabel.wantsLayer = true + runtimeSurfaceCreationFailureLabel.layer?.cornerRadius = 8 + runtimeSurfaceCreationFailureLabel.layer?.masksToBounds = true + runtimeSurfaceCreationFailureLabel.isHidden = true + addSubview(runtimeSurfaceCreationFailureLabel, positioned: .above, relativeTo: nil) + NSLayoutConstraint.activate([ + runtimeSurfaceCreationFailureLabel.centerXAnchor.constraint(equalTo: centerXAnchor), + runtimeSurfaceCreationFailureLabel.centerYAnchor.constraint(equalTo: centerYAnchor), + runtimeSurfaceCreationFailureLabel.leadingAnchor.constraint( + greaterThanOrEqualTo: leadingAnchor, + constant: 24 + ), + runtimeSurfaceCreationFailureLabel.trailingAnchor.constraint( + lessThanOrEqualTo: trailingAnchor, + constant: -24 + ), + ]) synchronizeScrollbarAppearance() inactiveOverlayView.wantsLayer = true inactiveOverlayView.layer?.backgroundColor = NSColor.clear.cgColor From bc867fb71f81c2a0560325307c7fa99353198f7e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:39:49 -0700 Subject: [PATCH 405/521] fix(tui): remove timeout qualification --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index a0138545f363..e40d23261387 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -5149,7 +5149,7 @@ mod tests { drop(client); assert!( - matches!(shutdown, Err(std::sync::mpsc::RecvTimeoutError::Disconnected)), + matches!(shutdown, Err(RecvTimeoutError::Disconnected)), "reader exit left the event channel live: {shutdown:?}" ); } From 72e54f74946fe3a1e0d577328b0a0c3b48c8485b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:36:25 -0700 Subject: [PATCH 406/521] test(tui): model adopted host identity --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 30 ++++++++++---- cmux-tui/crates/cmux-tui-core/src/surface.rs | 43 +++++++++++++++++++- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5286a300c847..c1c343da3214 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -87,7 +87,7 @@ const TERMINAL_HOST_CLEANUP_RECEIPT_FIELD: &str = "terminal_host_cleanup"; const MAX_TERMINAL_HOST_CLEANUP_IDENTITIES: usize = 4_096; #[cfg(all(test, unix))] type TerminalAdoptionSurfaceFactory = - Arc anyhow::Result> + Send + Sync>; + Arc anyhow::Result> + Send + Sync>; #[cfg(all(test, unix))] type TerminalHostRecordLoader = Arc< dyn Fn(std::path::PathBuf, usize, Instant) -> anyhow::Result + Send + Sync, @@ -4316,7 +4316,13 @@ impl Mux { self.terminal_adoption_surface_factory.lock().unwrap().clone(); #[cfg(test)] let adopted = if let Some(factory) = adoption_surface_factory { - factory(id) + factory( + id, + TerminalHostIdentity { + terminal_id: task.record.terminal_id.clone(), + incarnation: task.record.incarnation.clone(), + }, + ) } else { self.adopt_restored_terminal( restored_binding, @@ -26406,7 +26412,7 @@ mod tests { let (release_tx, release_rx) = std::sync::mpsc::channel(); let release_rx = Arc::new(Mutex::new(release_rx)); *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ - move |_| { + move |_, _| { started_tx.send(()).unwrap(); release_rx.lock().unwrap().recv().unwrap(); anyhow::bail!("blocked startup handshake") @@ -26859,7 +26865,7 @@ mod tests { *mux.terminal_adoption_surface_factory.lock().unwrap() = Some(Arc::new({ let mux = Arc::downgrade(&mux); let options = options.clone(); - move |id| { + move |id, identity| { let call = calls.fetch_add(1, Ordering::AcqRel); if call == 0 { first_started_tx.send(()).unwrap(); @@ -26867,7 +26873,12 @@ mod tests { } else if call == 1 { let _ = second_started_tx.try_send(()); } - Surface::spawn_for_test(id, options.clone(), mux.clone()) + Surface::spawn_for_test_with_terminal_host_identity( + id, + options.clone(), + mux.clone(), + identity, + ) } })); @@ -26949,8 +26960,13 @@ mod tests { let mux = Arc::downgrade(&mux); let options = options.clone(); let kill_attempts = kill_attempts.clone(); - move |id| { - let surface = Surface::spawn_for_test(id, options.clone(), mux.clone())?; + move |id, identity| { + let surface = Surface::spawn_for_test_with_terminal_host_identity( + id, + options.clone(), + mux.clone(), + identity, + )?; let (_, attempts) = surface.set_recovering_server_shutdown_for_test(); *kill_attempts.lock().unwrap() = Some(attempts); Ok(surface) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 192688acc778..660cbaef4963 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -4843,6 +4843,47 @@ impl Surface { resource_identity: Option, lifetime: PtyLifetime, cell_pixels: (u16, u16), + ) -> anyhow::Result> { + Self::spawn_for_test_with_lifetime_and_host_identity_at_cell_pixels( + id, + opts, + mux, + resource_identity, + lifetime, + cell_pixels, + None, + ) + } + + #[cfg(all(test, unix))] + pub(crate) fn spawn_for_test_with_terminal_host_identity( + id: SurfaceId, + opts: SurfaceOptions, + mux: Weak, + host_identity: crate::terminal_host_runtime::TerminalHostIdentity, + ) -> anyhow::Result> { + let cell_pixels = + mux.upgrade().map(|mux| mux.cell_pixel_creation_size()).unwrap_or((8, 16)); + Self::spawn_for_test_with_lifetime_and_host_identity_at_cell_pixels( + id, + opts, + mux, + Some(TabResourceIdentity::terminal(None)?), + PtyLifetime::SessionOwned, + cell_pixels, + Some(host_identity), + ) + } + + #[cfg(test)] + fn spawn_for_test_with_lifetime_and_host_identity_at_cell_pixels( + id: SurfaceId, + opts: SurfaceOptions, + mux: Weak, + resource_identity: Option, + lifetime: PtyLifetime, + cell_pixels: (u16, u16), + host_identity: Option, ) -> anyhow::Result> { let terminal_public_id = resource_identity .as_ref() @@ -4930,7 +4971,7 @@ impl Surface { hosted_shutdown_owner: None, lifetime, supports_clear_history_key_fallback: AtomicBool::new(false), - host_identity: None, + host_identity, #[cfg(unix)] host_exit_record_path: None, pid: Some(id as u32), From 6a014248fc1a0edb8394bc7fc38ef2de8f92824b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:36:31 -0700 Subject: [PATCH 407/521] test(tui): wait for renderer EOF on macOS --- cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 602aa5ca1a44..5d2264c2989d 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -3545,7 +3545,9 @@ fn wait_for_stream_disconnect(stream: &mut UnixStream, timeout: Duration) -> boo 1 => { // SAFETY: kevent initialized exactly one event after returning one. let event = unsafe { event.assume_init() }; - return event.filter == libc::EVFILT_READ && event.flags & libc::EV_EOF != 0; + if event.filter == libc::EVFILT_READ && event.flags & libc::EV_EOF != 0 { + return true; + } } 0 => return false, _ if std::io::Error::last_os_error().kind() == std::io::ErrorKind::Interrupted => { From bfbf7dcceceeba627e508ece253b7687cf994f55 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:56:41 -0700 Subject: [PATCH 408/521] test(macOS): cover stalled close admission --- .../FakeTerminalSurfacePaneHost.swift | 7 + ...lTerminalSurfaceRuntimeTeardownClock.swift | 120 +++++++++++++++ ...nalSurfaceRestoreSpawnSchedulerTests.swift | 141 ++++++++++++++++++ 3 files changed, 268 insertions(+) create mode 100644 Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/ManualTerminalSurfaceRuntimeTeardownClock.swift diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift index eb30bd32bd20..a9c3f1392f00 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift @@ -8,12 +8,18 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { private let onAttach: (() -> Void)? private(set) var explicitInputCount = 0 private(set) var runtimeSurfaceCreationFailureMessages: [String] = [] + let runtimeSurfaceCreationFailures: AsyncStream + private let runtimeSurfaceCreationFailureContinuation: + AsyncStream.Continuation init( surfaceView: FakeTerminalSurfaceNativeView, attachesThroughSurfaceModel: Bool = false, onAttach: (() -> Void)? = nil ) { + (runtimeSurfaceCreationFailures, + runtimeSurfaceCreationFailureContinuation) = + AsyncStream.makeStream(of: String.self) self.surfaceView = surfaceView self.attachesThroughSurfaceModel = attachesThroughSurfaceModel self.onAttach = onAttach @@ -42,6 +48,7 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { func showRuntimeSurfaceCreationFailure(message: String) { runtimeSurfaceCreationFailureMessages.append(message) + runtimeSurfaceCreationFailureContinuation.yield(message) } func terminalSurfaceDidReceiveExplicitInput() { diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/ManualTerminalSurfaceRuntimeTeardownClock.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/ManualTerminalSurfaceRuntimeTeardownClock.swift new file mode 100644 index 000000000000..c2562933cc71 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/ManualTerminalSurfaceRuntimeTeardownClock.swift @@ -0,0 +1,120 @@ +import Foundation +import os + +struct ManualTerminalSurfaceRuntimeTeardownInstant: + InstantProtocol +{ + typealias Duration = Swift.Duration + + let offset: Duration + + func advanced(by duration: Duration) -> Self { + Self(offset: offset + duration) + } + + func duration(to other: Self) -> Duration { + other.offset - offset + } + + static func < ( + lhs: ManualTerminalSurfaceRuntimeTeardownInstant, + rhs: ManualTerminalSurfaceRuntimeTeardownInstant + ) -> Bool { + lhs.offset < rhs.offset + } +} + +/// Manual clock whose positive registration stream is the test's watchdog +/// readiness signal. Advancing one registration resumes only that watchdog. +final class ManualTerminalSurfaceRuntimeTeardownClock: + Clock, + @unchecked Sendable +{ + typealias Duration = Swift.Duration + typealias Instant = ManualTerminalSurfaceRuntimeTeardownInstant + + private struct Waiter { + let deadline: Instant + let continuation: CheckedContinuation + } + + private struct State { + var now = Instant(offset: .zero) + var nextID = 0 + var waiters: [Int: Waiter] = [:] + var cancelledIDs: Set = [] + } + + let registrations: AsyncStream + private let registrationContinuation: AsyncStream.Continuation + private let state = OSAllocatedUnfairLock(initialState: State()) + + init() { + (registrations, registrationContinuation) = + AsyncStream.makeStream(of: Int.self) + } + + var now: Instant { + state.withLock(\.now) + } + + var minimumResolution: Duration { + .nanoseconds(1) + } + + func sleep( + until deadline: Instant, + tolerance: Duration? + ) async throws { + try Task.checkCancellation() + let id = state.withLock { state in + let id = state.nextID + state.nextID += 1 + return id + } + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { continuation in + let registered = state.withLock { state in + if state.cancelledIDs.remove(id) != nil { + return false + } + if deadline <= state.now { + return false + } + state.waiters[id] = Waiter( + deadline: deadline, + continuation: continuation + ) + return true + } + if registered { + registrationContinuation.yield(id) + } else if Task.isCancelled { + continuation.resume(throwing: CancellationError()) + } else { + continuation.resume() + } + } + } onCancel: { + let waiter = state.withLock { state in + guard let waiter = state.waiters.removeValue(forKey: id) else { + state.cancelledIDs.insert(id) + return nil + } + return waiter + } + waiter?.continuation.resume(throwing: CancellationError()) + } + } + + func fire(_ id: Int) { + let waiter = state.withLock { state in + guard let waiter = state.waiters.removeValue(forKey: id) else { + return nil + } + state.now = max(state.now, waiter.deadline) + return waiter + } + waiter?.continuation.resume() + } +} diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index d4be47dd7de4..50ed4739b6bf 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -2,6 +2,7 @@ import AppKit import Dispatch import Foundation import GhosttyKit +import os import Testing import CmuxTerminalCore @testable import CmuxTerminal @@ -633,6 +634,146 @@ import CmuxTerminalCore #expect(scheduler.scheduledSurfaceIds.last == fixtures[0].surface.id) } + @Test func stalledCloseWorkersFailDeferredCreationAndRecoverSafely() async throws { + let clock = ManualTerminalSurfaceRuntimeTeardownClock() + let stalledSlots = AsyncStream.makeStream() + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2, + closeTeardownTimeout: .seconds(5), + closeTeardownClock: clock, + closeTeardownStalledObserver: { slot in + stalledSlots.continuation.yield(slot) + } + ) + let pointers = (0..<2).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = (0..<2).map { _ in DispatchSemaphore(value: 0) } + let freedPointerBits = OSAllocatedUnfairLock(initialState: [UInt]()) + defer { + for releaseFree in releaseFrees { + releaseFree.signal() + } + for pointer in pointers { + pointer.deallocate() + } + freeStarted.continuation.finish() + stalledSlots.continuation.finish() + } + let reservations = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let tickets = try pointers.enumerated().map { index, pointer in + try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.stalledClose.\(index)", + surface: pointer, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: reservations[index], + freeSurface: { pointer in + freeStarted.continuation.yield(index) + releaseFrees[index].wait() + freedPointerBits.withLock { + $0.append(UInt(bitPattern: pointer)) + } + } + ) + ) + } + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = try #require(await freeStartedIterator.next()) + _ = try #require(await freeStartedIterator.next()) + var registrationIterator = clock.registrations.makeAsyncIterator() + let firstWatchdog = try #require(await registrationIterator.next()) + let secondWatchdog = try #require(await registrationIterator.next()) + + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let deferredFixtures = (0..<3).map { _ in + makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + } + for (index, fixture) in deferredFixtures.enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + } + + var stalledSlotIterator = stalledSlots.stream.makeAsyncIterator() + clock.fire(firstWatchdog) + _ = try #require(await stalledSlotIterator.next()) + #expect( + deferredFixtures.allSatisfy { + $0.paneHost.runtimeSurfaceCreationFailureMessages.isEmpty + } + ) + + var failureIterators = deferredFixtures.map { + $0.paneHost.runtimeSurfaceCreationFailures.makeAsyncIterator() + } + clock.fire(secondWatchdog) + _ = try #require(await stalledSlotIterator.next()) + let expectedMessage = String( + localized: "terminal.surface.runtimeCreation.capacityExceeded", + defaultValue: + "Unable to start this terminal because too many terminal sessions are still closing." + ) + for index in failureIterators.indices { + #expect( + try #require(await failureIterators[index].next()) + == expectedMessage + ) + #expect( + deferredFixtures[index].surface + .runtimeSurfaceAdmissionDeferredCreationSource == nil + ) + } + assertOverflowStorageIsEmpty(coordinator) + + let rejectedAfterStall = makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + var postStallFailureIterator = rejectedAfterStall.paneHost + .runtimeSurfaceCreationFailures.makeAsyncIterator() + rejectedAfterStall.surface.createSurface( + for: rejectedAfterStall.nativeView + ) + scheduler.runScheduledOperation(at: deferredFixtures.count) + #expect( + try #require(await postStallFailureIterator.next()) + == expectedMessage + ) + + releaseFrees[0].signal() + #expect(await tickets[0].wait(timeout: nil)) + #expect( + freedPointerBits.withLock { $0 } + == [UInt(bitPattern: pointers[0])] + ) + let recoveredReservation = try #require( + coordinator.reserveRuntimeSurfaceOwnership() + ) + coordinator.cancelRuntimeSurfaceOwnership(recoveredReservation) + #expect(await !coordinator.debugCloseTeardownAllStalled) + + releaseFrees[1].signal() + #expect(await tickets[1].wait(timeout: nil)) + #expect( + Set(freedPointerBits.withLock { $0 }) + == Set(pointers.map { UInt(bitPattern: $0) }) + ) + #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) + } + @Test func lifecycleCancellationSynchronouslyRemovesOverflowEntries() throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 From c50e8f350733004b8b87d3a3313dac9de1cbc018 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:02:21 -0700 Subject: [PATCH 409/521] fix(macOS): fail stalled close admission --- ...TerminalSurfaceRuntimeActiveTeardown.swift | 1 + ...inalSurfaceRuntimeOwnershipAdmission.swift | 58 ++++++++++++- ...urfaceRuntimeOwnershipAdmissionState.swift | 1 + ...timeOwnershipRecoveryAdmissionResult.swift | 3 + ...SurfaceRuntimeOwnershipRecoveryEntry.swift | 1 + ...timeOwnershipRecoveryRescanScheduler.swift | 68 +++++++++++++-- ...faceRuntimeTeardownCoordinator+Debug.swift | 9 +- ...nalSurfaceRuntimeTeardownCoordinator.swift | 84 ++++++++++++++++++- .../TerminalSurface+RuntimeLifecycle.swift | 21 ++++- 9 files changed, 231 insertions(+), 15 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift index 9303590fa226..8424a0a01fea 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeActiveTeardown.swift @@ -3,4 +3,5 @@ import Foundation internal struct TerminalSurfaceRuntimeActiveTeardown: Sendable { let ticketID: UUID let task: Task + let watchdogTask: Task? } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index b4f9dad28217..cd742cac4285 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -3,6 +3,8 @@ internal import os typealias TerminalSurfaceRuntimeOwnershipRecovery = @MainActor @Sendable (TerminalSurfaceRuntimeOwnershipReservation) -> Void +typealias TerminalSurfaceRuntimeOwnershipRecoveryFailure = + @MainActor @Sendable () -> Void /// Synchronous, bounded admission for native-surface ownership. /// @@ -43,6 +45,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func reserve() -> TerminalSurfaceRuntimeOwnershipReservation? { state.withLock { state in guard !state.closeTeardownDegraded, + !state.closeTeardownAllStalled, state.reservationIDs.count < maximumOwnerCount, state.ingressReservationIDs.count < maximumOwnerCount else { return nil @@ -54,6 +57,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func reserve( recoveryID: UUID, onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + onFailure: @escaping TerminalSurfaceRuntimeOwnershipRecoveryFailure = {}, capacityReservation: TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? = nil ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { @@ -72,6 +76,9 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } else { claimedCapacity = false } + guard !state.closeTeardownAllStalled else { + return (.closeTeardownStalled, claimedCapacity) + } if !state.closeTeardownDegraded, state.reservationIDs.count < maximumOwnerCount, state.ingressReservationIDs.count < maximumOwnerCount @@ -84,6 +91,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery + state.recoveryEntriesByID[recoveryID]?.failure = onFailure return (.deferred, claimedCapacity) } guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { @@ -91,6 +99,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } enqueueRecoveryAction( onRecovery, + onFailure: onFailure, id: recoveryID, in: &state ) @@ -105,7 +114,10 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func claimRecoveryCapacity() -> TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? { state.withLock { state in - guard recoveryCapacityIsOpen(in: state) else { return nil } + guard !state.closeTeardownAllStalled, + recoveryCapacityIsOpen(in: state) else { + return nil + } let reservation = TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation() state.recoveryCapacityReservationIDs.insert(reservation.id) @@ -114,7 +126,9 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func recoveryCapacityIsOpen() -> Bool { - state.withLock { recoveryCapacityIsOpen(in: $0) } + state.withLock { + !$0.closeTeardownAllStalled && recoveryCapacityIsOpen(in: $0) + } } func releaseRecoveryCapacity( @@ -243,6 +257,39 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } } + func failRecoveriesForAllStalledCloseTeardowns() + -> [TerminalSurfaceRuntimeOwnershipRecoveryFailure] { + state.withLock { state in + state.closeTeardownAllStalled = true + var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] = [] + while let recoveryID = state.recoveryHeadID, + let failure = state.recoveryEntriesByID[recoveryID]?.failure { + failures.append(failure) + _ = removeRecoveryAction(recoveryID, from: &state) + } + state.recoveryRescanRequested = false + return failures + } + } + + func clearAllStalledCloseTeardowns() { + let output: ( + grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant?, + requestRescan: Bool + ) = state.withLock { state in + guard state.closeTeardownAllStalled else { + return (nil, false) + } + state.closeTeardownAllStalled = false + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() + } + } + func cancelRecovery(_ recoveryID: UUID) { let requestRescan = state.withLock { state in _ = removeRecoveryAction(recoveryID, from: &state) @@ -278,6 +325,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { private func enqueueRecoveryAction( _ action: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + onFailure: @escaping TerminalSurfaceRuntimeOwnershipRecoveryFailure, id recoveryID: UUID, in state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) { @@ -285,6 +333,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.recoveryEntriesByID[recoveryID] = TerminalSurfaceRuntimeOwnershipRecoveryEntry( action: action, + failure: onFailure, previousID: previousID, nextID: nil ) @@ -326,6 +375,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) -> TerminalSurfaceRuntimeOwnershipRecoveryGrant? { guard !state.closeTeardownDegraded, + !state.closeTeardownAllStalled, !state.recoveryGrantIsScheduled, state.reservationIDs.count < maximumOwnerCount, state.ingressReservationIDs.count < maximumOwnerCount, @@ -371,6 +421,10 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.withLock { $0.closeTeardownDegraded } } + var debugCloseTeardownAllStalled: Bool { + state.withLock { $0.closeTeardownAllStalled } + } + var debugOwnerCount: Int { state.withLock { $0.reservationIDs.count } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index 0b863b4ccf85..ebd6a1b83f97 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -7,6 +7,7 @@ struct TerminalSurfaceRuntimeOwnershipAdmissionState { /// Owner slots that have not transferred to an enqueue submission. var unclaimedOwnershipIngressReservationIDs: Set = [] var closeTeardownDegraded = false + var closeTeardownAllStalled = false var recoveryEntriesByID: [UUID: TerminalSurfaceRuntimeOwnershipRecoveryEntry] = [:] var recoveryCapacityReservationIDs: Set = [] diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift index 91d374e90204..e3d6fff616bd 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult.swift @@ -8,4 +8,7 @@ nonisolated enum TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult: Equatab /// The bounded recovery queue rejected the request without retaining it. case rejected + + /// Every bounded close worker exceeded its watchdog deadline. + case closeTeardownStalled } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift index f2c4f42af7f6..382020416d70 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift @@ -2,6 +2,7 @@ import Foundation struct TerminalSurfaceRuntimeOwnershipRecoveryEntry: Sendable { var action: TerminalSurfaceRuntimeOwnershipRecovery + var failure: TerminalSurfaceRuntimeOwnershipRecoveryFailure var previousID: UUID? var nextID: UUID? } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index b8a2840b2f6e..c2a88490acaf 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -40,6 +40,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var entriesByID: [UUID: OverflowEntry] = [:] var headID: UUID? var tailID: UUID? + var failureThroughSequence: UInt64? var rescanRequested = false var rescanTask: Task? } @@ -136,6 +137,22 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: startGate?.start() } + internal func failPendingOverflowCreations() { + var startGate: TerminalSurfaceRuntimeTeardownStartGate? + state.withLock { state in + guard let tailID = state.tailID, + let tail = state.entriesByID[tailID] + else { return } + state.failureThroughSequence = max( + state.failureThroughSequence ?? 0, + tail.sequence + ) + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + } + startGate?.start() + } + deinit { state.withLock { state in state.rescanTask?.cancel() @@ -165,6 +182,24 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var processedCount = 0 if shouldRun { while processedCount < Self.maximumBatchCount { + if let failedEntry = state.withLock({ state -> OverflowEntry? in + guard let cutoff = state.failureThroughSequence, + let headID = state.headID, + let entry = state.entriesByID[headID], + entry.sequence <= cutoff + else { + normalizeFailureCutoff(in: &state) + return nil + } + _ = removeOverflow(surfaceID: entry.surfaceID, from: &state) + normalizeFailureCutoff(in: &state) + return entry + }) { + failedEntry.surface? + .failRuntimeSurfaceCreationForTeardownCapacity() + processedCount += 1 + continue + } guard let entry = state.withLock({ state in state.headID.flatMap { state.entriesByID[$0] } @@ -201,7 +236,8 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } } - let followUp = state.withLock { state -> (OverflowEntry, Bool)? in + let followUp = state.withLock { + state -> (OverflowEntry, Bool, Bool)? in state.rescanTask = nil let externallyRequested = state.rescanRequested state.rescanRequested = false @@ -210,19 +246,29 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: else { return nil } - return (entry, externallyRequested) + return ( + entry, + externallyRequested, + state.failureThroughSequence != nil + ) + } + guard let (entry, externallyRequested, failurePending) = followUp else { + return } - guard let (entry, externallyRequested) = followUp else { return } let batchWasExhausted = processedCount == Self.maximumBatchCount let capacityRemains: Bool - if let surface = entry.surface { + if failurePending { + capacityRemains = true + } else if let surface = entry.surface { capacityRemains = surface.runtimeTeardown .runtimeSurfaceOwnershipRecoveryCapacityIsOpen() } else { capacityRemains = true } - if capacityRemains && (batchWasExhausted || externallyRequested) { + if capacityRemains + && (batchWasExhausted || externallyRequested || failurePending) + { requestRescan() } } @@ -248,8 +294,20 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: if state.entriesByID.isEmpty { state.headID = nil state.tailID = nil + state.failureThroughSequence = nil state.rescanRequested = false } return entry } + + private func normalizeFailureCutoff(in state: inout State) { + guard let cutoff = state.failureThroughSequence else { return } + guard let headID = state.headID, + let head = state.entriesByID[headID], + head.sequence <= cutoff + else { + state.failureThroughSequence = nil + return + } + } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift index 2b25229eb7cb..176012840586 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift @@ -13,12 +13,17 @@ extension TerminalSurfaceRuntimeTeardownCoordinator { pendingRequestsById.count } - /// Test support: whether every close worker has exceeded its deadline and - /// new native-surface ownership is fenced until one worker returns. + /// Test support: whether both close slots are active and new ownership is + /// fenced until one worker returns. public var debugCloseTeardownDegraded: Bool { runtimeOwnershipAdmission.debugCloseTeardownDegraded } + /// Test support: both close workers exceeded their watchdog deadlines. + public var debugCloseTeardownAllStalled: Bool { + runtimeOwnershipAdmission.debugCloseTeardownAllStalled + } + /// Test support: live plus retained native surfaces charged to admission. public nonisolated var debugRuntimeSurfaceOwnerCount: Int { runtimeOwnershipAdmission.debugOwnerCount diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 838d5cef4471..79ee44a39fd4 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -13,7 +13,9 @@ internal import CMUXDebugLog /// Every native surface holds a bounded ownership reservation from creation /// through completed free. While both close slots are occupied, new ownership /// waits for a worker-completion signal, so retained frees cannot grow without -/// bound. +/// bound. If both workers exceed their five-second watchdog, queued creations +/// report a visible failure. A late native free keeps its pointer ownership and +/// restores admission only after it returns. /// Each admitted hibernation also owns one independently startable utility slot. /// Slot occupancy fences new ownership until one active free completes. The app constructs /// exactly one instance and injects it through @@ -28,6 +30,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Largest batch that can own independently startable native-free slots. public static let maximumIsolatedHibernationTeardownCount = 2 + private let closeTeardownTimeout: Duration + private nonisolated let closeTeardownClock: any Clock + private let closeTeardownStalledObserver: @Sendable (Int) -> Void private nonisolated let submissionDrain: TerminalSurfaceRuntimeTeardownSubmissionDrain private nonisolated let recoveryRescanScheduler: @@ -46,6 +51,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { [Int: TerminalSurfaceRuntimeActiveTeardown] = [:] private var activeHibernationTeardownsBySlot: [Int: TerminalSurfaceRuntimeActiveTeardown] = [:] + private var stalledCloseExecutionSlots: Set = [] private var cancelledTicketIDs: Set = [] #if DEBUG // Readable at internal scope in DEBUG so the debug-only extension in @@ -62,6 +68,9 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { /// Creates the process's teardown coordinator. public init() { + closeTeardownTimeout = .seconds(5) + closeTeardownClock = ContinuousClock() + closeTeardownStalledObserver = { _ in } let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: Self.maximumRuntimeSurfaceOwnerCount @@ -80,7 +89,16 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ) } - init(maximumRuntimeSurfaceOwnerCount: Int) { + init( + maximumRuntimeSurfaceOwnerCount: Int, + closeTeardownTimeout: Duration = .seconds(5), + closeTeardownClock: any Clock = ContinuousClock(), + closeTeardownStalledObserver: @escaping @Sendable (Int) -> Void = { _ in } + ) { + precondition(closeTeardownTimeout > .zero) + self.closeTeardownTimeout = closeTeardownTimeout + self.closeTeardownClock = closeTeardownClock + self.closeTeardownStalledObserver = closeTeardownStalledObserver let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: maximumRuntimeSurfaceOwnerCount @@ -102,9 +120,11 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { deinit { for active in activeCloseTeardownsBySlot.values { active.task.cancel() + active.watchdogTask?.cancel() } for active in activeHibernationTeardownsBySlot.values { active.task.cancel() + active.watchdogTask?.cancel() } } @@ -116,12 +136,14 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { nonisolated func reserveRuntimeSurfaceOwnership( recoveryID: UUID, onRecovery: @escaping TerminalSurfaceRuntimeOwnershipRecovery, + onFailure: @escaping TerminalSurfaceRuntimeOwnershipRecoveryFailure = {}, capacityReservation: TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? = nil ) -> TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult { runtimeOwnershipAdmission.reserve( recoveryID: recoveryID, onRecovery: onRecovery, + onFailure: onFailure, capacityReservation: capacityReservation ) } @@ -462,7 +484,12 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { ) let active = TerminalSurfaceRuntimeActiveTeardown( ticketID: request.ticketID, - task: prepared.task + task: prepared.task, + watchdogTask: makeCloseTeardownWatchdog( + request: request, + executionLane: executionLane, + executionSlot: executionSlot + ) ) switch executionLane { case .boundedClose: @@ -526,10 +553,12 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { active = activeHibernationTeardownsBySlot[executionSlot] } guard active?.ticketID == request.ticketID else { return } + active?.watchdogTask?.cancel() switch executionLane { case .boundedClose: activeCloseTeardownsBySlot.removeValue(forKey: executionSlot) + stalledCloseExecutionSlots.remove(executionSlot) case .isolatedHibernation: activeHibernationTeardownsBySlot.removeValue(forKey: executionSlot) } @@ -552,12 +581,61 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { } private func updateCloseTeardownAdmission() { + if stalledCloseExecutionSlots.count + < Self.maximumConcurrentCloseTeardownCount { + runtimeOwnershipAdmission.clearAllStalledCloseTeardowns() + } runtimeOwnershipAdmission.setCloseTeardownDegraded( activeCloseTeardownsBySlot.count == Self.maximumConcurrentCloseTeardownCount ) } + private func makeCloseTeardownWatchdog( + request: TerminalSurfaceRuntimeTeardownRequest, + executionLane: TerminalSurfaceRuntimeTeardownExecutionLane, + executionSlot: Int + ) -> Task? { + guard executionLane == .boundedClose else { return nil } + let timeout = closeTeardownTimeout + let clock = closeTeardownClock + return Task { + do { + try await clock.sleep(for: timeout, tolerance: nil) + } catch { + return + } + guard !Task.isCancelled else { return } + closeTeardownWatchdogExpired( + ticketID: request.ticketID, + executionSlot: executionSlot + ) + } + } + + private func closeTeardownWatchdogExpired( + ticketID: UUID, + executionSlot: Int + ) { + guard activeCloseTeardownsBySlot[executionSlot]?.ticketID == ticketID, + stalledCloseExecutionSlots.insert(executionSlot).inserted else { + return + } + closeTeardownStalledObserver(executionSlot) + guard stalledCloseExecutionSlots.count + == Self.maximumConcurrentCloseTeardownCount else { + return + } + let failures = runtimeOwnershipAdmission + .failRecoveriesForAllStalledCloseTeardowns() + recoveryRescanScheduler.failPendingOverflowCreations() + Task { @MainActor in + for failure in failures { + failure() + } + } + } + private nonisolated func freeNativeSurface( _ request: TerminalSurfaceRuntimeTeardownRequest ) { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 66f77bfaf6d1..662849c21626 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -761,6 +761,14 @@ extension TerminalSurface { } } + @MainActor + private func runtimeSurfaceOwnershipRecoveryFailure() + -> TerminalSurfaceRuntimeOwnershipRecoveryFailure { + { [weak self] in + self?.failRuntimeSurfaceCreationForTeardownCapacity() + } + } + @MainActor private func reserveRuntimeSurfaceOwnershipForCreation( view: any TerminalSurfaceNativeViewing, @@ -778,7 +786,8 @@ extension TerminalSurface { recoveryID: id, onRecovery: runtimeSurfaceOwnershipRecovery( teardownCoordinator: teardownCoordinator - ) + ), + onFailure: runtimeSurfaceOwnershipRecoveryFailure() ) switch admissionResult { case .reserved(let reservation): @@ -804,9 +813,12 @@ extension TerminalSurface { case .registered(let sequence), .updated(let sequence): runtimeSurfaceAdmissionOverflowSequence = sequence case .rejected: - failRuntimeSurfaceCreationForOverflowCapacity() + failRuntimeSurfaceCreationForTeardownCapacity() } return nil + case .closeTeardownStalled: + failRuntimeSurfaceCreationForTeardownCapacity() + return nil } } @@ -837,6 +849,7 @@ extension TerminalSurface { onRecovery: runtimeSurfaceOwnershipRecovery( teardownCoordinator: teardownCoordinator ), + onFailure: runtimeSurfaceOwnershipRecoveryFailure(), capacityReservation: capacityReservation ) switch admissionResult { @@ -854,6 +867,8 @@ extension TerminalSurface { case .rejected: teardownCoordinator .requestRuntimeSurfaceOwnershipRecoveryRescan() + case .closeTeardownStalled: + failRuntimeSurfaceCreationForTeardownCapacity() } } @@ -895,7 +910,7 @@ extension TerminalSurface { } @MainActor - private func failRuntimeSurfaceCreationForOverflowCapacity() { + func failRuntimeSurfaceCreationForTeardownCapacity() { cancelRuntimeSurfaceCreationAfterAdmissionRecovery() paneHost.showRuntimeSurfaceCreationFailure( message: String( From 658ce4ce8b10f7c70f19bed2a97907676227084a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:14:03 -0700 Subject: [PATCH 410/521] test(tui): exercise provider slot release --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 46 ++++++++++++++++-------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c1c343da3214..1b1ef007245f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -8770,6 +8770,10 @@ impl Mux { } } + #[cfg(test)] + if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { + hook(); + } let connected = BrowserRuntime::connect_provider(&lease.endpoint, &lease.authentication); let mut state = self.browser_runtime.state.lock().unwrap(); match connected { @@ -27807,13 +27811,7 @@ mod tests { #[test] fn browser_runtime_connection_does_not_hold_the_shared_slot() { - let mux = Mux::new_for_test( - "browser-runtime-slot", - SurfaceOptions { - cdp_url: Some("ws://127.0.0.1:9/devtools/browser/unreachable".into()), - ..SurfaceOptions::default() - }, - ); + let mux = Mux::new_for_test("browser-runtime-slot", SurfaceOptions::default()); let (connect_started_tx, connect_started_rx) = std::sync::mpsc::sync_channel(1); let (release_connect_tx, release_connect_rx) = std::sync::mpsc::sync_channel(1); let release_connect_rx = Arc::new(Mutex::new(release_connect_rx)); @@ -27823,22 +27821,42 @@ mod tests { release_connect_rx.lock().unwrap().recv().unwrap(); } })); - mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); - connect_started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let lease = BrowserProviderTargetLease { + provider_id: "browser-runtime-slot".into(), + endpoint: "ws://127.0.0.1:9/devtools/browser/unreachable".into(), + authentication: crate::browser_provider::BrowserProviderAuthentication::None, + tab_id: TabPublicId::parse("tab_00000000000000000000000000000001").unwrap(), + target_id: "target-1".into(), + revision: 1, + }; + let (connect_done_tx, connect_done_rx) = std::sync::mpsc::sync_channel(1); + let connect = std::thread::spawn({ + let mux = mux.clone(); + move || { + connect_done_tx.send(mux.browser_runtime_for_provider(&lease).is_err()).unwrap(); + } + }); + let connect_started = connect_started_rx.recv_timeout(Duration::from_secs(1)).is_ok(); let slot_available = mux.browser_runtime.lock_available_for_test(); - release_connect_tx.send(()).unwrap(); - assert!( - mux.async_surface_creations.wait_until_idle(Instant::now() + Duration::from_secs(1)), - "browser surface creation did not finish" - ); + let _ = release_connect_tx.send(()); + let connect_failed = connect_done_rx.recv_timeout(Duration::from_secs(1)).ok(); + if connect_failed.is_some() { + connect.join().unwrap(); + } mux.request_daemon_shutdown(); mux.shutdown().unwrap(); + assert!(connect_started, "provider connection did not reach the blocking test hook"); assert!( slot_available, "browser connection setup held the shared runtime slot across blocking work" ); + assert_eq!( + connect_failed, + Some(true), + "provider connection did not fail within the final bound" + ); } #[test] From 8c0257750b2e0940a3e8c67e71035324fc180464 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:29:04 -0700 Subject: [PATCH 411/521] fix(tui): preserve terminal cleanup identity --- cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index 5beef633ce8f..db07d57ba460 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2373,7 +2373,7 @@ impl Mux { let terminal = registry.terminal_record(terminal_id)?.ok_or_else(|| { terminal_close_state_error(format!("terminal close omitted host {terminal_id}")) })?; - vec![(terminal_id.to_string(), Some(terminal.incarnation))] + vec![(terminal_id.to_string(), terminal.incarnation)] } else { Vec::new() }; From 0a72f8319144e8e0bff962ff465b29e2f70ae992 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:08:19 -0700 Subject: [PATCH 412/521] test(tui): cover restored terminal tab ordering --- .../cmux-tui-core/src/mux/resource_content.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs index 34d1505eb816..3dd0e8b86029 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs @@ -1362,3 +1362,53 @@ fn public_tab_value(tab: &RegistryTab, focused: bool) -> Value { "content_id": tab.content_id.as_str(), }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Pane; + use crate::resource::{PublicSlotIndexes, TerminalPublicId}; + + #[test] + fn restored_terminal_tab_order_uses_durable_identity_before_adoption() { + let pane_slot = 7; + let surface_slot = 16; + let pane_id = PanePublicId::parse("pane_00000000000000000000000000000007").unwrap(); + let tab_id = TabPublicId::parse("tab_00000000000000000000000000000016").unwrap(); + let terminal_id = TerminalPublicId::parse("term_00000000000000000000000000000016").unwrap(); + let mut indexes = PublicSlotIndexes::default(); + indexes.tab_ids.insert(surface_slot, tab_id.clone()); + indexes.content_ids.insert(surface_slot, ContentPublicId::Terminal(terminal_id.clone())); + let state = State { + workspaces: Vec::new(), + workspace_index_by_id: HashMap::new(), + workspace_id_by_key: HashMap::new(), + workspace_revision: 0, + pane_revision: 0, + resource_revision: 0, + focus_sequence: 0, + active_workspace: 0, + panes: HashMap::from([( + pane_slot, + Pane { + id: pane_slot, + public_id: pane_id, + name: None, + tabs: vec![surface_slot], + active_tab: 0, + active_at: 0, + focused_at: 0, + }, + )]), + surfaces: HashMap::new(), + terminal_catalog: HashMap::new(), + terminal_catalog_by_runtime: HashMap::new(), + split_screens: HashMap::new(), + resource_indexes: indexes, + }; + + let ordered = ordered_terminal_tab_ids(&state).unwrap(); + + assert_eq!(ordered.get(&terminal_id), Some(&vec![tab_id])); + } +} From 50f196506881f7c1c7e8ef196e608fc91407bf9d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:09:20 -0700 Subject: [PATCH 413/521] fix(tui): project restored terminal tabs --- .../cmux-tui-core/src/mux/resource_content.rs | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs index 3dd0e8b86029..259014a7d839 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_content.rs @@ -727,22 +727,8 @@ impl Mux { let mut tab_order = Vec::with_capacity(pane.tabs.len()); for (position, surface_slot) in pane.tabs.iter().enumerate() { - // Restored terminal tabs exist in the topology before - // their host runtime is adopted. A structural commit - // must preserve those durable views instead of making - // unrelated host adoption a precondition. let surface = state.surfaces.get(surface_slot); - let identity = surface - .and_then(|surface| surface.resource_identity().cloned()) - .or_else(|| { - Some(TabResourceIdentity::new( - state.resource_indexes.tab_ids.get(surface_slot)?.clone(), - state.resource_indexes.content_ids.get(surface_slot)?.clone(), - )) - }) - .with_context(|| { - format!("pane surface {surface_slot} has no resource identity") - })?; + let identity = tab_resource_identity(state, *surface_slot)?; let before_tab = before_tabs.get(&identity.tab_id); live_tabs.insert(identity.tab_id.clone()); tab_order.push(identity.tab_id.clone()); @@ -1078,13 +1064,7 @@ fn ordered_terminal_tab_ids( let mut tabs = Vec::new(); for pane in state.panes.values() { for (position, surface_slot) in pane.tabs.iter().enumerate() { - let surface = state - .surfaces - .get(surface_slot) - .with_context(|| format!("pane references missing surface {surface_slot}"))?; - let identity = surface - .resource_identity() - .with_context(|| format!("pane surface {surface_slot} has no resource identity"))?; + let identity = tab_resource_identity(state, *surface_slot)?; if let ContentPublicId::Terminal(terminal_id) = &identity.content_id { tabs.push(( terminal_id.clone(), @@ -1098,6 +1078,32 @@ fn ordered_terminal_tab_ids( Ok(terminal_tab_ids_in_canonical_order(tabs)) } +/// A restored tab is durable before its terminal or browser runtime is +/// materialized. Structural projection must use the durable indexes during +/// that bounded startup state instead of making runtime adoption a +/// precondition for unrelated mutations. +fn tab_resource_identity( + state: &State, + surface_slot: crate::SurfaceId, +) -> anyhow::Result { + if let Some(identity) = + state.surfaces.get(&surface_slot).and_then(|surface| surface.resource_identity()) + { + return Ok(identity.clone()); + } + let tab_id = state + .resource_indexes + .tab_ids + .get(&surface_slot) + .cloned() + .with_context(|| format!("pane surface {surface_slot} has no public tab identity"))?; + let content_id = + state.resource_indexes.content_ids.get(&surface_slot).cloned().with_context(|| { + format!("pane surface {surface_slot} has no public content identity") + })?; + Ok(TabResourceIdentity::new(tab_id, content_id)) +} + fn registry_screen_from_live( state: &State, workspace_id: &WorkspacePublicId, From 4423a96f91c693af9d665fe90c4d1efc6a30fb84 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:53:31 -0700 Subject: [PATCH 414/521] test(tui): exercise shutdown fanout ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 399615fb1735..a1d39ca87ab7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -27484,7 +27484,7 @@ mod tests { } #[test] - fn bulk_surface_close_uses_one_shared_termination_deadline() { + fn server_shutdown_uses_bounded_parallel_termination_fanout() { let mux = test_mux(); let first = mux.new_workspace(None, Some((80, 24))).unwrap(); let pane = mux.with_state(|state| state.pane_of(first.id).unwrap()); @@ -27504,7 +27504,10 @@ mod tests { } let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); - let close = std::thread::spawn(move || result_tx.send(mux.close_pane(pane)).unwrap()); + let close_mux = mux.clone(); + let close = std::thread::spawn(move || { + result_tx.send(close_mux.close_all_surfaces_for_shutdown()).unwrap() + }); assert!( shutdown_gate.wait_until_entered( SHUTDOWN_FANOUT_WORKERS, @@ -27513,8 +27516,18 @@ mod tests { "bulk close did not use the bounded parallel shutdown fanout" ); shutdown_gate.release(); - assert!(result_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap()); + let error = result_rx + .recv_timeout(Duration::from_secs(1)) + .unwrap() + .expect_err("gated terminal owners unexpectedly completed shutdown"); + assert!(error.to_string().contains("could not terminate 64 surface process")); close.join().unwrap(); + + for surface in &surfaces { + surface.set_server_shutdown_failure_for_test(false); + } + mux.close_all_surfaces_for_shutdown().unwrap(); + assert!(mux.shutdown_owners.is_empty()); } #[cfg(unix)] From e322e8f166c894e72d077739d2a19f64b9a0f472 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:00:13 -0700 Subject: [PATCH 415/521] fix(macOS): lock recovery scheduler state --- ...erminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index c2a88490acaf..785a387f8097 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -33,7 +33,8 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } } - private struct State { + // Safety: this state never escapes `state`'s OSAllocatedUnfairLock. + private struct State: @unchecked Sendable { var nextSequence: UInt64 = 0 // Every indexed entry is one linked FIFO node. Synchronous cancellation // removes both views under this lock, so no weak tombstone survives. From 6304e4f91a2d890abfbf278ac546c2db20717362 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:07:51 -0700 Subject: [PATCH 416/521] fix(macOS): type teardown gate continuation --- .../Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift index c3ee3c86800e..2d6e857554d3 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownStartGate.swift @@ -36,7 +36,7 @@ final class TerminalSurfaceRuntimeTeardownStartGate: @unchecked Sendable { } func start() { - let continuation = state.withLock { state in + let continuation: CheckedContinuation? = state.withLock { state in switch state { case .pending: state = .started From 001c62d17094362c3d14e6218b7bbf93c52c1216 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:15:34 -0700 Subject: [PATCH 417/521] fix(macOS): isolate recovery scheduler lock state --- ...timeOwnershipRecoveryRescanScheduler.swift | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 785a387f8097..3824fa2d7d0e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -33,8 +33,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } } - // Safety: this state never escapes `state`'s OSAllocatedUnfairLock. - private struct State: @unchecked Sendable { + private struct State { var nextSequence: UInt64 = 0 // Every indexed entry is one linked FIFO node. Synchronous cancellation // removes both views under this lock, so no weak tombstone survives. @@ -46,7 +45,10 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var rescanTask: Task? } - private let state = OSAllocatedUnfairLock(initialState: State()) + // Safety: this lock owns all mutable links and task state. The weak, + // non-Sendable surface references are dereferenced only by the MainActor + // batch in `runScheduledBatch()`. + private let state = OSAllocatedUnfairLock(uncheckedState: State()) internal init(maximumEntryCount: Int) { precondition(maximumEntryCount > 0) @@ -62,7 +64,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: tailID: UUID? ) { - state.withLock { state in + state.withLockUnchecked { state in var linkedIDs = Set() var currentID = state.headID while let nodeID = currentID, linkedIDs.insert(nodeID).inserted { @@ -83,7 +85,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: surface: TerminalSurface ) -> TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration { var startGate: TerminalSurfaceRuntimeTeardownStartGate? - let registration = state.withLock { state in + let registration = state.withLockUnchecked { state in if let entry = state.entriesByID[surfaceID] { entry.surface = surface state.rescanRequested = true @@ -120,7 +122,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } internal func cancelOverflow(surfaceID: UUID) { - let removed = state.withLock { state in + let removed = state.withLockUnchecked { state in removeOverflow(surfaceID: surfaceID, from: &state) != nil } if removed { @@ -130,7 +132,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: internal func requestRescan() { var startGate: TerminalSurfaceRuntimeTeardownStartGate? - state.withLock { state in + state.withLockUnchecked { state in guard state.headID != nil else { return } state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) @@ -140,7 +142,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: internal func failPendingOverflowCreations() { var startGate: TerminalSurfaceRuntimeTeardownStartGate? - state.withLock { state in + state.withLockUnchecked { state in guard let tailID = state.tailID, let tail = state.entriesByID[tailID] else { return } @@ -155,7 +157,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } deinit { - state.withLock { state in + state.withLockUnchecked { state in state.rescanTask?.cancel() state.rescanTask = nil } @@ -176,14 +178,14 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: @MainActor private func runScheduledBatch() { - let shouldRun = state.withLock { state in + let shouldRun = state.withLockUnchecked { state in state.rescanRequested = false return state.headID != nil } var processedCount = 0 if shouldRun { while processedCount < Self.maximumBatchCount { - if let failedEntry = state.withLock({ state -> OverflowEntry? in + if let failedEntry = state.withLockUnchecked({ state -> OverflowEntry? in guard let cutoff = state.failureThroughSequence, let headID = state.headID, let entry = state.entriesByID[headID], @@ -202,14 +204,14 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: continue } guard - let entry = state.withLock({ state in + let entry = state.withLockUnchecked({ state in state.headID.flatMap { state.entriesByID[$0] } }) else { break } guard let surface = entry.surface else { - state.withLock { state in + state.withLockUnchecked { state in _ = removeOverflow( surfaceID: entry.surfaceID, from: &state @@ -230,14 +232,14 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: ) processedCount += 1 - let madeProgress = state.withLock { state in + let madeProgress = state.withLockUnchecked { state in state.entriesByID[entry.surfaceID] == nil } guard madeProgress else { break } } } - let followUp = state.withLock { + let followUp = state.withLockUnchecked { state -> (OverflowEntry, Bool, Bool)? in state.rescanTask = nil let externallyRequested = state.rescanRequested From a78d60e7ec4bd578ba3a6e2238480a66ca0e719f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:20:47 -0700 Subject: [PATCH 418/521] fix(macOS): type overflow registration result --- ...timeOwnershipRecoveryRescanScheduler.swift | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 3824fa2d7d0e..2aea9e960c83 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -85,38 +85,39 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: surface: TerminalSurface ) -> TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration { var startGate: TerminalSurfaceRuntimeTeardownStartGate? - let registration = state.withLockUnchecked { state in - if let entry = state.entriesByID[surfaceID] { - entry.surface = surface - state.rescanRequested = true - prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) - return .updated(sequence: entry.sequence) - } + let registration: TerminalSurfaceRuntimeOwnershipRecoveryOverflowRegistration = + state.withLockUnchecked { state in + if let entry = state.entriesByID[surfaceID] { + entry.surface = surface + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + return .updated(sequence: entry.sequence) + } - guard state.entriesByID.count < maximumEntryCount else { - return .rejected - } + guard state.entriesByID.count < maximumEntryCount else { + return .rejected + } - precondition(state.nextSequence < UInt64.max) - state.nextSequence += 1 - let previousID = state.tailID - let entry = OverflowEntry( - surfaceID: surfaceID, - sequence: state.nextSequence, - surface: surface, - previousID: previousID - ) - state.entriesByID[surfaceID] = entry - if let previousID { - state.entriesByID[previousID]?.nextID = surfaceID - } else { - state.headID = surfaceID + precondition(state.nextSequence < UInt64.max) + state.nextSequence += 1 + let previousID = state.tailID + let entry = OverflowEntry( + surfaceID: surfaceID, + sequence: state.nextSequence, + surface: surface, + previousID: previousID + ) + state.entriesByID[surfaceID] = entry + if let previousID { + state.entriesByID[previousID]?.nextID = surfaceID + } else { + state.headID = surfaceID + } + state.tailID = surfaceID + state.rescanRequested = true + prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) + return .registered(sequence: entry.sequence) } - state.tailID = surfaceID - state.rescanRequested = true - prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) - return .registered(sequence: entry.sequence) - } startGate?.start() return registration } From 0d4fbad71ad254d3b08bedd147311f39960ce658 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:37:16 -0700 Subject: [PATCH 419/521] fix(tui): satisfy shutdown fanout lint --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index a1d39ca87ab7..5c275e1d334f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -27506,7 +27506,7 @@ mod tests { let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1); let close_mux = mux.clone(); let close = std::thread::spawn(move || { - result_tx.send(close_mux.close_all_surfaces_for_shutdown()).unwrap() + result_tx.send(close_mux.close_all_surfaces_for_shutdown()).unwrap(); }); assert!( shutdown_gate.wait_until_entered( From 53f824c08c0af309bd0b478398e07894457f20d4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:57:53 -0700 Subject: [PATCH 420/521] test(macOS): bound stalled recovery failures --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 2290f27b8eba..cbee398cf380 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -179,6 +179,113 @@ private func requireTeardownTicket( #expect(weakCoordinator == nil) } + @Test func stalledCloseFailuresYieldMainActorAfterBoundedBatch() async throws { + let batchCount = 32 + let failureCount = batchCount + 1 + let clock = ManualTerminalSurfaceRuntimeTeardownClock() + let stalledSlots = AsyncStream.makeStream() + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: failureCount, + closeTeardownTimeout: .seconds(5), + closeTeardownClock: clock, + closeTeardownStalledObserver: { slot in + stalledSlots.continuation.yield(slot) + } + ) + let pointers = (0..<2).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = (0..<2).map { _ in DispatchSemaphore(value: 0) } + let recordedFailureCount = OSAllocatedUnfairLock(initialState: 0) + let failureEvents = AsyncStream.makeStream() + let mainActorMarker = AsyncStream.makeStream() + defer { + for releaseFree in releaseFrees { + releaseFree.signal() + } + for pointer in pointers { + pointer.deallocate() + } + freeStarted.continuation.finish() + stalledSlots.continuation.finish() + failureEvents.continuation.finish() + mainActorMarker.continuation.finish() + } + + let reservations = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let tickets = try pointers.enumerated().map { index, pointer in + try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.stalledFailureBatch.\(index)", + surface: pointer, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: reservations[index], + freeSurface: { _ in + freeStarted.continuation.yield(index) + releaseFrees[index].wait() + } + ) + ) + } + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = try #require(await freeStartedIterator.next()) + _ = try #require(await freeStartedIterator.next()) + var registrationIterator = clock.registrations.makeAsyncIterator() + let firstWatchdog = try #require(await registrationIterator.next()) + let secondWatchdog = try #require(await registrationIterator.next()) + + for _ in 0.. Date: Mon, 10 Aug 2026 20:00:28 -0700 Subject: [PATCH 421/521] fix(macOS): batch stalled recovery failures --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 23 ++++ ...urfaceRuntimeOwnershipAdmissionState.swift | 1 + ...RuntimeOwnershipRecoveryFailureDrain.swift | 101 ++++++++++++++++++ ...nalSurfaceRuntimeTeardownCoordinator.swift | 24 +++-- 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index cd742cac4285..9c64fe7d0f0d 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -267,11 +267,33 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { failures.append(failure) _ = removeRecoveryAction(recoveryID, from: &state) } + state.pendingRecoveryFailureCount += failures.count + precondition( + state.pendingRecoveryFailureCount <= maximumOwnerCount, + "stalled recovery failures must remain bounded" + ) state.recoveryRescanRequested = false return failures } } + func completeStalledCloseRecoveryFailures(_ completedCount: Int) { + precondition(completedCount > 0) + let output = state.withLock { state in + precondition( + completedCount <= state.pendingRecoveryFailureCount, + "completed stalled recovery failures must be owned" + ) + state.pendingRecoveryFailureCount -= completedCount + let grant = takeNextRecoveryGrant(from: &state) + return (grant, takeRecoveryRescanRequest(from: &state)) + } + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() + } + } + func clearAllStalledCloseTeardowns() { let output: ( grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant?, @@ -350,6 +372,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) -> Bool { state.recoveryEntriesByID.count + state.recoveryCapacityReservationIDs.count + + state.pendingRecoveryFailureCount < maximumOwnerCount } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index ebd6a1b83f97..f5f46bc3daed 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -13,6 +13,7 @@ struct TerminalSurfaceRuntimeOwnershipAdmissionState { var recoveryCapacityReservationIDs: Set = [] var recoveryHeadID: UUID? var recoveryTailID: UUID? + var pendingRecoveryFailureCount = 0 var recoveryGrantIsScheduled = false var recoveryRescanRequested = false } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift new file mode 100644 index 000000000000..9177584b959d --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift @@ -0,0 +1,101 @@ +internal import os + +/// Delivers stalled native-creation failures in bounded MainActor batches. +/// +/// Admission keeps each queued callback counted against its bounded recovery +/// capacity until this drain delivers it. One transient task owns each batch; +/// idle state has no task cycle. +internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: + @unchecked Sendable +{ + private static let maximumBatchCount = 32 + + private struct State { + var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] = [] + var nextFailureIndex = 0 + var task: Task? + } + + private let maximumFailureCount: Int + private let admission: TerminalSurfaceRuntimeOwnershipAdmission + private let state = OSAllocatedUnfairLock(initialState: State()) + + internal init( + maximumFailureCount: Int, + admission: TerminalSurfaceRuntimeOwnershipAdmission + ) { + precondition(maximumFailureCount > 0) + self.maximumFailureCount = maximumFailureCount + self.admission = admission + } + + internal func enqueue( + _ failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] + ) { + guard !failures.isEmpty else { return } + var startGate: TerminalSurfaceRuntimeTeardownStartGate? + state.withLock { state in + let pendingCount = state.failures.count - state.nextFailureIndex + precondition( + pendingCount + failures.count <= maximumFailureCount, + "stalled recovery failure drain must remain bounded" + ) + if pendingCount == 0 { + state.failures.removeAll(keepingCapacity: true) + state.nextFailureIndex = 0 + } + state.failures.append(contentsOf: failures) + prepareTaskIfNeeded(state: &state, startGate: &startGate) + } + startGate?.start() + } + + deinit { + state.withLock { state in + state.task?.cancel() + state.task = nil + } + } + + private func prepareTaskIfNeeded( + state: inout State, + startGate: inout TerminalSurfaceRuntimeTeardownStartGate? + ) { + guard state.task == nil else { return } + let gate = TerminalSurfaceRuntimeTeardownStartGate() + state.task = Task { @MainActor [self] in + await gate.wait() + runScheduledBatch() + } + startGate = gate + } + + @MainActor + private func runScheduledBatch() { + let batch = state.withLock { state in + let endIndex = min( + state.nextFailureIndex + Self.maximumBatchCount, + state.failures.count + ) + let batch = Array(state.failures[state.nextFailureIndex.. Date: Mon, 10 Aug 2026 20:13:40 -0700 Subject: [PATCH 422/521] fix(macOS): isolate failure drain lock state --- ...SurfaceRuntimeOwnershipRecoveryFailureDrain.swift | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift index 9177584b959d..cb69f3468a06 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift @@ -18,7 +18,9 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: private let maximumFailureCount: Int private let admission: TerminalSurfaceRuntimeOwnershipAdmission - private let state = OSAllocatedUnfairLock(initialState: State()) + // Safety: this lock owns all mutable failure and task state. Failure + // callbacks leave the lock before they run on the MainActor. + private let state = OSAllocatedUnfairLock(uncheckedState: State()) internal init( maximumFailureCount: Int, @@ -34,7 +36,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: ) { guard !failures.isEmpty else { return } var startGate: TerminalSurfaceRuntimeTeardownStartGate? - state.withLock { state in + state.withLockUnchecked { state in let pendingCount = state.failures.count - state.nextFailureIndex precondition( pendingCount + failures.count <= maximumFailureCount, @@ -51,7 +53,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: } deinit { - state.withLock { state in + state.withLockUnchecked { state in state.task?.cancel() state.task = nil } @@ -72,7 +74,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: @MainActor private func runScheduledBatch() { - let batch = state.withLock { state in + let batch = state.withLockUnchecked { state in let endIndex = min( state.nextFailureIndex + Self.maximumBatchCount, state.failures.count @@ -87,7 +89,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: admission.completeStalledCloseRecoveryFailures(batch.count) var startGate: TerminalSurfaceRuntimeTeardownStartGate? - state.withLock { state in + state.withLockUnchecked { state in state.task = nil if state.nextFailureIndex == state.failures.count { state.failures.removeAll(keepingCapacity: true) From 9ae05b74be82e6b67b85d6b5d394339f1cc984ff Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:45:45 -0700 Subject: [PATCH 423/521] test(tui): signal CDP reader shutdown (#9926) --- cmux-tui/crates/cmux-tui-cdp/src/client.rs | 49 +++++++++++++++------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/client.rs b/cmux-tui/crates/cmux-tui-cdp/src/client.rs index e40d23261387..cfae92c25f99 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/client.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/client.rs @@ -95,6 +95,29 @@ impl ResolverTestGate { } } +#[cfg(test)] +#[derive(Default)] +struct ReaderStoppedSignal { + stopped: Mutex, + changed: Condvar, +} + +#[cfg(test)] +impl ReaderStoppedSignal { + fn mark(&self) { + let mut stopped = self.stopped.lock().unwrap(); + *stopped = true; + self.changed.notify_all(); + } + + fn wait(&self, timeout: Duration) -> bool { + let stopped = self.stopped.lock().unwrap(); + let (stopped, _) = + self.changed.wait_timeout_while(stopped, timeout, |stopped| !*stopped).unwrap(); + *stopped + } +} + #[cfg(test)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ResolverReaperTestEvent { @@ -863,7 +886,7 @@ struct Inner { web_socket_url: Arc, peer_addr: SocketAddr, #[cfg(test)] - reader_stopped: Arc, + reader_stopped: Arc, } struct PendingCall { @@ -1362,7 +1385,7 @@ impl CdpClient { web_socket_url: Arc::from(web_socket_url), peer_addr: addr, #[cfg(test)] - reader_stopped: Arc::new(AtomicBool::new(false)), + reader_stopped: Arc::new(ReaderStoppedSignal::default()), }), }; client.spawn_reader(ws, outbound_rx, events)?; @@ -1381,7 +1404,7 @@ impl CdpClient { std::thread::Builder::new().name("cmux-tui-cdp-reader".into()).spawn(move || { reader_loop(&weak, ws, &outbound, &event_output); #[cfg(test)] - reader_stopped.store(true, Ordering::Release); + reader_stopped.mark(); })?; Ok(()) } @@ -3064,7 +3087,7 @@ mod tests { timeout: Duration::from_secs(1), web_socket_url: Arc::from("ws://127.0.0.1:1/devtools/browser/test"), peer_addr: "127.0.0.1:1".parse().unwrap(), - reader_stopped: Arc::new(AtomicBool::new(false)), + reader_stopped: Arc::new(ReaderStoppedSignal::default()), }), outbound_rx, ) @@ -4560,11 +4583,10 @@ mod tests { ) .unwrap(); drop(command); - let close_deadline = Instant::now() + Duration::from_secs(1); - while !client.inner.reader_stopped.load(Ordering::Acquire) { - assert!(Instant::now() < close_deadline, "initial CDP reader did not stop"); - thread::yield_now(); - } + assert!( + client.inner.reader_stopped.wait(Duration::from_secs(1)), + "initial CDP reader did not stop" + ); let unused_gate = Arc::new(ResolverTestGate::default()); *NEXT_RESOLVE_GATE.lock().unwrap() = Some(unused_gate.clone()); @@ -5138,11 +5160,10 @@ mod tests { CdpClient::connect(&format!("ws://{addr}/devtools/browser/fake"), event_tx).unwrap(); server.join().unwrap(); - let deadline = Instant::now() + Duration::from_millis(500); - while !client.inner.reader_stopped.load(Ordering::Acquire) { - assert!(Instant::now() < deadline, "CDP reader did not stop"); - thread::yield_now(); - } + assert!( + client.inner.reader_stopped.wait(Duration::from_millis(500)), + "CDP reader did not stop" + ); assert!(matches!(event_rx.recv(), Ok(CdpEvent::Other { .. }))); let shutdown = event_rx.recv_timeout(Duration::from_millis(500)); From ada14a9ea47de66c5c104725f4a8fe859240b631 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:56:30 -0700 Subject: [PATCH 424/521] test(tui): cover durable exit cleanup race --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 51 ++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5c275e1d334f..227ecde2ea9e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -20254,6 +20254,57 @@ mod tests { assert!(format!("{error:#}").contains("terminal-host")); } + #[cfg(unix)] + #[test] + fn prepared_terminal_host_cleanup_acknowledges_exit_after_record_disappears() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let mux = test_mux(); + let root = std::env::temp_dir().join(format!( + "cmux-prepared-host-exit-cleanup-{}", + crate::workspace_registry::new_uuid_v4() + )); + std::fs::create_dir_all(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + let terminal_id = TerminalId::random().unwrap().to_hex(); + let incarnation = TerminalId::random().unwrap().to_hex(); + let uid = std::fs::metadata(&root).unwrap().uid(); + let record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 4, + terminal_id: terminal_id.clone(), + incarnation: incarnation.clone(), + endpoint: format!("/tmp/cmux-th-{uid}/{terminal_id}.sock"), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: std::process::id(), + host_start_nonce: "02".repeat(32), + workspace_key: String::new(), + supports_set_defaults: true, + supports_terminate_only: true, + supports_clear_history: true, + supports_terminate_ack: true, + }; + let record_path = record.record_path(&root); + let exit_record = crate::terminal_host_runtime::TerminalHostExitRecord::new( + &TerminalHostIdentity { terminal_id, incarnation }, + TerminalExit { + outcome: crate::terminal_host_protocol::TerminalExitOutcome::Signal { + signal: libc::SIGHUP, + core_dumped: false, + }, + exited_at_ms: 1_234_567, + }, + ); + let exit_path = record_path.with_extension("exit"); + std::fs::write(&exit_path, serde_json::to_vec(&exit_record).unwrap()).unwrap(); + std::fs::set_permissions(&exit_path, std::fs::Permissions::from_mode(0o600)).unwrap(); + + mux.terminate_prepared_terminal_hosts(vec![(record_path, record)]).unwrap(); + + let exit_remained = exit_path.exists(); + let _ = std::fs::remove_dir_all(root); + assert!(!exit_remained, "post-commit host cleanup retained its exact exit sidecar"); + } + #[cfg(unix)] #[test] fn workspace_close_reports_exact_host_record_read_failure() { From 5b4bf8532ce8333e8b741a157c5825fcd6ebb49b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:57:14 -0700 Subject: [PATCH 425/521] fix(tui): acknowledge prepared host exits --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 227ecde2ea9e..ddd31ae00c28 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9387,6 +9387,21 @@ impl Mux { "could not clean up terminal host {}", record.terminal_id ); + if let Some((exit_path, exit)) = + crate::terminal_host_runtime::terminal_host_exit_record(&path)? + { + anyhow::ensure!( + exit.terminal_id == record.terminal_id + && exit.incarnation == record.incarnation, + "terminal-host exit identity changed while closing {}", + record.terminal_id + ); + anyhow::ensure!( + acknowledge_exact_terminal_host_exit(&exit_path, &exit), + "could not acknowledge terminal-host exit {}", + record.terminal_id + ); + } } Ok(()) } From d52439cdcc8e9e4b91df09124bf4657cffa9eb25 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:58:59 -0700 Subject: [PATCH 426/521] test(tui): cover detached cleanup identity (#9945) --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 85 ++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 5c275e1d334f..95b9c205a813 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -29715,6 +29715,91 @@ mod tests { std::fs::remove_dir_all(root).unwrap(); } + #[cfg(unix)] + #[test] + fn detached_terminal_close_rejects_stale_cleanup_identity_before_commit() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + const TERMINAL: &str = "0000000000004000800000000000000d"; + const INCARNATION: &str = "1000000000004000800000000000000d"; + const OTHER_INCARNATION: &str = "1000000000004000800000000000000e"; + let root = std::env::temp_dir().join(format!( + "cmux-mux-detached-cleanup-{}", + crate::workspace_registry::new_uuid_v4() + )); + let host_root = crate::terminal_host_runtime::terminal_host_root(&root, "detached-cleanup"); + let options = SurfaceOptions { + terminal_host_root: Some(host_root.clone()), + ..SurfaceOptions::default() + }; + let mux = Mux::new_for_test("detached-cleanup", options); + let workspace = + mux.create_empty_workspace(Some("detached-cleanup".into()), None, None).unwrap(); + let surface = + mux.seed_running_terminal_for_test(TERMINAL, INCARNATION, &workspace.key).unwrap(); + let terminal = + mux.workspace_registry.lock().unwrap().terminal_resource_id(TERMINAL).unwrap().unwrap(); + assert!( + mux.persist_terminal_exit_for_test( + &terminal, + &TerminalExit { + outcome: crate::terminal_host_protocol::TerminalExitOutcome::Exit { code: 0 }, + exited_at_ms: 1, + }, + ) + .unwrap() + ); + mux.surface_exited(surface); + let detached = mux.resolve_terminal(TERMINAL).unwrap().unwrap(); + assert_eq!(detached.surface, None); + assert_eq!(detached.terminal.lifecycle, TerminalLifecycle::Exited); + + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = true; + std::fs::create_dir_all(&host_root).unwrap(); + let uid = std::fs::metadata(&host_root).unwrap().uid(); + let host_record = crate::terminal_host_runtime::TerminalHostRecord { + record_version: 1, + terminal_id: TERMINAL.into(), + incarnation: OTHER_INCARNATION.into(), + endpoint: format!("/tmp/cmux-th-{uid}/{TERMINAL}.sock"), + owner_token: "01".repeat(crate::terminal_host::CAPABILITY_TOKEN_LEN), + host_pid: 0, + host_start_nonce: String::new(), + workspace_key: workspace.key, + supports_set_defaults: false, + supports_terminate_only: false, + supports_terminate_ack: false, + supports_clear_history: false, + }; + let host_record_path = host_root.join(format!("{TERMINAL}.json")); + std::fs::write(&host_record_path, serde_json::to_vec(&host_record).unwrap()).unwrap(); + let mut permissions = std::fs::metadata(&host_record_path).unwrap().permissions(); + permissions.set_mode(0o600); + std::fs::set_permissions(&host_record_path, permissions).unwrap(); + + let error = mux + .close_terminal(TERMINAL, INCARNATION) + .expect_err("detached terminal close accepted a different host incarnation"); + assert!(format!("{error:#}").contains("terminal-host incarnation changed")); + assert_eq!( + mux.resolve_terminal(TERMINAL).unwrap().unwrap().terminal.lifecycle, + TerminalLifecycle::Exited, + "rejected host cleanup changed the detached terminal lifecycle" + ); + + std::fs::remove_file(host_record_path).unwrap(); + let closed = mux.close_terminal(TERMINAL, INCARNATION).unwrap(); + assert_eq!(closed.surface, None); + assert_eq!( + mux.resolve_terminal(TERMINAL).unwrap().unwrap().terminal.lifecycle, + TerminalLifecycle::Tombstoned + ); + mux.terminal_adoption_coordinator.state.lock().unwrap().stopping = false; + mux.shutdown().unwrap(); + drop(mux); + std::fs::remove_dir_all(root).unwrap(); + } + #[cfg(unix)] #[test] fn hosted_terminal_exit_removes_sole_surface_but_preserves_exact_wait_result() { From 06ab13d57262a8d370854690a6f1b1979cea54e5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:00:51 -0700 Subject: [PATCH 427/521] test(macOS): cover stalled close recovery order --- ...faceRuntimeTeardownCoordinator+Debug.swift | 2 +- ...nalSurfaceRuntimeTeardownCoordinator.swift | 13 ++++++++-- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 24 ++++++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift index 176012840586..a1ef85c01f99 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator+Debug.swift @@ -20,7 +20,7 @@ extension TerminalSurfaceRuntimeTeardownCoordinator { } /// Test support: both close workers exceeded their watchdog deadlines. - public var debugCloseTeardownAllStalled: Bool { + public nonisolated var debugCloseTeardownAllStalled: Bool { runtimeOwnershipAdmission.debugCloseTeardownAllStalled } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 061df12a2a28..7a8d3a46aef7 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -33,6 +33,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { private let closeTeardownTimeout: Duration private nonisolated let closeTeardownClock: any Clock private let closeTeardownStalledObserver: @Sendable (Int) -> Void + private let closeTeardownRecoveredObserver: @Sendable (Int) -> Void private nonisolated let submissionDrain: TerminalSurfaceRuntimeTeardownSubmissionDrain private nonisolated let recoveryRescanScheduler: @@ -73,6 +74,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { closeTeardownTimeout = .seconds(5) closeTeardownClock = ContinuousClock() closeTeardownStalledObserver = { _ in } + closeTeardownRecoveredObserver = { _ in } let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: Self.maximumRuntimeSurfaceOwnerCount @@ -101,12 +103,14 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { maximumRuntimeSurfaceOwnerCount: Int, closeTeardownTimeout: Duration = .seconds(5), closeTeardownClock: any Clock = ContinuousClock(), - closeTeardownStalledObserver: @escaping @Sendable (Int) -> Void = { _ in } + closeTeardownStalledObserver: @escaping @Sendable (Int) -> Void = { _ in }, + closeTeardownRecoveredObserver: @escaping @Sendable (Int) -> Void = { _ in } ) { precondition(closeTeardownTimeout > .zero) self.closeTeardownTimeout = closeTeardownTimeout self.closeTeardownClock = closeTeardownClock self.closeTeardownStalledObserver = closeTeardownStalledObserver + self.closeTeardownRecoveredObserver = closeTeardownRecoveredObserver let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: maximumRuntimeSurfaceOwnerCount @@ -569,14 +573,19 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { guard active?.ticketID == request.ticketID else { return } active?.watchdogTask?.cancel() + var recoveredStalledCloseSlot = false switch executionLane { case .boundedClose: activeCloseTeardownsBySlot.removeValue(forKey: executionSlot) - stalledCloseExecutionSlots.remove(executionSlot) + recoveredStalledCloseSlot = stalledCloseExecutionSlots.remove(executionSlot) != nil case .isolatedHibernation: activeHibernationTeardownsBySlot.removeValue(forKey: executionSlot) } + if recoveredStalledCloseSlot { + closeTeardownRecoveredObserver(executionSlot) + } + await finishFree(request) complete(requestID: request.ticketID) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 50ed4739b6bf..f4b4677641e2 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -637,12 +637,25 @@ import CmuxTerminalCore @Test func stalledCloseWorkersFailDeferredCreationAndRecoverSafely() async throws { let clock = ManualTerminalSurfaceRuntimeTeardownClock() let stalledSlots = AsyncStream.makeStream() + let recoveredSlots = AsyncStream.makeStream() + let releaseFirstRecoveredSlot = DispatchSemaphore(value: 0) + let shouldBlockRecoveredSlot = OSAllocatedUnfairLock(initialState: true) let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2, closeTeardownTimeout: .seconds(5), closeTeardownClock: clock, closeTeardownStalledObserver: { slot in stalledSlots.continuation.yield(slot) + }, + closeTeardownRecoveredObserver: { slot in + recoveredSlots.continuation.yield(slot) + let shouldBlock = shouldBlockRecoveredSlot.withLock { shouldBlock in + defer { shouldBlock = false } + return shouldBlock + } + if shouldBlock { + releaseFirstRecoveredSlot.wait() + } } ) let pointers = (0..<2).map { _ in @@ -660,6 +673,8 @@ import CmuxTerminalCore } freeStarted.continuation.finish() stalledSlots.continuation.finish() + recoveredSlots.continuation.finish() + releaseFirstRecoveredSlot.signal() } let reservations = try (0..<2).map { _ in try #require(coordinator.reserveRuntimeSurfaceOwnership()) @@ -753,7 +768,14 @@ import CmuxTerminalCore == expectedMessage ) + var recoveredSlotIterator = recoveredSlots.stream.makeAsyncIterator() releaseFrees[0].signal() + _ = try #require(await recoveredSlotIterator.next()) + #expect( + !coordinator.debugCloseTeardownAllStalled, + "returned close worker retained the all-stalled admission failure" + ) + releaseFirstRecoveredSlot.signal() #expect(await tickets[0].wait(timeout: nil)) #expect( freedPointerBits.withLock { $0 } @@ -763,7 +785,7 @@ import CmuxTerminalCore coordinator.reserveRuntimeSurfaceOwnership() ) coordinator.cancelRuntimeSurfaceOwnership(recoveredReservation) - #expect(await !coordinator.debugCloseTeardownAllStalled) + #expect(!coordinator.debugCloseTeardownAllStalled) releaseFrees[1].signal() #expect(await tickets[1].wait(timeout: nil)) From 51a3e447865e5111d87c60861957214f62ef7ad4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:01:52 -0700 Subject: [PATCH 428/521] fix(macOS): clear stalled admission before cleanup --- .../Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 7a8d3a46aef7..e95a8f90d7aa 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -583,6 +583,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { } if recoveredStalledCloseSlot { + updateCloseTeardownAdmission() closeTeardownRecoveredObserver(executionSlot) } From dfdc00e4a6f69a637c6337fea56d96cad16de997 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:13:45 -0700 Subject: [PATCH 429/521] test(macOS): gate stalled recovery at cleanup --- ...nalSurfaceRuntimeTeardownCoordinator.swift | 7 +--- .../FakeTerminalByteTeeLease.swift | 10 ++++- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 38 +++++++++---------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index e95a8f90d7aa..248770fdad4b 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -33,7 +33,6 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { private let closeTeardownTimeout: Duration private nonisolated let closeTeardownClock: any Clock private let closeTeardownStalledObserver: @Sendable (Int) -> Void - private let closeTeardownRecoveredObserver: @Sendable (Int) -> Void private nonisolated let submissionDrain: TerminalSurfaceRuntimeTeardownSubmissionDrain private nonisolated let recoveryRescanScheduler: @@ -74,7 +73,6 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { closeTeardownTimeout = .seconds(5) closeTeardownClock = ContinuousClock() closeTeardownStalledObserver = { _ in } - closeTeardownRecoveredObserver = { _ in } let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: Self.maximumRuntimeSurfaceOwnerCount @@ -103,14 +101,12 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { maximumRuntimeSurfaceOwnerCount: Int, closeTeardownTimeout: Duration = .seconds(5), closeTeardownClock: any Clock = ContinuousClock(), - closeTeardownStalledObserver: @escaping @Sendable (Int) -> Void = { _ in }, - closeTeardownRecoveredObserver: @escaping @Sendable (Int) -> Void = { _ in } + closeTeardownStalledObserver: @escaping @Sendable (Int) -> Void = { _ in } ) { precondition(closeTeardownTimeout > .zero) self.closeTeardownTimeout = closeTeardownTimeout self.closeTeardownClock = closeTeardownClock self.closeTeardownStalledObserver = closeTeardownStalledObserver - self.closeTeardownRecoveredObserver = closeTeardownRecoveredObserver let recoveryRescanScheduler = TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler( maximumEntryCount: maximumRuntimeSurfaceOwnerCount @@ -584,7 +580,6 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { if recoveredStalledCloseSlot { updateCloseTeardownAdmission() - closeTeardownRecoveredObserver(executionSlot) } await finishFree(request) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalByteTeeLease.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalByteTeeLease.swift index fb7e892afeec..eab05fde9868 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalByteTeeLease.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalByteTeeLease.swift @@ -1,5 +1,13 @@ @testable import CmuxTerminal final class FakeTerminalByteTeeLease: TerminalByteTeeLease { - func release() {} + private let onRelease: @Sendable () -> Void + + init(onRelease: @escaping @Sendable () -> Void = {}) { + self.onRelease = onRelease + } + + func release() { + onRelease() + } } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index f4b4677641e2..0a5dd1f1e8a6 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -637,25 +637,14 @@ import CmuxTerminalCore @Test func stalledCloseWorkersFailDeferredCreationAndRecoverSafely() async throws { let clock = ManualTerminalSurfaceRuntimeTeardownClock() let stalledSlots = AsyncStream.makeStream() - let recoveredSlots = AsyncStream.makeStream() - let releaseFirstRecoveredSlot = DispatchSemaphore(value: 0) - let shouldBlockRecoveredSlot = OSAllocatedUnfairLock(initialState: true) + let finishFreeStarted = DispatchSemaphore(value: 0) + let releaseFinishFree = DispatchSemaphore(value: 0) let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2, closeTeardownTimeout: .seconds(5), closeTeardownClock: clock, closeTeardownStalledObserver: { slot in stalledSlots.continuation.yield(slot) - }, - closeTeardownRecoveredObserver: { slot in - recoveredSlots.continuation.yield(slot) - let shouldBlock = shouldBlockRecoveredSlot.withLock { shouldBlock in - defer { shouldBlock = false } - return shouldBlock - } - if shouldBlock { - releaseFirstRecoveredSlot.wait() - } } ) let pointers = (0..<2).map { _ in @@ -673,13 +662,18 @@ import CmuxTerminalCore } freeStarted.continuation.finish() stalledSlots.continuation.finish() - recoveredSlots.continuation.finish() - releaseFirstRecoveredSlot.signal() + releaseFinishFree.signal() } let reservations = try (0..<2).map { _ in try #require(coordinator.reserveRuntimeSurfaceOwnership()) } let tickets = try pointers.enumerated().map { index, pointer in + let byteTeeLease: (any TerminalByteTeeLease)? = index == 0 + ? FakeTerminalByteTeeLease { + finishFreeStarted.signal() + releaseFinishFree.wait() + } + : nil try #require( coordinator.enqueueRuntimeTeardown( id: UUID(), @@ -688,7 +682,7 @@ import CmuxTerminalCore surface: pointer, callbackContext: nil, manualIOContext: nil, - byteTeeLease: nil, + byteTeeLease: byteTeeLease, runtimeOwnershipReservation: reservations[index], freeSurface: { pointer in freeStarted.continuation.yield(index) @@ -768,14 +762,18 @@ import CmuxTerminalCore == expectedMessage ) - var recoveredSlotIterator = recoveredSlots.stream.makeAsyncIterator() + let stalledStateAfterFree = Task.detached { + finishFreeStarted.wait() + let stalled = coordinator.debugCloseTeardownAllStalled + releaseFinishFree.signal() + return stalled + } releaseFrees[0].signal() - _ = try #require(await recoveredSlotIterator.next()) + let remainedAllStalled = await stalledStateAfterFree.value #expect( - !coordinator.debugCloseTeardownAllStalled, + !remainedAllStalled, "returned close worker retained the all-stalled admission failure" ) - releaseFirstRecoveredSlot.signal() #expect(await tickets[0].wait(timeout: nil)) #expect( freedPointerBits.withLock { $0 } From 3e19143c0b7acfe2a64b41bfbf2db78fafa93e3e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:49:31 -0700 Subject: [PATCH 430/521] test(tui): order final cleanup retry --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7dc71deae127..53be23bbba45 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26444,6 +26444,7 @@ mod tests { .unwrap(); let (second_failing, _) = second.set_recovering_server_shutdown_for_test(); let injected = Arc::new(AtomicBool::new(false)); + let (injected_tx, injected_rx) = std::sync::mpsc::sync_channel(1); *mux.shutdown_owner_reconciler.after_attempt.lock().unwrap() = Some(Arc::new({ let mux = mux.clone(); let injected = injected.clone(); @@ -26456,15 +26457,21 @@ mod tests { mux.retire_surface_runtime(second.clone()); first_failing.store(false, Ordering::Release); second_failing.store(false, Ordering::Release); + let _ = injected_tx.send(()); } })); + let deadline = Instant::now() + Duration::from_secs(2); assert!(mux.close_surface(first.id).unwrap()); - let completed = mux - .shutdown_owner_reconciler - .wait_until(Instant::now() + Duration::from_secs(2), || { - injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty() - }); + let remaining = deadline + .checked_duration_since(Instant::now()) + .expect("final cleanup attempt exceeded its failure deadline"); + injected_rx + .recv_timeout(remaining) + .expect("final cleanup attempt did not stage the second owner"); + let completed = mux.shutdown_owner_reconciler.wait_until(deadline, || { + injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty() + }); let reconciled = completed && injected.load(Ordering::Acquire) && mux.shutdown_owners.is_empty(); if !reconciled { From 44cb4a9ee459b51c48e60ed67f24c0d9909df720 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:13:25 -0700 Subject: [PATCH 431/521] test(macOS): cover recovery failure handoff --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 0a5dd1f1e8a6..3322e6c16ab2 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -794,6 +794,86 @@ import CmuxTerminalCore #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) } + @Test func completedFailureBatchRescansOverflowAndIgnoresStaleFailure() async throws { + let capacity = 34 + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: capacity + ) + let owners = try (0.. Date: Mon, 10 Aug 2026 22:15:13 -0700 Subject: [PATCH 432/521] fix(macOS): order recovery failure handoff --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 9 ++++++- .../TerminalSurface+RuntimeLifecycle.swift | 25 ++++++++++++++++--- .../Surface/TerminalSurface.swift | 1 + 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 9c64fe7d0f0d..618fbf00de49 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -284,9 +284,16 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { completedCount <= state.pendingRecoveryFailureCount, "completed stalled recovery failures must be owned" ) + let capacityWasOpen = recoveryCapacityIsOpen(in: state) state.pendingRecoveryFailureCount -= completedCount + let recoveryCapacityOpened = + !capacityWasOpen && recoveryCapacityIsOpen(in: state) let grant = takeNextRecoveryGrant(from: &state) - return (grant, takeRecoveryRescanRequest(from: &state)) + return ( + grant, + takeRecoveryRescanRequest(from: &state) + || recoveryCapacityOpened + ) } schedule(output.0) if output.1 { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 662849c21626..e389bd5f4f11 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -762,10 +762,18 @@ extension TerminalSurface { } @MainActor - private func runtimeSurfaceOwnershipRecoveryFailure() + private func runtimeSurfaceOwnershipRecoveryFailure( + generation: UInt64 + ) -> TerminalSurfaceRuntimeOwnershipRecoveryFailure { { [weak self] in - self?.failRuntimeSurfaceCreationForTeardownCapacity() + guard let self, + self.runtimeSurfaceAdmissionCreationGeneration == generation, + self.surface == nil, + self.runtimeSurfaceAdmissionDeferredCreationSource != nil else { + return + } + self.failRuntimeSurfaceCreationForTeardownCapacity() } } @@ -774,6 +782,9 @@ extension TerminalSurface { view: any TerminalSurfaceNativeViewing, source: RuntimeSurfaceCreationSource ) -> TerminalSurfaceRuntimeOwnershipReservation? { + precondition(runtimeSurfaceAdmissionCreationGeneration < UInt64.max) + runtimeSurfaceAdmissionCreationGeneration += 1 + let creationGeneration = runtimeSurfaceAdmissionCreationGeneration runtimeSurfaceAdmissionDeferredCreationSource = ( runtimeSurfaceAdmissionDeferredCreationSource @@ -787,7 +798,9 @@ extension TerminalSurface { onRecovery: runtimeSurfaceOwnershipRecovery( teardownCoordinator: teardownCoordinator ), - onFailure: runtimeSurfaceOwnershipRecoveryFailure() + onFailure: runtimeSurfaceOwnershipRecoveryFailure( + generation: creationGeneration + ) ) switch admissionResult { case .reserved(let reservation): @@ -849,7 +862,9 @@ extension TerminalSurface { onRecovery: runtimeSurfaceOwnershipRecovery( teardownCoordinator: teardownCoordinator ), - onFailure: runtimeSurfaceOwnershipRecoveryFailure(), + onFailure: runtimeSurfaceOwnershipRecoveryFailure( + generation: runtimeSurfaceAdmissionCreationGeneration + ), capacityReservation: capacityReservation ) switch admissionResult { @@ -904,6 +919,8 @@ extension TerminalSurface { runtimeTeardown.cancelRuntimeSurfaceOwnershipRecovery(id) runtimeTeardown .cancelRuntimeSurfaceOwnershipRecoveryOverflow(surfaceID: id) + precondition(runtimeSurfaceAdmissionCreationGeneration < UInt64.max) + runtimeSurfaceAdmissionCreationGeneration += 1 runtimeSurfaceAdmissionOverflowSequence = nil runtimeSurfaceAdmissionDeferredCreationSource = nil runtimeSurfaceAdmissionDeferredCreationView = nil diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift index 99318cb342c7..264a96d70336 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface.swift @@ -295,6 +295,7 @@ public final class TerminalSurface: Identifiable, ObservableObject { weak var runtimeSurfaceAdmissionDeferredCreationView: (any TerminalSurfaceNativeViewing)? var runtimeSurfaceAdmissionOverflowSequence: UInt64? + var runtimeSurfaceAdmissionCreationGeneration: UInt64 = 0 var requiresRestoreSpawnPacing = false var runtimeSurfaceSuspendedForAgentHibernation = false /// Bounded ownership of the live native surface. Transferred to the From 159d36ee374c8f2a3b3a0d526235a809e3dd59af Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:33:58 -0700 Subject: [PATCH 433/521] test(macOS): clear recovered creation failure --- .../FakeTerminalSurfacePaneHost.swift | 2 ++ ...nalSurfaceRestoreSpawnSchedulerTests.swift | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift index a9c3f1392f00..7805e9b84482 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift @@ -8,6 +8,7 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { private let onAttach: (() -> Void)? private(set) var explicitInputCount = 0 private(set) var runtimeSurfaceCreationFailureMessages: [String] = [] + private(set) var activeRuntimeSurfaceCreationFailureMessage: String? let runtimeSurfaceCreationFailures: AsyncStream private let runtimeSurfaceCreationFailureContinuation: AsyncStream.Continuation @@ -47,6 +48,7 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { func setMobileViewportBorder(size: CGSize?, drawRight: Bool, drawBottom: Bool) {} func showRuntimeSurfaceCreationFailure(message: String) { + activeRuntimeSurfaceCreationFailureMessage = message runtimeSurfaceCreationFailureMessages.append(message) runtimeSurfaceCreationFailureContinuation.yield(message) } diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 3322e6c16ab2..817afd3d4598 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -634,6 +634,32 @@ import CmuxTerminalCore #expect(scheduler.scheduledSurfaceIds.last == fixtures[0].surface.id) } + @Test func successfulRuntimeSurfaceCreationClearsCapacityFailure() { + let fixture = makeSurfaceFixture( + registry: FakeSurfaceRegistry(), + scheduler: RecordingRestoreSpawnScheduler(), + runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator() + ) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in } + defer { + fixture.surface.releaseSurfaceForTesting() + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil + } + + fixture.surface.failRuntimeSurfaceCreationForTeardownCapacity() + #expect( + fixture.paneHost.activeRuntimeSurfaceCreationFailureMessage != nil + ) + + fixture.surface.installRuntimeSurfaceForTesting( + UnsafeMutableRawPointer(bitPattern: 0x7542)! + ) + + #expect( + fixture.paneHost.activeRuntimeSurfaceCreationFailureMessage == nil + ) + } + @Test func stalledCloseWorkersFailDeferredCreationAndRecoverSafely() async throws { let clock = ManualTerminalSurfaceRuntimeTeardownClock() let stalledSlots = AsyncStream.makeStream() From 77f992d4a0b0155c23a086390d4bbe84fe5bf181 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:34:34 -0700 Subject: [PATCH 434/521] fix(macOS): clear recovered creation failure --- .../CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift | 3 +++ .../Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift | 1 + .../Surface/TerminalSurface+RuntimeLifecycle.swift | 1 + .../CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift | 4 ++++ Sources/GhosttyTerminalView.swift | 5 +++++ 5 files changed, 14 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift index b5ecb23e6156..c7390fc88739 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Hosting/TerminalSurfacePaneHosting.swift @@ -29,6 +29,9 @@ public protocol TerminalSurfacePaneHosting: NSView { /// Shows a permanent failure message when the native terminal cannot start. func showRuntimeSurfaceCreationFailure(message: String) + /// Clears a prior creation failure after the native terminal starts. + func clearRuntimeSurfaceCreationFailure() + /// Notifies the pane that explicit terminal input is about to be sent. func terminalSurfaceDidReceiveExplicitInput() diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift index b10ccd000d35..44b3b02014cc 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift @@ -277,6 +277,7 @@ extension TerminalSurface { on: runtimeSurface, callbackContext: callbackContext ) + paneHost.clearRuntimeSurfaceCreationFailure() } #endif } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index e389bd5f4f11..954d946824bd 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -1086,6 +1086,7 @@ extension TerminalSurface { registry.registerRuntimeSurface(createdSurface, ownerId: id) cacheControllingTTYIdentity(for: createdSurface) recordRuntimeSurfaceCreation() + paneHost.clearRuntimeSurfaceCreationFailure() // Install the shared PTY tee so output consumers receive every byte // the read thread produces, in order, before the VT parser runs. // Paired iPhones consume these bytes via `terminal.bytes` events diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift index 7805e9b84482..805a0a028b53 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeTerminalSurfacePaneHost.swift @@ -53,6 +53,10 @@ final class FakeTerminalSurfacePaneHost: NSView, TerminalSurfacePaneHosting { runtimeSurfaceCreationFailureContinuation.yield(message) } + func clearRuntimeSurfaceCreationFailure() { + activeRuntimeSurfaceCreationFailureMessage = nil + } + func terminalSurfaceDidReceiveExplicitInput() { explicitInputCount += 1 } diff --git a/Sources/GhosttyTerminalView.swift b/Sources/GhosttyTerminalView.swift index a9f0367a94a2..596087ba8afe 100644 --- a/Sources/GhosttyTerminalView.swift +++ b/Sources/GhosttyTerminalView.swift @@ -299,6 +299,11 @@ extension GhosttySurfaceScrollView: TerminalSurfacePaneHosting { runtimeSurfaceCreationFailureLabel.stringValue = message runtimeSurfaceCreationFailureLabel.isHidden = false } + + func clearRuntimeSurfaceCreationFailure() { + runtimeSurfaceCreationFailureLabel.stringValue = "" + runtimeSurfaceCreationFailureLabel.isHidden = true + } } extension TerminalSurface { From d2799b735fd80ace904ff908409fe6266621c657 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:38:10 -0700 Subject: [PATCH 435/521] test(tui): signal final cleanup retry --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 53be23bbba45..2a4d370d99b2 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26449,6 +26449,16 @@ mod tests { let mux = mux.clone(); let injected = injected.clone(); move |attempt| { + if injected.load(Ordering::Acquire) { + return; + } + if attempt < SHUTDOWN_RECONCILE_MAX_ATTEMPTS { + // Publish real pending work to bypass only the retry + // backoff. The worker consumes this signal before its next + // attempt, so hosted load cannot delay the final hook. + mux.shutdown_owner_reconciler.schedule(); + return; + } if attempt != SHUTDOWN_RECONCILE_MAX_ATTEMPTS || injected.swap(true, Ordering::AcqRel) { From a5569fa14ea700176e98b99719ab279367bed1ab Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:40:01 -0700 Subject: [PATCH 436/521] test(tui): stage hook launcher fixtures (#9970) --- tests/test_tui_package_launchers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_tui_package_launchers.py b/tests/test_tui_package_launchers.py index c6892f1978f6..674b0a8e7669 100644 --- a/tests/test_tui_package_launchers.py +++ b/tests/test_tui_package_launchers.py @@ -4,6 +4,7 @@ import platform import runpy import shutil +import stat import subprocess import sys import zipfile @@ -159,6 +160,10 @@ def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: binaries / f"cmux-tui-{target}", "printf '%s\\n' \"$CMUX_TUI_LAUNCHER_COMMAND\"", ) + write_executable( + binaries / f"cmux-tui-hook-{target}", + "exit 0", + ) wheels = tmp_path / "wheels" packaged = run( @@ -178,7 +183,10 @@ def test_pypi_launcher_preserves_uvx_invocation(tmp_path: Path) -> None: wheel = wheels / f"cmux-1.2.3-py3-none-{wheel_tag}.whl" installed = tmp_path / "installed" with zipfile.ZipFile(wheel) as archive: + hook = archive.getinfo("cmux_tui/bin/cmux-tui-hook") + assert hook.external_attr >> 16 & stat.S_IXUSR archive.extractall(installed) + assert (installed / "cmux_tui/bin/cmux-tui-hook").is_file() (installed / "cmux_tui/bin/cmux-tui").chmod(0o755) environment = os.environ.copy() From b2277499aa42209b04e103e8143083a51294b96f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:43:25 -0700 Subject: [PATCH 437/521] test(tui): keep failure probes owner-scoped --- .../crates/cmux-tui/src/machine_provider_client.rs | 13 ++++++++----- .../crates/cmux-tui/tests/terminal_host_recovery.rs | 6 ------ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs index 2363823fc41e..555a1e653b14 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs @@ -2502,11 +2502,14 @@ mod tests { accept_hello(&mut stream, &mut reader, "provider-secret"); let request: RequestEnvelope = read_test_frame(&mut reader); assert!(matches!(request.request, ProviderRequest::Snapshot(_))); - stream - .write_all(&vec![b'x'; MAX_CONTROL_FRAME_BYTES + 1]) - .expect("write oversized frame"); - stream.write_all(b"\n").expect("finish oversized frame"); - stream.flush().expect("flush oversized frame"); + let write_result = (|| -> std::io::Result<()> { + stream.write_all(&vec![b'x'; MAX_CONTROL_FRAME_BYTES + 1])?; + stream.write_all(b"\n")?; + stream.flush() + })(); + if let Err(error) = write_result { + assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe, "publish oversized frame"); + } }); let (provider, _) = ProviderClient::connect_authenticated( diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 5d2264c2989d..4575e42baf61 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -2280,12 +2280,6 @@ fn stalled_renderer_is_disconnected_without_freezing_the_host() { "stalled renderer silently froze instead of being disconnected" ); - request( - &harness.socket, - serde_json::json!({"id": 31, "cmd": "send", "surface": surface, "text": "\u{3}"}), - ); - std::thread::sleep(Duration::from_millis(50)); - // Overflow is isolated to the stalled renderer. The daemon proxy and PTY // remain responsive and the durable host record remains adoptable. let after = format!("host-still-live-{}", std::process::id()); From 9c125cdf207a473126a372eaa22448e1d1c296e6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:53:07 -0700 Subject: [PATCH 438/521] test(tui): use imported frame errors --- cmux-tui/crates/cmux-tui/src/machine_provider_client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs b/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs index 555a1e653b14..e6e1c446cf97 100644 --- a/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs +++ b/cmux-tui/crates/cmux-tui/src/machine_provider_client.rs @@ -2502,13 +2502,13 @@ mod tests { accept_hello(&mut stream, &mut reader, "provider-secret"); let request: RequestEnvelope = read_test_frame(&mut reader); assert!(matches!(request.request, ProviderRequest::Snapshot(_))); - let write_result = (|| -> std::io::Result<()> { + let write_result = (|| -> io::Result<()> { stream.write_all(&vec![b'x'; MAX_CONTROL_FRAME_BYTES + 1])?; stream.write_all(b"\n")?; stream.flush() })(); if let Err(error) = write_result { - assert_eq!(error.kind(), std::io::ErrorKind::BrokenPipe, "publish oversized frame"); + assert_eq!(error.kind(), io::ErrorKind::BrokenPipe, "publish oversized frame"); } }); From 06256836cf54950fc9168957d5b75cb19254fd26 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:04:06 -0700 Subject: [PATCH 439/521] test(tui): retire cleanup owners directly --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2a4d370d99b2..8b1d643015f8 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26433,15 +26433,11 @@ mod tests { #[test] fn cleanup_scheduled_during_the_final_retry_gets_a_fresh_retry_budget() { let mux = test_mux(); - let first = mux.new_workspace(None, Some((80, 24))).unwrap(); - let first = mux.surface(first.id).unwrap(); + let options = mux.surface_options.lock().unwrap().clone(); + let first = + Surface::spawn_for_test(mux.next_id(), options.clone(), Arc::downgrade(&mux)).unwrap(); let (first_failing, _) = first.set_recovering_server_shutdown_for_test(); - let second = Surface::spawn_for_test( - mux.next_id(), - mux.surface_options.lock().unwrap().clone(), - Arc::downgrade(&mux), - ) - .unwrap(); + let second = Surface::spawn_for_test(mux.next_id(), options, Arc::downgrade(&mux)).unwrap(); let (second_failing, _) = second.set_recovering_server_shutdown_for_test(); let injected = Arc::new(AtomicBool::new(false)); let (injected_tx, injected_rx) = std::sync::mpsc::sync_channel(1); @@ -26472,7 +26468,7 @@ mod tests { })); let deadline = Instant::now() + Duration::from_secs(2); - assert!(mux.close_surface(first.id).unwrap()); + mux.retire_surface_runtime(first); let remaining = deadline .checked_duration_since(Instant::now()) .expect("final cleanup attempt exceeded its failure deadline"); From d51dffef64c2c5144a3dfaa40a4c98bac850d81e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:37:52 -0700 Subject: [PATCH 440/521] test(macOS): retry stalled close recovery --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 817afd3d4598..887178cd2250 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -767,10 +767,13 @@ import CmuxTerminalCore ) #expect( deferredFixtures[index].surface - .runtimeSurfaceAdmissionDeferredCreationSource == nil + .runtimeSurfaceAdmissionDeferredCreationSource != nil ) } - assertOverflowStorageIsEmpty(coordinator) + #expect( + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + .entryCount == 1 + ) let rejectedAfterStall = makeSurfaceFixture( registry: registry, @@ -787,6 +790,12 @@ import CmuxTerminalCore try #require(await postStallFailureIterator.next()) == expectedMessage ) + #expect( + rejectedAfterStall.surface + .runtimeSurfaceAdmissionDeferredCreationSource != nil + ) + let firstDeferredAttemptCount = deferredFixtures[0].surface + .debugRuntimeSurfaceCreateAttemptCountForTesting() let stalledStateAfterFree = Task.detached { finishFreeStarted.wait() @@ -801,6 +810,16 @@ import CmuxTerminalCore "returned close worker retained the all-stalled admission failure" ) #expect(await tickets[0].wait(timeout: nil)) + await waitForMainActorQueueBarrier() + #expect( + deferredFixtures[0].surface + .debugRuntimeSurfaceCreateAttemptCountForTesting() + == firstDeferredAttemptCount + 1 + ) + #expect( + deferredFixtures[0].surface + .runtimeSurfaceAdmissionDeferredCreationSource == nil + ) #expect( freedPointerBits.withLock { $0 } == [UInt(bitPattern: pointers[0])] From 1047d232362774188d8be6ce1e069bb0c960ed40 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:47:45 -0700 Subject: [PATCH 441/521] fix(macOS): retry stalled close recovery --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 28 +++++----- ...SurfaceRuntimeOwnershipRecoveryEntry.swift | 1 + ...timeOwnershipRecoveryRescanScheduler.swift | 51 +++++++++++-------- ...nalSurfaceRuntimeTeardownCoordinator.swift | 5 +- .../TerminalSurface+RuntimeLifecycle.swift | 35 +++++++++++-- 5 files changed, 78 insertions(+), 42 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 618fbf00de49..9396018fcaab 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -92,6 +92,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery state.recoveryEntriesByID[recoveryID]?.failure = onFailure + state.recoveryEntriesByID[recoveryID]?.failureReported = false return (.deferred, claimedCapacity) } guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { @@ -262,10 +263,13 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { state.withLock { state in state.closeTeardownAllStalled = true var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] = [] - while let recoveryID = state.recoveryHeadID, - let failure = state.recoveryEntriesByID[recoveryID]?.failure { - failures.append(failure) - _ = removeRecoveryAction(recoveryID, from: &state) + var recoveryID = state.recoveryHeadID + while let currentID = recoveryID, + let entry = state.recoveryEntriesByID[currentID] { + recoveryID = entry.nextID + guard !entry.failureReported else { continue } + state.recoveryEntriesByID[currentID]?.failureReported = true + failures.append(entry.failure) } state.pendingRecoveryFailureCount += failures.count precondition( @@ -302,21 +306,17 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func clearAllStalledCloseTeardowns() { - let output: ( - grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant?, - requestRescan: Bool - ) = state.withLock { state in + let grant = state.withLock { state in guard state.closeTeardownAllStalled else { - return (nil, false) + return nil } state.closeTeardownAllStalled = false let grant = takeNextRecoveryGrant(from: &state) - return (grant, takeRecoveryRescanRequest(from: &state)) - } - schedule(output.0) - if output.1 { - recoveryRescanScheduler.requestRescan() + _ = takeRecoveryRescanRequest(from: &state) + return grant } + schedule(grant) + recoveryRescanScheduler.requestRescan() } func cancelRecovery(_ recoveryID: UUID) { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift index 382020416d70..820a5828727c 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift @@ -3,6 +3,7 @@ import Foundation struct TerminalSurfaceRuntimeOwnershipRecoveryEntry: Sendable { var action: TerminalSurfaceRuntimeOwnershipRecovery var failure: TerminalSurfaceRuntimeOwnershipRecoveryFailure + var failureReported = false var previousID: UUID? var nextID: UUID? } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 2aea9e960c83..2e512a753a29 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -17,6 +17,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: let surfaceID: UUID let sequence: UInt64 weak var surface: TerminalSurface? + var failureReported = false var previousID: UUID? var nextID: UUID? @@ -41,6 +42,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var headID: UUID? var tailID: UUID? var failureThroughSequence: UInt64? + var failureCursorID: UUID? var rescanRequested = false var rescanTask: Task? } @@ -89,6 +91,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: state.withLockUnchecked { state in if let entry = state.entriesByID[surfaceID] { entry.surface = surface + entry.failureReported = false state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) return .updated(sequence: entry.sequence) @@ -151,6 +154,9 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: state.failureThroughSequence ?? 0, tail.sequence ) + if state.failureCursorID == nil { + state.failureCursorID = state.headID + } state.rescanRequested = true prepareRescanTaskIfNeeded(state: &state, startGate: &startGate) } @@ -186,21 +192,13 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var processedCount = 0 if shouldRun { while processedCount < Self.maximumBatchCount { - if let failedEntry = state.withLockUnchecked({ state -> OverflowEntry? in - guard let cutoff = state.failureThroughSequence, - let headID = state.headID, - let entry = state.entriesByID[headID], - entry.sequence <= cutoff - else { - normalizeFailureCutoff(in: &state) - return nil - } - _ = removeOverflow(surfaceID: entry.surfaceID, from: &state) - normalizeFailureCutoff(in: &state) - return entry + if let failedEntry = state.withLockUnchecked({ state in + takeNextFailureEntry(from: &state) }) { failedEntry.surface? - .failRuntimeSurfaceCreationForTeardownCapacity() + .failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: true + ) processedCount += 1 continue } @@ -295,23 +293,34 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: } else { state.tailID = entry.previousID } + if state.failureCursorID == surfaceID { + state.failureCursorID = entry.nextID + } if state.entriesByID.isEmpty { state.headID = nil state.tailID = nil state.failureThroughSequence = nil + state.failureCursorID = nil state.rescanRequested = false } return entry } - private func normalizeFailureCutoff(in state: inout State) { - guard let cutoff = state.failureThroughSequence else { return } - guard let headID = state.headID, - let head = state.entriesByID[headID], - head.sequence <= cutoff - else { - state.failureThroughSequence = nil - return + private func takeNextFailureEntry( + from state: inout State + ) -> OverflowEntry? { + guard let cutoff = state.failureThroughSequence else { return nil } + while let currentID = state.failureCursorID, + let entry = state.entriesByID[currentID] + { + guard entry.sequence <= cutoff else { break } + state.failureCursorID = entry.nextID + guard !entry.failureReported else { continue } + entry.failureReported = true + return entry } + state.failureThroughSequence = nil + state.failureCursorID = nil + return nil } } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 248770fdad4b..a0d6b055cca9 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -14,8 +14,9 @@ internal import CMUXDebugLog /// through completed free. While both close slots are occupied, new ownership /// waits for a worker-completion signal, so retained frees cannot grow without /// bound. If both workers exceed their five-second watchdog, queued creations -/// report a visible failure. A late native free keeps its pointer ownership and -/// restores admission only after it returns. +/// report a visible failure and retain their bounded FIFO owners. A late native +/// free keeps its pointer ownership, restores admission only after it returns, +/// and retries those creations in their original order. /// Each admitted hibernation also owns one independently startable utility slot. /// Slot occupancy fences new ownership until one active free completes. The app constructs /// exactly one instance and injects it through diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index 954d946824bd..b1b73bfe265a 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -773,7 +773,9 @@ extension TerminalSurface { self.runtimeSurfaceAdmissionDeferredCreationSource != nil else { return } - self.failRuntimeSurfaceCreationForTeardownCapacity() + self.failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: true + ) } } @@ -830,7 +832,7 @@ extension TerminalSurface { } return nil case .closeTeardownStalled: - failRuntimeSurfaceCreationForTeardownCapacity() + retainRuntimeSurfaceCreationUntilTeardownRecovers() return nil } } @@ -883,7 +885,9 @@ extension TerminalSurface { teardownCoordinator .requestRuntimeSurfaceOwnershipRecoveryRescan() case .closeTeardownStalled: - failRuntimeSurfaceCreationForTeardownCapacity() + failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: true + ) } } @@ -927,8 +931,29 @@ extension TerminalSurface { } @MainActor - func failRuntimeSurfaceCreationForTeardownCapacity() { - cancelRuntimeSurfaceCreationAfterAdmissionRecovery() + private func retainRuntimeSurfaceCreationUntilTeardownRecovers() { + let registration = runtimeTeardown + .registerRuntimeSurfaceOwnershipRecoveryOverflow( + surfaceID: id, + surface: self + ) + switch registration { + case .registered(let sequence), .updated(let sequence): + runtimeSurfaceAdmissionOverflowSequence = sequence + failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: true + ) + case .rejected: + failRuntimeSurfaceCreationForTeardownCapacity() + } + } + + func failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: Bool = false + ) { + if !preservingRecoveryOwner { + cancelRuntimeSurfaceCreationAfterAdmissionRecovery() + } paneHost.showRuntimeSurfaceCreationFailure( message: String( localized: "terminal.surface.runtimeCreation.capacityExceeded", From ebf50c1b300d6519a50efc217e6efb1ffdade8b9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:26:32 -0700 Subject: [PATCH 442/521] fix(macOS): type stalled recovery grant --- .../Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 9396018fcaab..b0570ba431ab 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -306,7 +306,8 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func clearAllStalledCloseTeardowns() { - let grant = state.withLock { state in + let grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant? + grant = state.withLock { state in guard state.closeTeardownAllStalled else { return nil } From d155d864a9218202cb1c174fb9afb8cc7df5081c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:26:32 -0700 Subject: [PATCH 443/521] test(tui): match live agent projections --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 8b1d643015f8..bdb9fb75dea9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -23681,13 +23681,12 @@ mod tests { mux.close_terminal(&host.terminal_id, &host.incarnation).unwrap(); assert!(mux.list_agents(None, None).is_empty()); - assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 1); - assert_eq!( + assert_eq!(mux.resource_agent_projection_count_for_test().unwrap(), 0); + assert!( crate::resource_api::public_session_snapshot(&mux).unwrap()["agents"] .as_array() .unwrap() - .len(), - 1 + .is_empty() ); assert!(mux.surface_notification(first.id).is_none()); assert!(mux.surface(first.id).is_none()); From b84df58d76650198a062203a73c851152064ac31 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:29:27 -0700 Subject: [PATCH 444/521] test: restore snapshot resume command assertion (#9976) --- cmuxTests/PiVaultAgentPersistenceTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmuxTests/PiVaultAgentPersistenceTests.swift b/cmuxTests/PiVaultAgentPersistenceTests.swift index db8dc7b86be7..0e8d313aa415 100644 --- a/cmuxTests/PiVaultAgentPersistenceTests.swift +++ b/cmuxTests/PiVaultAgentPersistenceTests.swift @@ -1108,7 +1108,7 @@ final class PiVaultAgentPersistenceTests: XCTestCase { XCTAssertEqual(loadedAgent.kind.rawValue, "pi") XCTAssertEqual(loadedAgent.sessionId, sessionPath) XCTAssertEqual( - loadedAgent.copyResumeCommand, + loadedAgent.resumeCommand, TerminalStartupWorkingDirectoryPrefix.prefix( "'env' 'PI_CODING_AGENT_SESSION_DIR=\(tempDir.path)' '/opt/homebrew/bin/pi' '--session' '\(sessionPath)' '--session-dir' '\(tempDir.path)'", workingDirectory: "/tmp/pi repo" From 54c3c66407746e83b7b6b698abbef6ef02f05326 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:41:54 -0700 Subject: [PATCH 445/521] fix(macOS): isolate recovery failure UI --- .../CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index b1b73bfe265a..f52e875ca023 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -948,6 +948,7 @@ extension TerminalSurface { } } + @MainActor func failRuntimeSurfaceCreationForTeardownCapacity( preservingRecoveryOwner: Bool = false ) { From f6c3d95343d0d2eb884a8c31ac356320e847a3bc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:00:24 -0700 Subject: [PATCH 446/521] test(tui): signal transient host adoption --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 ++++ .../cmux-tui/tests/terminal_host_recovery.rs | 31 ++++++++++++------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bdb9fb75dea9..2c05d414afaf 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -4134,6 +4134,12 @@ impl Mux { if complete || mux.shutting_down.load(Ordering::Acquire) { mux.terminal_adoptions.lock().unwrap().remove(&task.record.terminal_id); } else { + #[cfg(debug_assertions)] + if let Some(signal) = + std::env::var_os("CMUX_TUI_TEST_TERMINAL_ADOPTION_RETRY_SIGNAL") + { + let _ = std::fs::write(signal, b"1"); + } task.delay = (task.delay * 2).min(Duration::from_secs(5)); task.next_attempt = Instant::now() + task.delay; state.requeue_retry(task); diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index 4575e42baf61..b62f2dc3d952 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -176,6 +176,16 @@ impl RecoveryHarness { wait_for_socket(&self.socket); } + fn restart_with_adoption_retry_signals(&mut self, retry: &Path, adopted: &Path) { + assert!(self.child.is_none()); + let mut command = self.daemon_command(); + command + .env("CMUX_TUI_TEST_TERMINAL_ADOPTION_RETRY_SIGNAL", retry) + .env("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL", adopted); + self.child = Some(command.spawn().unwrap()); + wait_for_socket(&self.socket); + } + fn daemon_command(&self) -> Command { let mut command = Command::new(bin()); command @@ -2934,7 +2944,7 @@ fn daemon_crash_after_record_before_ready_adopts_same_live_host() { } #[test] -fn interrupted_creation_waits_for_transient_host_adoption_before_serving() { +fn interrupted_creation_retries_transient_host_adoption_before_running() { let mut harness = RecoveryHarness::start_with_host_ready_delay("pre-ready-retry", 2_000); let stream = transport::connect(&harness.socket).unwrap(); let mut writer = stream.try_clone_box().unwrap(); @@ -2957,17 +2967,14 @@ fn interrupted_creation_waits_for_transient_host_adoption_before_serving() { drop(writer); drop(stream); fs::rename(&endpoint, &held_endpoint).unwrap(); - let restore_endpoint = std::thread::spawn(move || { - std::thread::sleep(Duration::from_millis(500)); - fs::rename(held_endpoint, endpoint).unwrap(); - }); - let started = Instant::now(); - harness.restart(); - restore_endpoint.join().unwrap(); - assert!( - started.elapsed() >= Duration::from_millis(400), - "daemon served before interrupted host adoption settled" - ); + let retry_signal_path = harness.dir.join("terminal-adoption-retry"); + let adopted_signal_path = harness.dir.join("terminal-adopted-after-retry"); + let mut retry_signal = TestFifoSignal::create(&retry_signal_path); + let mut adopted_signal = TestFifoSignal::create(&adopted_signal_path); + harness.restart_with_adoption_retry_signals(&retry_signal_path, &adopted_signal_path); + retry_signal.receive(Duration::from_secs(15)); + fs::rename(held_endpoint, endpoint).unwrap(); + adopted_signal.receive(Duration::from_secs(15)); let resolved = request( &harness.socket, From 5b2b75d46dd1308f9df69d1975447c638112a4e4 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:19:53 -0700 Subject: [PATCH 447/521] test(tui): keep process retry coverage live (#9927) * test(tui): keep process retry coverage live * test(tui): name termination coordinator * test(tui): use termination attempt deadline --- .../cmux-tui/src/server_lifecycle/legacy_process.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs index a8b1cb5b80c6..06085cc5ef3d 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle/legacy_process.rs @@ -222,12 +222,15 @@ pub(super) fn terminate_process_tree_until( process: ProcessIdentity, deadline: Instant, ) -> io::Result<()> { - let tree = FrozenProcessTree::freeze(process, deadline)?; - tree.terminate_until(deadline) + terminate_process_tree_until_with(process, deadline, |attempt_deadline| { + let tree = FrozenProcessTree::freeze(process, attempt_deadline)?; + tree.terminate_until(attempt_deadline) + }) } #[cfg(test)] -fn retry_process_tree_termination( +// This injected-attempt helper and all of its callers are test-only. +fn terminate_process_tree_until_with( process: ProcessIdentity, deadline: Instant, mut attempt: impl FnMut(Instant) -> io::Result<()>, @@ -1003,7 +1006,7 @@ mod tests { ProcessIdentity::capture(libc::pid_t::try_from(child.id()).unwrap()).unwrap().unwrap(); let mut attempts = 0; - let result = retry_process_tree_termination( + let result = terminate_process_tree_until_with( process, Instant::now() + Duration::from_secs(1), |_| { From 2a4d11271fe6e0c49e236a851e7a5aba1000b164 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:27:49 -0700 Subject: [PATCH 448/521] test(tui): release adoption before listener wait --- cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index b62f2dc3d952..cecdae85fab8 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -176,14 +176,13 @@ impl RecoveryHarness { wait_for_socket(&self.socket); } - fn restart_with_adoption_retry_signals(&mut self, retry: &Path, adopted: &Path) { + fn spawn_with_adoption_retry_signals(&mut self, retry: &Path, adopted: &Path) { assert!(self.child.is_none()); let mut command = self.daemon_command(); command .env("CMUX_TUI_TEST_TERMINAL_ADOPTION_RETRY_SIGNAL", retry) .env("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL", adopted); self.child = Some(command.spawn().unwrap()); - wait_for_socket(&self.socket); } fn daemon_command(&self) -> Command { @@ -2971,10 +2970,11 @@ fn interrupted_creation_retries_transient_host_adoption_before_running() { let adopted_signal_path = harness.dir.join("terminal-adopted-after-retry"); let mut retry_signal = TestFifoSignal::create(&retry_signal_path); let mut adopted_signal = TestFifoSignal::create(&adopted_signal_path); - harness.restart_with_adoption_retry_signals(&retry_signal_path, &adopted_signal_path); + harness.spawn_with_adoption_retry_signals(&retry_signal_path, &adopted_signal_path); retry_signal.receive(Duration::from_secs(15)); fs::rename(held_endpoint, endpoint).unwrap(); adopted_signal.receive(Duration::from_secs(15)); + wait_for_socket(&harness.socket); let resolved = request( &harness.socket, From 56fd01917704d915b1c48b7739256023a5952871 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:42:32 -0700 Subject: [PATCH 449/521] test(macOS): return fake surface snapshot --- .../Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift index 5d2758984ac9..5fc9165aa5e6 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/FakeSurfaceRegistry.swift @@ -81,6 +81,6 @@ final class FakeSurfaceRegistry: @unchecked Sendable, TerminalSurfaceRegistering } func allSurfaces() -> [any TerminalSurfacing] { allSurfacesCallCountLock.withLock { $0 += 1 } - backing.allSurfaces() + return backing.allSurfaces() } } From 5d0e40bf8d4cd93ee3e1f6efd5677ec57e5b2ce4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:44:04 -0700 Subject: [PATCH 450/521] test(macOS): keep overflow recovery behind primary work --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 887178cd2250..df8ca9cf2a4d 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -408,6 +408,49 @@ import CmuxTerminalCore ) } + @Test func overflowRecoveryWaitsForEveryOlderPrimaryRequest() async throws { + let ownerCount = 4 + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: ownerCount + ) + let owners = try (0.. Date: Tue, 11 Aug 2026 01:45:03 -0700 Subject: [PATCH 451/521] fix(macOS): preserve recovery admission order --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index b0570ba431ab..fb692d33632e 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -115,8 +115,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { func claimRecoveryCapacity() -> TerminalSurfaceRuntimeOwnershipRecoveryCapacityReservation? { state.withLock { state in - guard !state.closeTeardownAllStalled, - recoveryCapacityIsOpen(in: state) else { + guard overflowRecoveryCapacityIsOpen(in: state) else { return nil } let reservation = @@ -127,9 +126,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func recoveryCapacityIsOpen() -> Bool { - state.withLock { - !$0.closeTeardownAllStalled && recoveryCapacityIsOpen(in: $0) - } + state.withLock { overflowRecoveryCapacityIsOpen(in: $0) } } func releaseRecoveryCapacity( @@ -384,6 +381,15 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { < maximumOwnerCount } + private func overflowRecoveryCapacityIsOpen( + in state: TerminalSurfaceRuntimeOwnershipAdmissionState + ) -> Bool { + !state.closeTeardownAllStalled + && state.recoveryHeadID == nil + && !state.recoveryGrantIsScheduled + && recoveryCapacityIsOpen(in: state) + } + private func takeRecoveryRescanRequest( from state: inout TerminalSurfaceRuntimeOwnershipAdmissionState ) -> Bool { @@ -436,15 +442,17 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } private func recoveryGrantDidComplete() { - let output = state.withLock { state in + let grant = state.withLock { state in state.recoveryGrantIsScheduled = false let grant = takeNextRecoveryGrant(from: &state) - return (grant, takeRecoveryRescanRequest(from: &state)) - } - schedule(output.0) - if output.1 { - recoveryRescanScheduler.requestRescan() + _ = takeRecoveryRescanRequest(from: &state) + return grant } + schedule(grant) + // The overflow FIFO can proceed only after every older primary grant + // has run. Recheck it after each grant, including the final grant that + // clears `recoveryGrantIsScheduled` without another state transition. + recoveryRescanScheduler.requestRescan() } #if DEBUG From 82e097727cde4f7fa07fc31a271f576287e40a3f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:17:53 -0700 Subject: [PATCH 452/521] fix(tui): retain catalog adoption owners --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 2c05d414afaf..864a8e9e0975 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -4064,8 +4064,9 @@ impl Mux { let live_incarnations = { let state = self.state.lock().unwrap(); let mut live = HashMap::>::new(); - for identity in - state.surfaces.values().filter_map(|surface| surface.live_terminal_host_identity()) + for identity in unique_surface_runtimes(&state) + .iter() + .filter_map(|surface| surface.live_terminal_host_identity()) { live.entry(identity.terminal_id).or_default().insert(identity.incarnation); } From cf01408848597fa61bf4457de255b85dac3c24c6 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:35:00 -0700 Subject: [PATCH 453/521] test(macOS): retain pending recovery failure ownership --- ...rfaceRuntimeTeardownCoordinatorTests.swift | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index cbee398cf380..3afd7438bfad 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -1140,6 +1140,66 @@ private func requireTeardownTicket( admission.release(secondOwner) } + @MainActor + @Test func updatedRecoveryWaitsForItsPendingFailureDelivery() throws { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + let owners = try (0..<2).map { _ in + try #require(admission.reserve()) + } + defer { + admission.cancelRecovery(recoveryID) + for owner in owners { + admission.release(owner) + } + } + let recoveryID = UUID() + var firstFailureCount = 0 + var updatedFailureCount = 0 + + #expect( + admission.reserve( + recoveryID: recoveryID, + onRecovery: { _ in }, + onFailure: { firstFailureCount += 1 } + ) == .deferred + ) + let firstFailures = + admission.failRecoveriesForAllStalledCloseTeardowns() + #expect(firstFailures.count == 1) + admission.clearAllStalledCloseTeardowns() + + #expect( + admission.reserve( + recoveryID: recoveryID, + onRecovery: { _ in }, + onFailure: { updatedFailureCount += 1 } + ) == .deferred + ) + let duplicateFailures = + admission.failRecoveriesForAllStalledCloseTeardowns() + #expect( + duplicateFailures.isEmpty, + "an updated recovery published twice before its first failure delivery completed" + ) + + for failure in firstFailures { + failure() + } + admission.completeStalledCloseRecoveryFailures(firstFailures.count) + for failure in duplicateFailures { + failure() + } + if !duplicateFailures.isEmpty { + admission.completeStalledCloseRecoveryFailures( + duplicateFailures.count + ) + } + #expect(firstFailureCount == 1) + #expect(updatedFailureCount == duplicateFailures.count) + } + @Test func stuckHibernationFreeDoesNotStrandAnotherAdmissionOrClose() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator() let isolatedSurface = UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) From f128fc02b3f0f785e38a958c6109d750ca0c30ae Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:36:34 -0700 Subject: [PATCH 454/521] fix(macOS): retain recovery failure delivery ownership --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 55 +++++++++++++------ ...urfaceRuntimeOwnershipAdmissionState.swift | 2 +- ...SurfaceRuntimeOwnershipRecoveryEntry.swift | 2 +- ...timeOwnershipRecoveryFailureDelivery.swift | 12 ++++ ...RuntimeOwnershipRecoveryFailureDrain.swift | 8 ++- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 8 ++- ...rfaceRuntimeTeardownCoordinatorTests.swift | 4 +- 7 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery.swift diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index fb692d33632e..615c0c025840 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -92,7 +92,6 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery state.recoveryEntriesByID[recoveryID]?.failure = onFailure - state.recoveryEntriesByID[recoveryID]?.failureReported = false return (.deferred, claimedCapacity) } guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { @@ -256,21 +255,33 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func failRecoveriesForAllStalledCloseTeardowns() - -> [TerminalSurfaceRuntimeOwnershipRecoveryFailure] { + -> [TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery] { state.withLock { state in state.closeTeardownAllStalled = true - var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] = [] + var failures: [ + TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery + ] = [] var recoveryID = state.recoveryHeadID while let currentID = recoveryID, let entry = state.recoveryEntriesByID[currentID] { recoveryID = entry.nextID - guard !entry.failureReported else { continue } - state.recoveryEntriesByID[currentID]?.failureReported = true - failures.append(entry.failure) + guard entry.pendingFailureDeliveryID == nil else { continue } + let delivery = + TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery( + recoveryID: currentID, + deliveryID: UUID(), + failure: entry.failure + ) + state.recoveryEntriesByID[currentID]? + .pendingFailureDeliveryID = delivery.deliveryID + state.pendingRecoveryFailureDeliveryIDs.insert( + delivery.deliveryID + ) + failures.append(delivery) } - state.pendingRecoveryFailureCount += failures.count precondition( - state.pendingRecoveryFailureCount <= maximumOwnerCount, + state.pendingRecoveryFailureDeliveryIDs.count + <= maximumOwnerCount, "stalled recovery failures must remain bounded" ) state.recoveryRescanRequested = false @@ -278,15 +289,27 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } } - func completeStalledCloseRecoveryFailures(_ completedCount: Int) { - precondition(completedCount > 0) + func completeStalledCloseRecoveryFailures( + _ completedDeliveries: [ + TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery + ] + ) { + precondition(!completedDeliveries.isEmpty) let output = state.withLock { state in - precondition( - completedCount <= state.pendingRecoveryFailureCount, - "completed stalled recovery failures must be owned" - ) let capacityWasOpen = recoveryCapacityIsOpen(in: state) - state.pendingRecoveryFailureCount -= completedCount + for delivery in completedDeliveries { + precondition( + state.pendingRecoveryFailureDeliveryIDs.remove( + delivery.deliveryID + ) != nil, + "completed stalled recovery failure must be owned" + ) + if state.recoveryEntriesByID[delivery.recoveryID]? + .pendingFailureDeliveryID == delivery.deliveryID { + state.recoveryEntriesByID[delivery.recoveryID]? + .pendingFailureDeliveryID = nil + } + } let recoveryCapacityOpened = !capacityWasOpen && recoveryCapacityIsOpen(in: state) let grant = takeNextRecoveryGrant(from: &state) @@ -377,7 +400,7 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { ) -> Bool { state.recoveryEntriesByID.count + state.recoveryCapacityReservationIDs.count - + state.pendingRecoveryFailureCount + + state.pendingRecoveryFailureDeliveryIDs.count < maximumOwnerCount } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift index f5f46bc3daed..a03381d5618c 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmissionState.swift @@ -13,7 +13,7 @@ struct TerminalSurfaceRuntimeOwnershipAdmissionState { var recoveryCapacityReservationIDs: Set = [] var recoveryHeadID: UUID? var recoveryTailID: UUID? - var pendingRecoveryFailureCount = 0 + var pendingRecoveryFailureDeliveryIDs: Set = [] var recoveryGrantIsScheduled = false var recoveryRescanRequested = false } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift index 820a5828727c..7450b745c263 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryEntry.swift @@ -3,7 +3,7 @@ import Foundation struct TerminalSurfaceRuntimeOwnershipRecoveryEntry: Sendable { var action: TerminalSurfaceRuntimeOwnershipRecovery var failure: TerminalSurfaceRuntimeOwnershipRecoveryFailure - var failureReported = false + var pendingFailureDeliveryID: UUID? var previousID: UUID? var nextID: UUID? } diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery.swift new file mode 100644 index 000000000000..6fd804cbe700 --- /dev/null +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery.swift @@ -0,0 +1,12 @@ +internal import Foundation + +internal struct TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery: Sendable { + internal let recoveryID: UUID + internal let deliveryID: UUID + internal let failure: TerminalSurfaceRuntimeOwnershipRecoveryFailure + + @MainActor + internal func callAsFunction() { + failure() + } +} diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift index cb69f3468a06..bc135b121ff9 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift @@ -11,7 +11,9 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: private static let maximumBatchCount = 32 private struct State { - var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] = [] + var failures: [ + TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery + ] = [] var nextFailureIndex = 0 var task: Task? } @@ -32,7 +34,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: } internal func enqueue( - _ failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailure] + _ failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery] ) { guard !failures.isEmpty else { return } var startGate: TerminalSurfaceRuntimeTeardownStartGate? @@ -86,7 +88,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: for failure in batch { failure() } - admission.completeStalledCloseRecoveryFailures(batch.count) + admission.completeStalledCloseRecoveryFailures(batch) var startGate: TerminalSurfaceRuntimeTeardownStartGate? state.withLockUnchecked { state in diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index df8ca9cf2a4d..3cab9fdce0f2 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -944,7 +944,9 @@ import CmuxTerminalCore ) coordinator.runtimeOwnershipAdmission - .completeStalledCloseRecoveryFailures(32) + .completeStalledCloseRecoveryFailures( + Array(failures.prefix(32)) + ) await waitForMainActorQueueBarrier() #expect( coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot @@ -955,7 +957,9 @@ import CmuxTerminalCore failure() } coordinator.runtimeOwnershipAdmission - .completeStalledCloseRecoveryFailures(capacity - 32) + .completeStalledCloseRecoveryFailures( + Array(failures.suffix(from: 32)) + ) #expect(target.paneHost.runtimeSurfaceCreationFailureMessages.isEmpty) overflow.surface.beginPortalCloseLifecycle( reason: "test.completedFailureBatch" diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index 3afd7438bfad..fad68ad89d35 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -1187,13 +1187,13 @@ private func requireTeardownTicket( for failure in firstFailures { failure() } - admission.completeStalledCloseRecoveryFailures(firstFailures.count) + admission.completeStalledCloseRecoveryFailures(firstFailures) for failure in duplicateFailures { failure() } if !duplicateFailures.isEmpty { admission.completeStalledCloseRecoveryFailures( - duplicateFailures.count + duplicateFailures ) } #expect(firstFailureCount == 1) From f93d06f9343cdf790eac4a041bfd40bd1037d5e1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:36:44 -0700 Subject: [PATCH 455/521] test(tui): remove stale shell quote helper --- cmux-tui/crates/cmux-tui/src/config.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/config.rs b/cmux-tui/crates/cmux-tui/src/config.rs index 09f0ebc449ff..de183b36e463 100644 --- a/cmux-tui/crates/cmux-tui/src/config.rs +++ b/cmux-tui/crates/cmux-tui/src/config.rs @@ -5987,11 +5987,6 @@ mod tests { }) } - #[cfg(all(unix, not(target_os = "macos")))] - fn shell_quote_path(path: &Path) -> String { - format!("'{}'", path.to_string_lossy().replace('\'', "'\\''")) - } - #[test] fn ghostty_config_helper_output_reader_drains_large_palette() { let mut output = String::new(); From 988dc3d484181ea46781f14d03a63f37777c671c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:40:55 -0700 Subject: [PATCH 456/521] test(macOS): declare recovery owner before cleanup --- .../TerminalSurfaceRuntimeTeardownCoordinatorTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift index fad68ad89d35..fbde4a9369c7 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRuntimeTeardownCoordinatorTests.swift @@ -1148,13 +1148,13 @@ private func requireTeardownTicket( let owners = try (0..<2).map { _ in try #require(admission.reserve()) } + let recoveryID = UUID() defer { admission.cancelRecovery(recoveryID) for owner in owners { admission.release(owner) } } - let recoveryID = UUID() var firstFailureCount = 0 var updatedFailureCount = 0 From 4cd584c468a6d823a842557f9548a835daee629e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:13:01 -0700 Subject: [PATCH 457/521] test(tui): wait for adoption reconnect receipt --- .../crates/cmux-tui/tests/terminal_host_recovery.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs index cecdae85fab8..9f9e78bf1f4d 100644 --- a/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs +++ b/cmux-tui/crates/cmux-tui/tests/terminal_host_recovery.rs @@ -2973,13 +2973,21 @@ fn interrupted_creation_retries_transient_host_adoption_before_running() { harness.spawn_with_adoption_retry_signals(&retry_signal_path, &adopted_signal_path); retry_signal.receive(Duration::from_secs(15)); fs::rename(held_endpoint, endpoint).unwrap(); - adopted_signal.receive(Duration::from_secs(15)); + let adoption_deadline = Instant::now() + Duration::from_secs(15); + adopted_signal.receive(adoption_deadline.saturating_duration_since(Instant::now())); wait_for_socket(&harness.socket); - let resolved = request( + let mut resolved = request( &harness.socket, serde_json::json!({"id":2,"cmd":"resolve-terminal","terminal_id":terminal_id}), ); + if resolved["lifecycle"] == "adopting" { + adopted_signal.receive(adoption_deadline.saturating_duration_since(Instant::now())); + resolved = request( + &harness.socket, + serde_json::json!({"id":2,"cmd":"resolve-terminal","terminal_id":terminal_id}), + ); + } let surface = resolved["surface"].as_u64().expect("recovered terminal has a surface"); assert_eq!(resolved["lifecycle"], "running"); assert_eq!(resolved["terminal_incarnation"], incarnation); From 23307cbe298a37d739c1f7226a9546d96a21647c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:13:23 -0700 Subject: [PATCH 458/521] test(tui): signal adoption reconnect completion --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 864a8e9e0975..c03ef7624b52 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -6979,9 +6979,16 @@ impl Mux { { return false; } - surface.is_some_and(|surface| { + let reconnected = surface.is_some_and(|surface| { self.reconcile_reconnected_kitty_image_surface(&surface, applied_kitty_limits) - }) + }); + #[cfg(debug_assertions)] + if reconnected + && let Some(signal) = std::env::var_os("CMUX_TUI_TEST_TERMINAL_ADOPTED_SIGNAL") + { + let _ = std::fs::write(signal, b"1"); + } + reconnected } pub(crate) fn lock_client_sizing_lifecycle(&self) -> MutexGuard<'_, ()> { From 1d3c6fdc8b48829ecae4345740eb71ac746af9be Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:16:44 -0700 Subject: [PATCH 459/521] fix(ci): require Blacksmith macOS verification --- scripts/verify-cmux-tui-hosted.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/verify-cmux-tui-hosted.sh b/scripts/verify-cmux-tui-hosted.sh index d0a563dcba84..ca5e07ace677 100755 --- a/scripts/verify-cmux-tui-hosted.sh +++ b/scripts/verify-cmux-tui-hosted.sh @@ -127,6 +127,7 @@ gh workflow run "$WORKFLOW" \ -f "commit=$commit" \ -f "mode=$mode" \ -f "test_filter=$test_filter" \ + -f "use_blacksmith_macos=true" \ -f "request_id=$request_id" run_id="" From cafaad9e611ae420fe97c23f498a96ee55700c98 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:48:42 -0700 Subject: [PATCH 460/521] refactor(tui): centralize browser runtime slot ownership (#9930) * refactor(tui): centralize browser runtime slot ownership * style(tui): apply browser slot formatting * refactor(tui): preserve bootstrap mux lifetime * refactor(tui): resolve browser bootstrap before runtime * fix(tui): serialize browser shutdown publication * style(tui): apply hosted browser slot formatting * test(tui): signal async creation shutdown --- cmux-tui/crates/cmux-tui-core/src/browser.rs | 15 +- cmux-tui/crates/cmux-tui-core/src/mux.rs | 488 +++++++++---------- 2 files changed, 241 insertions(+), 262 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/browser.rs b/cmux-tui/crates/cmux-tui-core/src/browser.rs index 7b8cf40141bc..0daa4c6d61df 100644 --- a/cmux-tui/crates/cmux-tui-core/src/browser.rs +++ b/cmux-tui/crates/cmux-tui-core/src/browser.rs @@ -888,18 +888,14 @@ impl BrowserRuntime { pub(crate) fn bootstrap_surface_sync( self: &Arc, surface: Arc, - bootstrap: BrowserBootstrap, + target_id: String, + url: String, mux: Weak, ) -> anyhow::Result<()> { if self.is_closed() { anyhow::bail!("CDP browser connection is closed"); } - let (target_id, normalized_url) = match bootstrap { - BrowserBootstrap::ExistingTarget { target_id, url } => (target_id, normalize_url(&url)), - BrowserBootstrap::Provider { .. } => { - anyhow::bail!("browser provider target was not resolved before CDP bootstrap") - } - }; + let normalized_url = normalize_url(&url); let session_id = self.client.attach_to_target(&target_id)?; let events = self.register(&target_id, &session_id); if surface.as_browser().is_none() { @@ -1170,11 +1166,6 @@ impl BrowserRuntime { } } -pub(crate) enum BrowserBootstrap { - ExistingTarget { target_id: String, url: String }, - Provider { tab_id: crate::resource::TabPublicId, url: String }, -} - pub(crate) fn new_surface( id: SurfaceId, url: String, diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index c03ef7624b52..e219a683d45e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -27,7 +27,7 @@ use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use zeroize::Zeroize; -use crate::browser::{self, BrowserBootstrap, BrowserRuntime, BrowserSource}; +use crate::browser::{self, BrowserRuntime, BrowserSource}; use crate::browser_provider::{ BrowserProviderRegistration, BrowserProviderRegistry, BrowserProviderSnapshot, BrowserProviderTargetLease, @@ -2188,6 +2188,7 @@ impl AsyncSurfaceCreationGate { fn stop_and_wait_until(&self, deadline: Instant) -> bool { let mut state = self.inner.state.lock().unwrap(); state.shutting_down = true; + self.inner.idle.notify_all(); while state.active != 0 { let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { return false; @@ -2201,6 +2202,22 @@ impl AsyncSurfaceCreationGate { true } + #[cfg(test)] + fn wait_until_stopping(&self, deadline: Instant) -> bool { + let mut state = self.inner.state.lock().unwrap(); + while !state.shutting_down { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self.inner.idle.wait_timeout(state, remaining).unwrap(); + state = next; + if timeout.timed_out() && !state.shutting_down { + return false; + } + } + true + } + #[cfg(test)] fn wait_until_idle(&self, deadline: Instant) -> bool { let mut state = self.inner.state.lock().unwrap(); @@ -2248,8 +2265,85 @@ struct BrowserRuntimeSlot { } impl BrowserRuntimeSlot { + fn get_or_connect( + &self, + is_shutting_down: impl Fn() -> bool, + can_reuse: impl Fn(&BrowserRuntime) -> bool, + connect: impl FnOnce() -> anyhow::Result>, + ) -> anyhow::Result> { + loop { + let mut state = self.state.lock().unwrap(); + if is_shutting_down() { + anyhow::bail!("server is shutting down"); + } + match &*state { + BrowserRuntimeSlotState::Ready(runtime) if can_reuse(runtime) => { + return Ok(runtime.clone()); + } + BrowserRuntimeSlotState::Ready(_) => { + *state = BrowserRuntimeSlotState::Empty; + } + BrowserRuntimeSlotState::Empty => { + *state = BrowserRuntimeSlotState::Connecting; + break; + } + BrowserRuntimeSlotState::Connecting => { + drop(self.changed.wait(state).unwrap()); + } + BrowserRuntimeSlotState::Stopping(_) => { + anyhow::bail!("server is shutting down"); + } + } + } + + // The connector can block on process or network work. The slot owner + // has already published Connecting and released its state lock here. + let connected = connect(); + let mut state = self.state.lock().unwrap(); + // This read belongs to the Connecting -> Ready transition. Shutdown + // can start while the connector runs without holding this lock. + let shutting_down = is_shutting_down(); + match connected { + Ok(created) + if matches!(*state, BrowserRuntimeSlotState::Connecting) && !shutting_down => + { + *state = BrowserRuntimeSlotState::Ready(created.clone()); + self.changed.notify_all(); + Ok(created) + } + Ok(created) => { + self.changed.notify_all(); + drop(state); + created.shutdown(); + anyhow::bail!("server is shutting down"); + } + Err(error) => { + let stopping = + shutting_down || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); + if matches!(*state, BrowserRuntimeSlotState::Connecting) { + *state = BrowserRuntimeSlotState::Empty; + } + self.changed.notify_all(); + drop(state); + if stopping { + anyhow::bail!("server is shutting down"); + } + Err(error) + } + } + } + fn stop(&self) { + self.stop_with_fence(|| {}); + } + + fn stop_with_fence(&self, fence: impl FnOnce()) { let mut state = self.state.lock().unwrap(); + // The global shutdown fence and this owner transition share one + // critical section, so a connector publishes before both or after + // neither. There is no state where shutdown is fenced but this slot + // can still publish Ready. + fence(); let current = std::mem::take(&mut *state); *state = match current { BrowserRuntimeSlotState::Ready(runtime) => { @@ -2313,6 +2407,11 @@ impl BrowserRuntimeSlot { } } +enum BrowserSurfaceBootstrap { + ExistingTarget { runtime: Arc, target_id: String, url: String }, + Provider { tab_id: TabPublicId, url: String }, +} + enum SurfaceResizeRestore { Complete(bool), Pending(Receiver), @@ -2849,10 +2948,6 @@ pub struct Mux { #[cfg(all(test, unix))] terminal_host_record_loader: Mutex>, #[cfg(test)] - browser_bootstrap_before_runtime: Mutex>>, - #[cfg(test)] - browser_runtime_connect: Mutex>>, - #[cfg(test)] shutdown_owner_capacity: AtomicUsize, #[cfg(test)] shutdown_attempt_timeout: Mutex>, @@ -3241,10 +3336,6 @@ impl Mux { #[cfg(all(test, unix))] terminal_host_record_loader: Mutex::new(None), #[cfg(test)] - browser_bootstrap_before_runtime: Mutex::new(None), - #[cfg(test)] - browser_runtime_connect: Mutex::new(None), - #[cfg(test)] shutdown_owner_capacity: AtomicUsize::new(usize::MAX), #[cfg(test)] shutdown_attempt_timeout: Mutex::new(None), @@ -3487,8 +3578,10 @@ impl Mux { RegistryBrowserReconnect::Recreate => { self.start_browser_bootstrap( surface, - BrowserBootstrap::Provider { tab_id: content.identity.tab_id.clone(), url }, - None, + BrowserSurfaceBootstrap::Provider { + tab_id: content.identity.tab_id.clone(), + url, + }, ); } } @@ -7628,8 +7721,7 @@ impl Mux { .clone(); self.start_browser_bootstrap( surface.clone(), - BrowserBootstrap::Provider { tab_id, url }, - None, + BrowserSurfaceBootstrap::Provider { tab_id, url }, ); Ok(surface) } @@ -8644,70 +8736,6 @@ impl Mux { self.workspace_registry.lock().unwrap().set_terminal_close_failure(enabled) } - fn browser_runtime(&self) -> anyhow::Result> { - loop { - let mut state = self.browser_runtime.state.lock().unwrap(); - if self.is_shutting_down() { - anyhow::bail!("server is shutting down"); - } - match &*state { - BrowserRuntimeSlotState::Ready(runtime) if !runtime.is_closed() => { - return Ok(runtime.clone()); - } - BrowserRuntimeSlotState::Ready(_) => { - *state = BrowserRuntimeSlotState::Empty; - } - BrowserRuntimeSlotState::Empty => { - *state = BrowserRuntimeSlotState::Connecting; - break; - } - BrowserRuntimeSlotState::Connecting => { - drop(self.browser_runtime.changed.wait(state).unwrap()); - } - BrowserRuntimeSlotState::Stopping(_) => { - anyhow::bail!("server is shutting down"); - } - } - } - - let opts = self.surface_options.lock().unwrap().clone(); - #[cfg(test)] - if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { - hook(); - } - let connected = BrowserRuntime::connect(&opts); - let mut state = self.browser_runtime.state.lock().unwrap(); - match connected { - Ok(created) - if matches!(*state, BrowserRuntimeSlotState::Connecting) - && !self.is_shutting_down() => - { - *state = BrowserRuntimeSlotState::Ready(created.clone()); - self.browser_runtime.changed.notify_all(); - Ok(created) - } - Ok(created) => { - self.browser_runtime.changed.notify_all(); - drop(state); - created.shutdown(); - anyhow::bail!("server is shutting down"); - } - Err(error) => { - let stopping = self.is_shutting_down() - || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); - if matches!(*state, BrowserRuntimeSlotState::Connecting) { - *state = BrowserRuntimeSlotState::Empty; - } - self.browser_runtime.changed.notify_all(); - drop(state); - if stopping { - anyhow::bail!("server is shutting down"); - } - Err(error) - } - } - } - pub(crate) fn register_browser_provider( self: &Arc, client: u64, @@ -8756,76 +8784,20 @@ impl Mux { &self, lease: &BrowserProviderTargetLease, ) -> anyhow::Result> { - loop { - let mut state = self.browser_runtime.state.lock().unwrap(); - if self.is_shutting_down() { - anyhow::bail!("server is shutting down"); - } - match &*state { - BrowserRuntimeSlotState::Ready(runtime) - if !runtime.is_closed() - && runtime.matches_provider(&lease.endpoint, &lease.authentication) => - { - return Ok(runtime.clone()); - } - BrowserRuntimeSlotState::Ready(_) => { - *state = BrowserRuntimeSlotState::Empty; - } - BrowserRuntimeSlotState::Empty => { - *state = BrowserRuntimeSlotState::Connecting; - break; - } - BrowserRuntimeSlotState::Connecting => { - drop(self.browser_runtime.changed.wait(state).unwrap()); - } - BrowserRuntimeSlotState::Stopping(_) => { - anyhow::bail!("server is shutting down"); - } - } - } - - #[cfg(test)] - if let Some(hook) = self.browser_runtime_connect.lock().unwrap().clone() { - hook(); - } - let connected = BrowserRuntime::connect_provider(&lease.endpoint, &lease.authentication); - let mut state = self.browser_runtime.state.lock().unwrap(); - match connected { - Ok(created) - if matches!(*state, BrowserRuntimeSlotState::Connecting) - && !self.is_shutting_down() => - { - *state = BrowserRuntimeSlotState::Ready(created.clone()); - self.browser_runtime.changed.notify_all(); - Ok(created) - } - Ok(created) => { - self.browser_runtime.changed.notify_all(); - drop(state); - created.shutdown(); - anyhow::bail!("server is shutting down"); - } - Err(error) => { - let stopping = self.is_shutting_down() - || matches!(*state, BrowserRuntimeSlotState::Stopping(_)); - if matches!(*state, BrowserRuntimeSlotState::Connecting) { - *state = BrowserRuntimeSlotState::Empty; - } - self.browser_runtime.changed.notify_all(); - drop(state); - if stopping { - anyhow::bail!("server is shutting down"); - } - Err(error) - } - } + self.browser_runtime.get_or_connect( + || self.is_shutting_down(), + |runtime| { + !runtime.is_closed() + && runtime.matches_provider(&lease.endpoint, &lease.authentication) + }, + || BrowserRuntime::connect_provider(&lease.endpoint, &lease.authentication), + ) } fn start_browser_bootstrap( self: &Arc, surface: Arc, - bootstrap: BrowserBootstrap, - runtime: Option>, + bootstrap: BrowserSurfaceBootstrap, ) { let bootstrap_guard = match self.async_surface_creations.begin() { Ok(guard) => guard, @@ -8834,7 +8806,7 @@ impl Mux { return; } }; - let provider_bootstrap = matches!(&bootstrap, BrowserBootstrap::Provider { .. }); + let provider_bootstrap = matches!(&bootstrap, BrowserSurfaceBootstrap::Provider { .. }); let weak_mux = Arc::downgrade(self); let providers = self.browser_providers.clone(); let id = surface.id; @@ -8845,11 +8817,7 @@ impl Mux { let _bootstrap_guard = bootstrap_guard; let result = (|| -> anyhow::Result<()> { match bootstrap { - BrowserBootstrap::Provider { tab_id, url } => { - anyhow::ensure!( - runtime.is_none(), - "provider bootstrap cannot override its CDP runtime" - ); + BrowserSurfaceBootstrap::Provider { tab_id, url } => { let mut retry_delay = Duration::from_millis(250); loop { let canceled = || { @@ -8878,10 +8846,8 @@ impl Mux { ); runtime.bootstrap_surface_sync( thread_surface.clone(), - BrowserBootstrap::ExistingTarget { - target_id: lease.target_id.clone(), - url: url.clone(), - }, + lease.target_id.clone(), + url.clone(), weak_mux.clone(), ) })(); @@ -8943,17 +8909,14 @@ impl Mux { } } } - bootstrap => { - let mux = weak_mux + BrowserSurfaceBootstrap::ExistingTarget { runtime, target_id, url } => { + let _mux = weak_mux .upgrade() .ok_or_else(|| anyhow::anyhow!("browser mux was dropped"))?; - let runtime = match runtime { - Some(runtime) => runtime, - None => mux.browser_runtime()?, - }; runtime.bootstrap_surface_sync( thread_surface.clone(), - bootstrap, + target_id, + url, weak_mux.clone(), ) } @@ -9003,7 +8966,7 @@ impl Mux { } let tab_id = identity.tab_id.clone(); let url = surface.browser_url().unwrap_or_else(|| "about:blank".to_string()); - self.start_browser_bootstrap(surface, BrowserBootstrap::Provider { tab_id, url }, None); + self.start_browser_bootstrap(surface, BrowserSurfaceBootstrap::Provider { tab_id, url }); } /// A fresh single-tab pane wrapping `surface`. @@ -10173,8 +10136,7 @@ impl Mux { crate::process_session::require_stable_process_signaling_until(deadline) .context("preflight process control for daemon exit")?; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; - self.shutting_down.store(true, Ordering::Release); - self.browser_runtime.stop(); + self.browser_runtime.stop_with_fence(|| self.shutting_down.store(true, Ordering::Release)); self.journal_kernel.wake_waiters(); let hook_deadline = deadline.min(Instant::now() + crate::journal_hooks::SHUTDOWN_WAIT); if !self.journal_hook_runtime.shutdown_until(hook_deadline) { @@ -10407,8 +10369,7 @@ impl Mux { .context("preflight process control for server shutdown")?; let _coordinator = self.lock_shutdown_coordinator_until(deadline)?; let cleanup_lifecycle = self.shutdown_cleanup_lifecycle.begin(); - self.shutting_down.store(true, Ordering::Release); - self.browser_runtime.stop(); + self.browser_runtime.stop_with_fence(|| self.shutting_down.store(true, Ordering::Release)); #[cfg(unix)] self.request_terminal_adoption_stop(); if !self.surface_creations.stop_and_wait_until(deadline) { @@ -10754,10 +10715,9 @@ impl Mux { /// and remain available for the replacement daemon to adopt. pub fn request_daemon_shutdown(&self) { self.shutdown_cleanup_lifecycle.schedule(); - self.shutting_down.store(true, Ordering::Release); + self.browser_runtime.stop_with_fence(|| self.shutting_down.store(true, Ordering::Release)); self.surface_creations.stop(); self.async_surface_creations.stop(); - self.browser_runtime.stop(); self.daemon_shutdown_requested.store(true, Ordering::Release); #[cfg(unix)] self.request_terminal_adoption_stop(); @@ -13822,8 +13782,7 @@ impl Mux { } self.start_browser_bootstrap( surface, - BrowserBootstrap::ExistingTarget { target_id, url }, - Some(runtime), + BrowserSurfaceBootstrap::ExistingTarget { runtime, target_id, url }, ); Ok(true) } @@ -23890,15 +23849,6 @@ mod tests { let mux = test_mux(); let initial = mux.new_workspace(None, Some((80, 24))).unwrap(); let pane = mux.with_state(|state| state.pane_of(initial.id).unwrap()); - let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); - let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); - let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); - *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ - move || { - bootstrap_reached_tx.send(()).unwrap(); - release_bootstrap_rx.lock().unwrap().recv().unwrap(); - } - })); *mux.browser_tab_after_spawn.lock().unwrap() = Some(Arc::new({ let mux = Arc::downgrade(&mux); let runtime = runtime.clone(); @@ -23914,9 +23864,7 @@ mod tests { })); let result = mux.new_browser_tab("about:blank".into(), Some(pane), Some((80, 24))); - bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); mux.request_daemon_shutdown(); - release_bootstrap_tx.send(()).unwrap(); assert!( mux.async_surface_creations.wait_until_idle(Instant::now() + Duration::from_secs(1)), "async browser creation did not finish" @@ -27911,19 +27859,9 @@ mod tests { } #[test] - fn server_shutdown_waits_for_async_browser_bootstrap() { + fn server_shutdown_waits_for_active_async_surface_creation() { let mux = test_mux(); - let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); - let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); - let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); - *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ - move || { - bootstrap_reached_tx.send(()).unwrap(); - release_bootstrap_rx.lock().unwrap().recv().unwrap(); - } - })); - mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); - bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let creation = mux.async_surface_creations.begin().unwrap(); let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); let shutdown = std::thread::spawn({ @@ -27934,48 +27872,41 @@ mod tests { }); assert!(shutdown_done_rx.recv_timeout(Duration::from_millis(100)).is_err()); - release_bootstrap_tx.send(()).unwrap(); - assert_eq!(shutdown_done_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(), 1); + drop(creation); + assert_eq!(shutdown_done_rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(), 0); shutdown.join().unwrap(); assert!(!mux.browser_runtime.has_runtime_for_test()); } #[test] - fn daemon_exit_retries_until_async_browser_bootstrap_releases_ownership() { + fn daemon_exit_waits_for_async_surface_creation_release() { let mux = test_mux(); - mux.set_shutdown_attempt_timeout_for_test(crate::test_timeout(Duration::from_millis(25))); - let (bootstrap_reached_tx, bootstrap_reached_rx) = std::sync::mpsc::sync_channel(1); - let (release_bootstrap_tx, release_bootstrap_rx) = std::sync::mpsc::sync_channel(1); - let release_bootstrap_rx = Arc::new(Mutex::new(release_bootstrap_rx)); - *mux.browser_bootstrap_before_runtime.lock().unwrap() = Some(Arc::new({ - move || { - bootstrap_reached_tx.send(()).unwrap(); - release_bootstrap_rx.lock().unwrap().recv().unwrap(); - } - })); - mux.new_browser_tab("about:blank".into(), None, Some((80, 24))).unwrap(); - bootstrap_reached_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + let creation = mux.async_surface_creations.begin().unwrap(); let (shutdown_done_tx, shutdown_done_rx) = std::sync::mpsc::sync_channel(1); let shutdown = std::thread::spawn({ let mux = mux.clone(); move || { - mux.shutdown().unwrap(); - shutdown_done_tx.send(()).unwrap(); + shutdown_done_tx.send(mux.shutdown()).unwrap(); } }); - let finished_early = - shutdown_done_rx.recv_timeout(crate::test_timeout(Duration::from_millis(100))).is_ok(); - release_bootstrap_tx.send(()).unwrap(); - if !finished_early { - shutdown_done_rx.recv_timeout(crate::test_timeout(Duration::from_secs(2))).unwrap(); - } - shutdown.join().unwrap(); - assert!( - !finished_early, - "daemon exit completed while browser bootstrap still owned in-flight work" + mux.async_surface_creations + .wait_until_stopping(Instant::now() + Duration::from_secs(1)), + "daemon exit did not stop the async creation owner" + ); + assert!( + matches!(shutdown_done_rx.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)), + "daemon exit completed while an async creation still owned in-flight work" ); + + drop(creation); + shutdown_done_rx + .recv_timeout(crate::test_timeout(Duration::from_secs(2))) + .unwrap() + .unwrap(); + shutdown.join().unwrap(); + assert!(!mux.browser_runtime.has_runtime_for_test()); } @@ -28037,52 +27968,109 @@ mod tests { #[test] fn browser_runtime_connection_does_not_hold_the_shared_slot() { - let mux = Mux::new_for_test("browser-runtime-slot", SurfaceOptions::default()); + let slot = Arc::new(BrowserRuntimeSlot::default()); let (connect_started_tx, connect_started_rx) = std::sync::mpsc::sync_channel(1); let (release_connect_tx, release_connect_rx) = std::sync::mpsc::sync_channel(1); - let release_connect_rx = Arc::new(Mutex::new(release_connect_rx)); - *mux.browser_runtime_connect.lock().unwrap() = Some(Arc::new({ - move || { - connect_started_tx.send(()).unwrap(); - release_connect_rx.lock().unwrap().recv().unwrap(); - } - })); - let lease = BrowserProviderTargetLease { - provider_id: "browser-runtime-slot".into(), - endpoint: "ws://127.0.0.1:9/devtools/browser/unreachable".into(), - authentication: crate::browser_provider::BrowserProviderAuthentication::None, - tab_id: TabPublicId::parse("tab_00000000000000000000000000000001").unwrap(), - target_id: "target-1".into(), - revision: 1, - }; let (connect_done_tx, connect_done_rx) = std::sync::mpsc::sync_channel(1); let connect = std::thread::spawn({ - let mux = mux.clone(); + let slot = slot.clone(); move || { - connect_done_tx.send(mux.browser_runtime_for_provider(&lease).is_err()).unwrap(); + let result = slot.get_or_connect( + || false, + |_| false, + || -> anyhow::Result> { + connect_started_tx.send(()).unwrap(); + release_connect_rx.recv().unwrap(); + anyhow::bail!("test connector released") + }, + ); + connect_done_tx.send(result.is_err()).unwrap(); } }); - let connect_started = connect_started_rx.recv_timeout(Duration::from_secs(1)).is_ok(); + connect_started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); - let slot_available = mux.browser_runtime.lock_available_for_test(); - let _ = release_connect_tx.send(()); - let connect_failed = connect_done_rx.recv_timeout(Duration::from_secs(1)).ok(); - if connect_failed.is_some() { - connect.join().unwrap(); - } - mux.request_daemon_shutdown(); - mux.shutdown().unwrap(); + let slot_available = slot.lock_available_for_test(); + release_connect_tx.send(()).unwrap(); + let connect_failed = connect_done_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + connect.join().unwrap(); - assert!(connect_started, "provider connection did not reach the blocking test hook"); assert!( slot_available, "browser connection setup held the shared runtime slot across blocking work" ); - assert_eq!( - connect_failed, - Some(true), - "provider connection did not fail within the final bound" + assert!(connect_failed, "test connector result was not published"); + } + + #[test] + fn browser_runtime_connection_does_not_publish_after_shutdown_starts() { + fn read_ws_json(ws: &mut tungstenite::WebSocket) -> Value { + loop { + match ws.read().unwrap() { + tungstenite::Message::Text(text) => { + return serde_json::from_str(&text).unwrap(); + } + tungstenite::Message::Binary(bytes) => { + return serde_json::from_slice(&bytes).unwrap(); + } + _ => {} + } + } + } + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut ws = tungstenite::accept(stream).unwrap(); + let discover = read_ws_json(&mut ws); + assert_eq!(discover["method"], "Target.setDiscoverTargets"); + ws.send(tungstenite::Message::Text( + serde_json::json!({"id": discover["id"], "result": {}}).to_string().into(), + )) + .unwrap(); + while ws.read().is_ok() {} + }); + let runtime = BrowserRuntime::connect_external_for_test(&format!( + "ws://{address}/devtools/browser/fake" + )) + .unwrap(); + let slot = Arc::new(BrowserRuntimeSlot::default()); + let shutting_down = Arc::new(AtomicBool::new(false)); + let (connect_started_tx, connect_started_rx) = std::sync::mpsc::sync_channel(1); + let (release_connect_tx, release_connect_rx) = std::sync::mpsc::sync_channel(1); + let (connect_done_tx, connect_done_rx) = std::sync::mpsc::sync_channel(1); + let connect = std::thread::spawn({ + let slot = slot.clone(); + let shutting_down = shutting_down.clone(); + let runtime = runtime.clone(); + move || { + let result = slot.get_or_connect( + || shutting_down.load(Ordering::Acquire), + |_| false, + || { + connect_started_tx.send(()).unwrap(); + release_connect_rx.recv().unwrap(); + Ok(runtime) + }, + ); + connect_done_tx.send(result.is_err()).unwrap(); + } + }); + connect_started_rx.recv_timeout(Duration::from_secs(1)).unwrap(); + + slot.stop_with_fence(|| shutting_down.store(true, Ordering::Release)); + release_connect_tx.send(()).unwrap(); + assert!(connect_done_rx.recv_timeout(Duration::from_secs(1)).unwrap()); + connect.join().unwrap(); + assert!( + !slot.has_runtime_for_test(), + "browser runtime was published after shutdown started" ); + assert!( + runtime.wait_until_closed(Instant::now() + Duration::from_secs(1)), + "browser runtime created during shutdown was not closed" + ); + server.join().unwrap(); } #[test] From c89bf5c83c12c043d9050b9925eca7052848e6f1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:49:53 -0700 Subject: [PATCH 461/521] test(macOS): cover nil AppKit subview insertion --- .../PortalHitTestingPerformanceTests.swift | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/cmuxTests/PortalHitTestingPerformanceTests.swift b/cmuxTests/PortalHitTestingPerformanceTests.swift index 8b98488d7ccf..80828a21f520 100644 --- a/cmuxTests/PortalHitTestingPerformanceTests.swift +++ b/cmuxTests/PortalHitTestingPerformanceTests.swift @@ -1,4 +1,5 @@ import AppKit +import ObjectiveC import Testing #if canImport(cmux_DEV) @@ -10,6 +11,14 @@ import Testing @MainActor @Suite(.serialized) struct PortalHitTestingPerformanceTests { + private typealias PositionedSubviewImplementation = @convention(c) ( + AnyObject, + Selector, + AnyObject?, + Int, + AnyObject? + ) -> Void + private final class CapturingView: NSView { override func hitTest(_ point: NSPoint) -> NSView? { bounds.contains(point) ? self : nil @@ -520,6 +529,36 @@ struct PortalHitTestingPerformanceTests { withExtendedLifetime(insertedSubtree.splitDelegate) {} } + @Test + func nilPositionedSubviewFromAppKitDoesNotCrashHierarchyTracking() throws { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 320, height: 180), + styleMask: [.titled, .closable], + backing: .buffered, + defer: false + ) + defer { window.orderOut(nil) } + + let rootView = try #require(window.contentView) + let invalidator = PortalSplitDividerCacheInvalidator() + invalidator.observe( + rootView: rootView, + geometryViews: [rootView], + hierarchyNodes: [(view: rootView, containsSplitView: false)], + onChange: {} + ) + + let selector = #selector(NSView.addSubview(_:positioned:relativeTo:)) + let method = try #require(class_getInstanceMethod(NSView.self, selector)) + let implementation = unsafeBitCast( + method_getImplementation(method), + to: PositionedSubviewImplementation.self + ) + implementation(rootView, selector, nil, NSWindow.OrderingMode.above.rawValue, nil) + + #expect(invalidator.isHierarchyCurrent(for: rootView)) + } + @Test func hierarchyMutationHooksGateBeforeWalkingAnUnrelatedWindow() throws { let window = NSWindow( From 304992a99a56a230e8db819901bf6f52dadc84ae Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:51:04 -0700 Subject: [PATCH 462/521] fix(macOS): tolerate nil AppKit subview hooks --- Sources/PortalSplitDividerCacheInvalidator.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/PortalSplitDividerCacheInvalidator.swift b/Sources/PortalSplitDividerCacheInvalidator.swift index a9bf69a600f2..e979ff410b23 100644 --- a/Sources/PortalSplitDividerCacheInvalidator.swift +++ b/Sources/PortalSplitDividerCacheInvalidator.swift @@ -9,9 +9,10 @@ private typealias PortalSubviewComparator = @convention(c) ( private extension NSView { @objc(cmux_portalAddSubview:) - func cmux_portalAddSubview(_ subview: NSView) { - let previousWindow = subview.window + func cmux_portalAddSubview(_ subview: NSView?) { + let previousWindow = subview?.window cmux_portalAddSubview(subview) + guard let subview else { return } PortalViewHierarchyMutationTracker.recordInsertion( parentView: self, insertedView: subview, @@ -21,12 +22,13 @@ private extension NSView { @objc(cmux_portalAddSubview:positioned:relativeTo:) func cmux_portalAddSubview( - _ subview: NSView, + _ subview: NSView?, positioned place: NSWindow.OrderingMode, relativeTo otherView: NSView? ) { - let previousWindow = subview.window + let previousWindow = subview?.window cmux_portalAddSubview(subview, positioned: place, relativeTo: otherView) + guard let subview else { return } PortalViewHierarchyMutationTracker.recordInsertion( parentView: self, insertedView: subview, From 0ca0d11acdb9c12e1e183567d5a1a628478d462f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 03:55:37 -0700 Subject: [PATCH 463/521] test(tui): detach explicit attach fixture --- cmux-tui/crates/cmux-tui/tests/cli.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 2180efad6cea..65a91969c4f5 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -3961,6 +3961,9 @@ fn explicit_attach_registers_a_full_session_tui_client() { } assert!(attached.iter().any(|id| id.as_str() == Some(terminal.as_str()))); assert!(attached.iter().any(|id| id.as_str() == Some(second_terminal.as_str()))); + let client_id = client["id"].as_str().expect("TUI client public id").to_string(); + let detached = cli(&server, &["--quiet", "client", client_id.as_str(), "detach"]); + assert_success(&detached); return; } } From 08a03da896d00e3adb4475c2a87e574c034621d2 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:34:42 -0700 Subject: [PATCH 464/521] test(tui): retain untracked shutdown fixture owner --- cmux-tui/crates/cmux-tui-core/src/surface.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/surface.rs b/cmux-tui/crates/cmux-tui-core/src/surface.rs index 660cbaef4963..91e2627b391e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/surface.rs +++ b/cmux-tui/crates/cmux-tui-core/src/surface.rs @@ -6503,10 +6503,12 @@ impl Surface { /// resource lifetime. Hosted runtimes have no local process owner here. pub(crate) fn local_process_shutdown_owner(&self) -> Option { let Surface::Pty(pty) = self else { return None }; - let Some(LocalProcess::Owned(process)) = pty.local_process.as_deref() else { return None }; - Some(SurfaceShutdownOwner { - kind: SurfaceShutdownOwnerKind::Local(LocalProcess::Owned(process.clone())), - }) + let process = match pty.local_process.as_deref()? { + LocalProcess::Owned(process) => LocalProcess::Owned(process.clone()), + #[cfg(test)] + LocalProcess::Untracked(killer) => LocalProcess::Untracked(killer.clone()), + }; + Some(SurfaceShutdownOwner { kind: SurfaceShutdownOwnerKind::Local(process) }) } #[cfg(test)] From 724749b26ecfbaee46d52969ab1933c19b6174a1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:54:24 -0700 Subject: [PATCH 465/521] fix(tui): retain failed attachment owners --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 30 +++++++----------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e219a683d45e..4df02b909bab 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9195,25 +9195,13 @@ impl Mux { _operation: &str, reason: &str, ) -> anyhow::Result<()> { - let Some(identity) = self.resource_terminal_host_identity(surface) else { - let removed = { - let mut state = self.state.lock().unwrap(); - remove_terminal_runtime_from_state(self, &mut state, surface).0 - }; - for placement in removed { - self.purge_surface_side_tables(placement.id); - } - self.purge_terminal_runtime_side_tables(surface); - if !surface.is_dead() { - surface.kill(); - } - return Ok(()); - }; - self.persist_terminal_exit( - &identity.terminal_id, - Some(&identity.incarnation), - &TerminalExit::unknown(reason), - )?; + if let Some(identity) = self.resource_terminal_host_identity(surface) { + self.persist_terminal_exit( + &identity.terminal_id, + Some(&identity.incarnation), + &TerminalExit::unknown(reason), + )?; + } let removed = { let mut state = self.state.lock().unwrap(); remove_terminal_runtime_from_state(self, &mut state, surface).0 @@ -9222,9 +9210,7 @@ impl Mux { self.purge_surface_side_tables(placement.id); } self.purge_terminal_runtime_side_tables(surface); - if !surface.is_dead() { - surface.kill(); - } + self.retire_surface_runtime(surface.clone()); Ok(()) } From 44ae65d733daa1c3b783926856631d1b533697fa Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:22:05 -0700 Subject: [PATCH 466/521] test(tui): close quota owners explicitly --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4df02b909bab..7f22e16a0d7e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -21854,11 +21854,11 @@ mod tests { ); *mux.kitty_image_budget_operation.lock().unwrap() = None; - assert!(mux.close_surface(replacement.id).unwrap()); + close_terminal_runtime_for_test(&mux, &replacement); for surface in survivors { - assert!(mux.close_surface(surface.id).unwrap()); + close_terminal_runtime_for_test(&mux, &surface); } - assert!(mux.close_surface(first.id).unwrap()); + close_terminal_runtime_for_test(&mux, &first); wait_for_kitty_image_budget(&mux); } From 4671bb229cf6f957717f721d2b20d13ad6923649 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:10:17 -0700 Subject: [PATCH 467/521] test(tui): retain failed quota cleanup owner --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 7f22e16a0d7e..d0c2ed70fb2a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -22331,12 +22331,15 @@ mod tests { while mux.kitty_image_budget.lock().unwrap().worker_running && Instant::now() < deadline { std::thread::sleep(Duration::from_millis(5)); } - let (stopped, survivor_blocked, closed_retired) = { + let (stopped, survivor_owns_quota, closed_cleanup_retained) = { let budget = mux.kitty_image_budget.lock().unwrap(); ( !budget.worker_running, - budget.blocked_surfaces.contains(&first.id), - !budget.entries.contains_key(&second.id), + budget.entries.get(&first.id).is_some_and(|entry| { + entry.owns_quota && !budget.blocked_surfaces.contains(&first.id) + }), + budget.entries.get(&second.id).is_some_and(|entry| entry.removing) + && budget.blocked_surfaces.contains(&second.id), ) }; @@ -22350,19 +22353,15 @@ mod tests { state.admitted_jobs = 0; mux.deadline_fanout_pool.inner.changed.notify_all(); } - let cleanup_deadline = Instant::now() + Duration::from_secs(1); - while mux.kitty_image_budget.lock().unwrap().worker_running - && Instant::now() < cleanup_deadline - { - std::thread::sleep(Duration::from_millis(5)); - } - assert!(stopped, "Kitty quota worker retried pool admission forever"); assert!( - survivor_blocked, - "pool admission exhaustion did not fail closed for the live surface" + survivor_owns_quota, + "a closing surface's failed cleanup disabled an unaffected live surface" + ); + assert!( + closed_cleanup_retained, + "pool admission exhaustion discarded the closing surface's cleanup owner" ); - assert!(closed_retired, "a closed surface remained in the Kitty quota registry"); } #[test] From d0779337d5d2c80eceb8ab5c522b6a8ecfb02a4f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:48:37 -0700 Subject: [PATCH 468/521] test(tui): keep undone terminals catalog-owned --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index d0c2ed70fb2a..f6476fe00755 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -24820,7 +24820,7 @@ mod tests { LayoutUndoResult::Undone { .. } )); - assert!(mux.surface(added.id).is_none()); + assert_terminal_view_detached(&mux, added.id); assert!( mux.shutdown_owners.is_empty(), "layout undo retained the removed surface owner after confirmed shutdown" From 5dccd410609b5476467987d70cf9a9f00954b14a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:01:56 -0700 Subject: [PATCH 469/521] test(macOS): retain repeated stalled recovery --- .../TerminalSurfaceRestoreSpawnSchedulerTests.swift | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 3cab9fdce0f2..2052fae1b6bf 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -837,6 +837,16 @@ import CmuxTerminalCore rejectedAfterStall.surface .runtimeSurfaceAdmissionDeferredCreationSource != nil ) + + deferredFixtures[0].surface.createSurface( + for: deferredFixtures[0].nativeView + ) + scheduler.runScheduledOperation(at: deferredFixtures.count + 1) + #expect( + deferredFixtures[0].paneHost.runtimeSurfaceCreationFailureMessages + == [expectedMessage], + "repeated creation discarded its existing primary recovery owner" + ) let firstDeferredAttemptCount = deferredFixtures[0].surface .debugRuntimeSurfaceCreateAttemptCountForTesting() From ed1f16d53d5b315c9fcfc785e8130aa9da771000 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:02:25 -0700 Subject: [PATCH 470/521] fix(macOS): retain repeated stalled recovery --- .../Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 615c0c025840..872b1df19185 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -77,6 +77,11 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { claimedCapacity = false } guard !state.closeTeardownAllStalled else { + if state.recoveryEntriesByID[recoveryID] != nil { + state.recoveryEntriesByID[recoveryID]?.action = onRecovery + state.recoveryEntriesByID[recoveryID]?.failure = onFailure + return (.deferred, claimedCapacity) + } return (.closeTeardownStalled, claimedCapacity) } if !state.closeTeardownDegraded, From 407ae05b678a83174f2a5383488dbcfe7eb7e0e9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:16:18 -0700 Subject: [PATCH 471/521] test(macOS): preserve stalled recovery order --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 2052fae1b6bf..0f7a329ef2ec 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -892,6 +892,88 @@ import CmuxTerminalCore #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) } + @Test func repeatedStalledRecoveryUsesLatestFailureDelivery() throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let owners = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let recoveryID = UUID() + var deliveredFailures: [String] = [] + #expect( + coordinator.reserveRuntimeSurfaceOwnership( + recoveryID: recoveryID, + onRecovery: { reservation in + coordinator.cancelRuntimeSurfaceOwnership(reservation) + }, + onFailure: { deliveredFailures.append("original") } + ) == .deferred + ) + let deliveries = coordinator.runtimeOwnershipAdmission + .failRecoveriesForAllStalledCloseTeardowns() + let delivery = try #require(deliveries.first) + + #expect( + coordinator.reserveRuntimeSurfaceOwnership( + recoveryID: recoveryID, + onRecovery: { reservation in + coordinator.cancelRuntimeSurfaceOwnership(reservation) + }, + onFailure: { deliveredFailures.append("latest") } + ) == .deferred + ) + delivery() + + #expect(deliveredFailures == ["latest"]) + coordinator.runtimeOwnershipAdmission + .completeStalledCloseRecoveryFailures([delivery]) + coordinator.cancelRuntimeSurfaceOwnershipRecovery(recoveryID) + coordinator.runtimeOwnershipAdmission.clearAllStalledCloseTeardowns() + for owner in owners { + coordinator.cancelRuntimeSurfaceOwnership(owner) + } + } + + @Test func newRecoveryDefersBehindScheduledOlderGrant() async throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let owners = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let olderRecoveryIDs = (0..<2).map { _ in UUID() } + for recoveryID in olderRecoveryIDs { + #expect( + coordinator.reserveRuntimeSurfaceOwnership( + recoveryID: recoveryID, + onRecovery: { reservation in + coordinator.cancelRuntimeSurfaceOwnership(reservation) + } + ) == .deferred + ) + } + + coordinator.cancelRuntimeSurfaceOwnership(owners[0]) + coordinator.cancelRuntimeSurfaceOwnership(owners[1]) + let newestRecoveryID = UUID() + let newestResult = coordinator.reserveRuntimeSurfaceOwnership( + recoveryID: newestRecoveryID, + onRecovery: { reservation in + coordinator.cancelRuntimeSurfaceOwnership(reservation) + } + ) + if case .reserved(let reservation) = newestResult { + coordinator.cancelRuntimeSurfaceOwnership(reservation) + } + + #expect(newestResult == .deferred) + for recoveryID in olderRecoveryIDs + [newestRecoveryID] { + coordinator.cancelRuntimeSurfaceOwnershipRecovery(recoveryID) + } + await waitForMainActorQueueBarrier() + } + @Test func completedFailureBatchRescansOverflowAndIgnoresStaleFailure() async throws { let capacity = 34 let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( From 6d4fade2e9144a252e2c83af5a53c90189c75969 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:17:31 -0700 Subject: [PATCH 472/521] fix(macOS): preserve recovery delivery order --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 872b1df19185..71f56da43743 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -85,6 +85,8 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { return (.closeTeardownStalled, claimedCapacity) } if !state.closeTeardownDegraded, + state.recoveryHeadID == nil, + !state.recoveryGrantIsScheduled, state.reservationIDs.count < maximumOwnerCount, state.ingressReservationIDs.count < maximumOwnerCount { @@ -271,11 +273,17 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { let entry = state.recoveryEntriesByID[currentID] { recoveryID = entry.nextID guard entry.pendingFailureDeliveryID == nil else { continue } + let deliveryID = UUID() let delivery = TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery( recoveryID: currentID, - deliveryID: UUID(), - failure: entry.failure + deliveryID: deliveryID, + failure: { [self] in + deliverStalledCloseRecoveryFailure( + recoveryID: currentID, + deliveryID: deliveryID + ) + } ) state.recoveryEntriesByID[currentID]? .pendingFailureDeliveryID = delivery.deliveryID @@ -294,6 +302,22 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } } + @MainActor + private func deliverStalledCloseRecoveryFailure( + recoveryID: UUID, + deliveryID: UUID + ) { + let failure: TerminalSurfaceRuntimeOwnershipRecoveryFailure? = + state.withLock { state in + guard let entry = state.recoveryEntriesByID[recoveryID], + entry.pendingFailureDeliveryID == deliveryID else { + return nil + } + return entry.failure + } + failure?() + } + func completeStalledCloseRecoveryFailures( _ completedDeliveries: [ TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery From 2956e1ea2e4959c3858349b51ccfd35282044721 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:35:18 -0700 Subject: [PATCH 473/521] test(tui): retry committed cleanup receipts --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index f6476fe00755..4945d947e2f7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -9339,9 +9339,7 @@ impl Mux { records: TerminalHostRecords, ) -> anyhow::Result<()> { #[cfg(test)] - if let Some(hook) = self.terminal_host_cleanup_hook.lock().unwrap().clone() { - anyhow::ensure!(hook(), "forced terminal-host cleanup failure"); - } + self.run_terminal_host_cleanup_hook()?; #[cfg(unix)] { for (path, record) in records { @@ -9379,6 +9377,8 @@ impl Mux { &self, result: &Value, ) -> anyhow::Result<()> { + #[cfg(test)] + self.run_terminal_host_cleanup_hook()?; let identities = terminal_host_cleanup_identities(result)?; let mut runtimes = Vec::new(); let mut removed = Vec::new(); @@ -9413,6 +9413,14 @@ impl Mux { self.terminate_discovered_terminal_hosts(&missing) } + #[cfg(test)] + fn run_terminal_host_cleanup_hook(&self) -> anyhow::Result<()> { + if let Some(hook) = self.terminal_host_cleanup_hook.lock().unwrap().clone() { + anyhow::ensure!(hook(), "forced terminal-host cleanup failure"); + } + Ok(()) + } + fn terminate_terminal_runtime(&self, runtime: &Arc) { let identity = self.resource_terminal_host_identity(runtime); #[cfg(unix)] From 09cba8e33493f01d2a8c3ee00bcc1490b621fcc8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:56:37 -0700 Subject: [PATCH 474/521] test(tui): await configured server exit event (#9995) --- cmux-tui/crates/cmux-tui/tests/cli.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index 65a91969c4f5..0beadf53792a 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -4061,17 +4061,12 @@ fn configured_websocket_server_does_not_attach_to_existing_session() { &["--socket", server.socket.to_str().unwrap()], &[("CMUX_TUI_CONFIG", config.as_os_str())], ); - let deadline = Instant::now() + Duration::from_secs(10); - - while Instant::now() < deadline { - if let Some(status) = tui.child.try_wait().unwrap() { - assert!(!status.success(), "server launch unexpectedly succeeded"); - return; - } - std::thread::sleep(Duration::from_millis(50)); - } - - panic!("configured WebSocket server attached instead of preserving server mode"); + let status = wait_for_child_exit( + native_pty_child(tui.child.as_mut()), + Duration::from_secs(10), + "configured WebSocket server attached instead of preserving server mode", + ); + assert!(!status.success(), "server launch unexpectedly succeeded"); } #[cfg(unix)] From 88abaefaeee1fa30a212ba0291c23648fd4e692c Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:04:46 -0700 Subject: [PATCH 475/521] test(tui): target terminal shutdown ownership --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 4945d947e2f7..fb5b85e6b623 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -26302,13 +26302,14 @@ mod tests { } #[test] - fn ordinary_surface_close_retains_failed_process_ownership_until_reconciled() { + fn ordinary_terminal_close_retains_failed_process_ownership_until_reconciled() { let mux = test_mux(); let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let owned = mux.surface(surface.id).unwrap(); + let identity = mux.resource_terminal_host_identity(&owned).unwrap(); owned.set_server_shutdown_failure_for_test(true); - assert!(mux.close_surface(surface.id).unwrap()); + mux.close_terminal(&identity.terminal_id, &identity.incarnation).unwrap(); assert!(mux.surface(surface.id).is_none()); assert_eq!(mux.shutdown_owners.len(), 1); @@ -26318,13 +26319,14 @@ mod tests { } #[test] - fn ordinary_surface_close_reconciles_a_transient_termination_failure() { + fn ordinary_terminal_close_reconciles_a_transient_termination_failure() { let mux = test_mux(); let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let owned = mux.surface(surface.id).unwrap(); + let identity = mux.resource_terminal_host_identity(&owned).unwrap(); let (failing, attempts) = owned.set_recovering_server_shutdown_for_test(); - assert!(mux.close_surface(surface.id).unwrap()); + mux.close_terminal(&identity.terminal_id, &identity.incarnation).unwrap(); assert_eq!(mux.shutdown_owners.len(), 1); assert_eq!(attempts.load(), 1); @@ -26342,14 +26344,15 @@ mod tests { } #[test] - fn retained_shutdown_owners_bound_future_surface_admission() { + fn retained_terminal_shutdown_owners_bound_future_surface_admission() { let mux = test_mux(); mux.set_shutdown_owner_capacity_for_test(1); let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let owned = mux.surface(surface.id).unwrap(); + let identity = mux.resource_terminal_host_identity(&owned).unwrap(); owned.set_server_shutdown_failure_for_test(true); - assert!(mux.close_surface(surface.id).unwrap()); + mux.close_terminal(&identity.terminal_id, &identity.incarnation).unwrap(); assert_eq!(mux.shutdown_owners.len(), 1); let error = mux.new_workspace(None, Some((80, 24))).unwrap_err(); @@ -26357,13 +26360,14 @@ mod tests { } #[test] - fn permanent_shutdown_owner_failure_stops_automatic_retry_sweeps() { + fn permanent_terminal_shutdown_owner_failure_stops_automatic_retry_sweeps() { let mux = test_mux(); let surface = mux.new_workspace(None, Some((80, 24))).unwrap(); let owned = mux.surface(surface.id).unwrap(); + let identity = mux.resource_terminal_host_identity(&owned).unwrap(); let (_failing, attempts) = owned.set_recovering_server_shutdown_for_test(); - assert!(mux.close_surface(surface.id).unwrap()); + mux.close_terminal(&identity.terminal_id, &identity.incarnation).unwrap(); assert!( mux.shutdown_owner_reconciler .wait_until_degraded(Instant::now() + Duration::from_secs(1)), From 2866d4a339208893ecad7ac36e8d73aaa373a44f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:40:31 -0700 Subject: [PATCH 476/521] fix(tui): retain terminal close ownership --- cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index db07d57ba460..c9a0cf452a22 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2814,7 +2814,7 @@ impl Mux { } if let Some(runtime) = effects.terminal_runtime { self.purge_terminal_runtime_side_tables(&runtime); - self.terminate_terminal_runtime(&runtime); + self.retire_surface_runtime(runtime); } let host_termination = self.terminate_prepared_terminal_hosts(host_cleanup); match effects.tree_publication { From d95c93afbcf2a4a16e28f6c587234058b566abac Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:48:30 -0700 Subject: [PATCH 477/521] test(tui): cover all process barrier launches --- .../crates/cmux-tui-core/src/journal_hooks.rs | 37 +++++++++++++++++++ .../crates/cmux-tui/src/agent_hook_install.rs | 37 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs index d20d8f2f81c6..fedf27543afa 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs @@ -1361,6 +1361,43 @@ mod tests { assert_eq!(delivery_worker_count(usize::MAX), 32); } + #[cfg(target_os = "macos")] + #[test] + fn journal_hook_launch_waits_for_process_creation_barrier() { + let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let delivery = JournalHookDelivery { + manifest: manifest(), + event: document("plugin.test.process_barrier", json!({})).record, + attempt: 0, + }; + let attempt = JournalHookAttempt { attempt: 1, causation_id: "barrier_test".into() }; + let (started_sender, started_receiver) = mpsc::sync_channel(1); + let (completed_sender, completed_receiver) = mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + started_sender.send(()).unwrap(); + completed_sender.send(execute_delivery(&delivery, &attempt)).unwrap(); + }); + started_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("journal hook launch worker did not start"); + + assert!( + matches!( + completed_receiver.recv_timeout(Duration::from_millis(250)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "journal hook launch bypassed the process creation barrier" + ); + drop(barrier); + + let (status, error) = completed_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("journal hook did not resume after the process creation barrier opened"); + assert_eq!(status, Some(0)); + assert_eq!(error, None); + worker.join().unwrap(); + } + #[cfg(target_os = "linux")] #[test] fn unix_hook_tree_kills_a_descendant_that_created_a_new_session() { diff --git a/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs b/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs index eb0ea48fa2dd..42b61296cb66 100644 --- a/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs +++ b/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs @@ -1420,6 +1420,43 @@ mod tests { assert!(started.elapsed() < Duration::from_secs(2)); } + #[cfg(target_os = "macos")] + #[test] + fn hermes_launch_waits_for_process_creation_barrier() { + let barrier = cmux_tui_process::ProcessCreationGuard::acquire(); + let (started_sender, started_receiver) = std::sync::mpsc::sync_channel(1); + let (completed_sender, completed_receiver) = std::sync::mpsc::sync_channel(1); + let worker = std::thread::spawn(move || { + started_sender.send(()).unwrap(); + completed_sender + .send(run_hermes_command_with_timeout( + Path::new("/usr/bin/true"), + &[], + Duration::from_secs(5), + )) + .unwrap(); + }); + started_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("Hermes launch worker did not start"); + + assert!( + matches!( + completed_receiver.recv_timeout(Duration::from_millis(250)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) + ), + "Hermes launch bypassed the process creation barrier" + ); + drop(barrier); + + let output = completed_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("Hermes launch did not resume after the process creation barrier opened") + .unwrap(); + assert!(output.status.success()); + worker.join().unwrap(); + } + #[cfg(not(unix))] #[test] fn public_hook_operations_are_rejected_on_unsupported_platforms() { From be63aaead1236b9b0c7f75ef0477682ed38b93f2 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:48:53 -0700 Subject: [PATCH 478/521] fix(tui): guard every production child launch --- cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs | 2 +- cmux-tui/crates/cmux-tui/src/agent_hook_install.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs b/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs index fedf27543afa..b5ef85abb2e1 100644 --- a/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs +++ b/cmux-tui/crates/cmux-tui-core/src/journal_hooks.rs @@ -902,7 +902,7 @@ fn execute_delivery_with_shutdown( } #[cfg(unix)] tree.configure(&mut command); - let mut child = match command.spawn() { + let mut child = match cmux_tui_process::spawn(&mut command) { Ok(child) => child, Err(error) => return (None, Some(format!("start hook executable: {error}"))), }; diff --git a/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs b/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs index 42b61296cb66..82857417aabf 100644 --- a/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs +++ b/cmux-tui/crates/cmux-tui/src/agent_hook_install.rs @@ -744,7 +744,8 @@ fn run_hermes_command_with_timeout( command.creation_flags(CREATE_SUSPENDED); #[cfg(unix)] tree.configure(&mut command); - let mut child = command.spawn().with_context(|| format!("run {}", binary.display()))?; + let mut child = cmux_tui_process::spawn(&mut command) + .with_context(|| format!("run {}", binary.display()))?; #[cfg(unix)] if let Err(error) = tree.bind(child.id()) { tree.terminate_until(deadline); From 90f0f4a612621a8584d9b29f58e59e6f47207ea6 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:54:17 -0700 Subject: [PATCH 479/521] test(macOS): preserve overflow admission order --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 0f7a329ef2ec..9df7f6034ac1 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -408,6 +408,59 @@ import CmuxTerminalCore ) } + @Test func newCreationStaysBehindOlderOverflowWhenCapacityReturns() throws { + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2 + ) + let owners = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let fixtures = (0..<4).map { _ in + makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + } + var firstOwnerReleased = false + defer { + for fixture in fixtures { + fixture.surface.beginPortalCloseLifecycle( + reason: "test.overflowAdmissionOrder" + ) + } + if !firstOwnerReleased { + coordinator.cancelRuntimeSurfaceOwnership(owners[0]) + } + coordinator.cancelRuntimeSurfaceOwnership(owners[1]) + } + + for (index, fixture) in fixtures.prefix(3).enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + } + coordinator.cancelRuntimeSurfaceOwnershipRecovery(fixtures[0].surface.id) + coordinator.cancelRuntimeSurfaceOwnershipRecovery(fixtures[1].surface.id) + let olderOverflow = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(olderOverflow.headID == fixtures[2].surface.id) + #expect(olderOverflow.tailID == fixtures[2].surface.id) + + coordinator.cancelRuntimeSurfaceOwnership(owners[0]) + firstOwnerReleased = true + fixtures[3].surface.createSurface(for: fixtures[3].nativeView) + scheduler.runScheduledOperation(at: 3) + + let orderedOverflow = + coordinator.debugRuntimeSurfaceOwnershipRecoveryOverflowSnapshot + #expect(orderedOverflow.entryCount == 2) + #expect(orderedOverflow.linkedNodeCount == 2) + #expect(orderedOverflow.headID == fixtures[2].surface.id) + #expect(orderedOverflow.tailID == fixtures[3].surface.id) + } + @Test func overflowRecoveryWaitsForEveryOlderPrimaryRequest() async throws { let ownerCount = 4 let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( From d09ac55e7e25554717f7281acc8d5e8f10548985 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:55:11 -0700 Subject: [PATCH 480/521] fix(macOS): preserve overflow recovery order --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 84 ++++++++++--------- ...timeOwnershipRecoveryRescanScheduler.swift | 11 +++ 2 files changed, 57 insertions(+), 38 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 71f56da43743..ada586428264 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -64,53 +64,61 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { let output: ( result: TerminalSurfaceRuntimeOwnershipRecoveryAdmissionResult, requestRescan: Bool - ) = state.withLock { state in - let claimedCapacity: Bool - if let capacityReservation { - guard state.recoveryCapacityReservationIDs.remove( - capacityReservation.id - ) != nil else { + ) = recoveryRescanScheduler.withOverflowQueueFence { + hasPendingOverflow in + state.withLock { state in + let claimedCapacity: Bool + if let capacityReservation { + guard state.recoveryCapacityReservationIDs.remove( + capacityReservation.id + ) != nil else { + return (.rejected, false) + } + claimedCapacity = true + } else { + claimedCapacity = false + } + guard !state.closeTeardownAllStalled else { + if state.recoveryEntriesByID[recoveryID] != nil { + state.recoveryEntriesByID[recoveryID]?.action = onRecovery + state.recoveryEntriesByID[recoveryID]?.failure = onFailure + return (.deferred, claimedCapacity) + } + return (.closeTeardownStalled, claimedCapacity) + } + if hasPendingOverflow, + capacityReservation == nil, + state.recoveryEntriesByID[recoveryID] == nil { return (.rejected, false) } - claimedCapacity = true - } else { - claimedCapacity = false - } - guard !state.closeTeardownAllStalled else { + if !state.closeTeardownDegraded, + state.recoveryHeadID == nil, + !state.recoveryGrantIsScheduled, + state.reservationIDs.count < maximumOwnerCount, + state.ingressReservationIDs.count < maximumOwnerCount + { + _ = removeRecoveryAction(recoveryID, from: &state) + return ( + .reserved(reserveOwnershipAndIngress(in: &state)), + takeRecoveryRescanRequest(from: &state) || claimedCapacity + ) + } if state.recoveryEntriesByID[recoveryID] != nil { state.recoveryEntriesByID[recoveryID]?.action = onRecovery state.recoveryEntriesByID[recoveryID]?.failure = onFailure return (.deferred, claimedCapacity) } - return (.closeTeardownStalled, claimedCapacity) - } - if !state.closeTeardownDegraded, - state.recoveryHeadID == nil, - !state.recoveryGrantIsScheduled, - state.reservationIDs.count < maximumOwnerCount, - state.ingressReservationIDs.count < maximumOwnerCount - { - _ = removeRecoveryAction(recoveryID, from: &state) - return ( - .reserved(reserveOwnershipAndIngress(in: &state)), - takeRecoveryRescanRequest(from: &state) || claimedCapacity + guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { + return (.rejected, false) + } + enqueueRecoveryAction( + onRecovery, + onFailure: onFailure, + id: recoveryID, + in: &state ) + return (.deferred, false) } - if state.recoveryEntriesByID[recoveryID] != nil { - state.recoveryEntriesByID[recoveryID]?.action = onRecovery - state.recoveryEntriesByID[recoveryID]?.failure = onFailure - return (.deferred, claimedCapacity) - } - guard claimedCapacity || recoveryCapacityIsOpen(in: state) else { - return (.rejected, false) - } - enqueueRecoveryAction( - onRecovery, - onFailure: onFailure, - id: recoveryID, - in: &state - ) - return (.deferred, false) } if output.requestRescan { recoveryRescanScheduler.requestRescan() diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 2e512a753a29..77dde854d549 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -57,6 +57,17 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: self.maximumEntryCount = maximumEntryCount } + /// Hold the overflow index stable while admission decides where a new + /// request belongs. The scheduler lock is always outside the admission + /// lock, and scheduler callbacks enter admission only after releasing it. + internal func withOverflowQueueFence( + _ body: (Bool) -> Result + ) -> Result { + state.withLockUnchecked { state in + body(state.headID != nil) + } + } + #if DEBUG internal var debugSnapshot: ( From 41994b91e0ddc206637238589c009875aa5c4690 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:05:50 -0700 Subject: [PATCH 481/521] test(tui): require exact exit sidecar cleanup --- .../src/terminal_host_runtime.rs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index b1a6e4eeb9a1..5c80698c4d3d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -9150,6 +9150,35 @@ mod unix { let _ = fs::remove_dir_all(root); } + #[test] + fn confirmed_host_termination_acknowledges_its_exact_exit_sidecar() { + let (record_path, record, lease) = record_fixture("confirmed-exit-sidecar"); + let root = record_path.parent().unwrap().to_path_buf(); + let exit_record = TerminalHostExitRecord::new( + &TerminalHostIdentity { + terminal_id: record.terminal_id.clone(), + incarnation: record.incarnation.clone(), + }, + TerminalExit { + outcome: crate::terminal_host_protocol::TerminalExitOutcome::Exit { code: 17 }, + exited_at_ms: 1_234_567, + }, + ); + let exit_path = record_path.with_extension("exit"); + write_exit_record(&exit_path, &exit_record).unwrap(); + drop(lease); + + assert!(terminate_and_confirm_terminal_host_record( + &record, + &record_path, + Instant::now() + Duration::from_secs(1), + )); + assert!(!record_path.exists(), "confirmed termination retained its discovery record"); + assert!(!exit_path.exists(), "confirmed termination retained its exact exit sidecar"); + + let _ = fs::remove_dir_all(root); + } + #[test] fn exit_sidecar_publication_never_clobbers_a_concurrent_outcome() { let (record_path, record, lease) = record_fixture("exit-sidecar-race"); From 3383ef542e8ab911bbf03b34440afd1b5374aa0e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:06:18 -0700 Subject: [PATCH 482/521] fix(tui): acknowledge confirmed host exits --- .../cmux-tui-core/src/terminal_host_runtime.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index 5c80698c4d3d..f68816b11b10 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -2948,6 +2948,22 @@ mod unix { .unwrap_or(TerminalHostLiveness::Indeterminate) { TerminalHostLiveness::Dead => { + let exit = match terminal_host_exit_record(record_path) { + Ok(exit) => exit, + Err(_) => return false, + }; + if let Some((exit_path, exit)) = exit { + if exit.terminal_id != record.terminal_id + || exit.incarnation != record.incarnation + { + return false; + } + match acknowledge_terminal_host_exit_record(&exit_path, &exit) { + Ok(true) => {} + Ok(false) if !exit_path.exists() => {} + Ok(false) | Err(_) => return false, + } + } return match remove_stale_terminal_host_record(record_path, record) { Ok(removed) => removed, Err(_) if !record_path.exists() => true, From 092b82736f04b171cee298962fde75b84418c797 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:08:57 -0700 Subject: [PATCH 483/521] refactor(macOS): use restore pacing entrypoint --- .../CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift index f52e875ca023..9fd47abad34c 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+RuntimeLifecycle.swift @@ -909,7 +909,7 @@ extension TerminalSurface { // slot. Return admission ownership and use the shared normal // entrypoint to queue a fresh paced slot before retrying Ghostty. runtimeTeardown.cancelRuntimeSurfaceOwnership(reservation) - createSurface(for: view, source: .normal) + createSurface(for: view) return } createSurface( From bfce2a85c9a7f240c358b2bc26ab89be51ac20b4 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:23:03 -0700 Subject: [PATCH 484/521] test(cdp): stop after child wait ownership loss --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index b13d8cbf5da8..bfabd87c4da9 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -840,6 +840,44 @@ mod tests { assert_eq!(next_reaper_retry_delay(REAPER_MAX_BACKOFF), REAPER_MAX_BACKOFF); } + #[cfg(unix)] + #[test] + fn reaper_releases_capacity_after_child_wait_ownership_is_lost() { + let _guard = REAPER_TEST_LOCK.lock().unwrap(); + let (_observer, events) = ReaperTestObserver::install(); + drop(chrome_reaper_lease().unwrap()); + let active = { + let slot = CHROME_REAPER.get().unwrap().lock().unwrap(); + slot.as_ref().unwrap().active.clone() + }; + let baseline = active.load(Ordering::Acquire); + let lease = chrome_reaper_lease().unwrap(); + assert_eq!(active.load(Ordering::Acquire), baseline + 1); + + unsafe extern "C" { + fn waitpid(pid: i32, status: *mut i32, options: i32) -> i32; + } + let mut child = Command::new("/bin/sh") + .args(["-c", "exit 0"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = i32::try_from(child.id()).unwrap(); + let mut status = 0; + // SAFETY: pid identifies this live test child and status is writable. + assert_eq!(unsafe { waitpid(pid, &raw mut status, 0) }, pid); + assert_eq!(child.try_wait().unwrap_err().raw_os_error(), Some(libc::ECHILD)); + let profile_dir = make_profile_dir().unwrap(); + + reap_child_detached(lease, child, Some(profile_dir.clone())); + wait_for_reaper_test_event(&events, ReaperTestEvent::Complete(pid as u32)); + + assert_eq!(active.load(Ordering::Acquire), baseline); + assert!(!profile_dir.exists(), "ownership loss retained the Chrome profile directory"); + } + #[cfg(unix)] #[test] fn terminal_reaper_wait_error_retains_ownership_for_retry() { From 7062138972b805693dc68c891d40c6f97174ec73 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:23:26 -0700 Subject: [PATCH 485/521] fix(cdp): stop reaping after wait ownership loss --- cmux-tui/crates/cmux-tui-cdp/src/chrome.rs | 36 +++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs index bfabd87c4da9..71cebe1474b3 100644 --- a/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs +++ b/cmux-tui/crates/cmux-tui-cdp/src/chrome.rs @@ -383,18 +383,18 @@ fn poll_reap_request(request: &mut ReapRequest) -> bool { publish_reaper_test_event(ReaperTestEvent::Attempt(request.child.id())); #[cfg(test)] { - if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { - let _ = request.child.kill(); - return false; - } if FORCE_REAPER_PENDING.load(Ordering::Acquire) { return false; } } - let _ = request.child.kill(); - match request.child.try_wait() { + match chrome_child_try_wait(request) { Ok(Some(_)) => {} - Ok(None) | Err(_) => return false, + Err(error) if chrome_child_wait_lost_ownership(&error) => {} + Ok(None) => { + let _ = request.child.kill(); + return false; + } + Err(_) => return false, } if let Some(profile_dir) = request.profile_dir.take() { let _ = std::fs::remove_dir_all(profile_dir); @@ -408,6 +408,28 @@ fn poll_reap_request(request: &mut ReapRequest) -> bool { true } +fn chrome_child_wait_lost_ownership(error: &std::io::Error) -> bool { + #[cfg(unix)] + { + error.raw_os_error() == Some(libc::ECHILD) + } + #[cfg(not(unix))] + { + let _ = error; + false + } +} + +fn chrome_child_try_wait( + request: &mut ReapRequest, +) -> std::io::Result> { + #[cfg(test)] + if FORCE_REAPER_WAIT_ERROR.load(Ordering::Acquire) { + return Err(std::io::Error::from(std::io::ErrorKind::Interrupted)); + } + request.child.try_wait() +} + #[cfg(test)] fn publish_reaper_test_event(event: ReaperTestEvent) { if let Some(sender) = REAPER_TEST_EVENT_SENDER.lock().unwrap().as_ref() { From c6d1ca72f60d95631ad21a38a8ba62a253c55ca7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:40:33 -0700 Subject: [PATCH 486/521] test(tui): model exact pending host reconnect --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 41 +++++++++++++----------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index fb5b85e6b623..54a84f49b7de 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -30134,7 +30134,7 @@ mod tests { #[cfg(unix)] #[test] - fn pending_topology_accepts_current_host_reconnect_without_advancing_lifecycle() { + fn pending_topology_accepts_reconnect_for_the_exact_live_host() { const TERMINAL: &str = "00000000000040008000000000000012"; const INCARNATION: &str = "10000000000040008000000000000012"; const PENDING_SURFACE: SurfaceId = 4242; @@ -30160,14 +30160,27 @@ mod tests { }, ) .unwrap(); + commit_terminal_lifecycle( + &mut registry, + "terminal-ready", + "terminal-ready", + TERMINAL, + TerminalLifecycle::Running, + Some(INCARNATION), + None, + ) + .unwrap(); } + let surface = Surface::spawn_for_test_with_terminal_host_identity( + PENDING_SURFACE, + mux.surface_options.lock().unwrap().clone(), + Arc::downgrade(&mux), + identity.clone(), + ) + .unwrap(); + insert_surface_checked(&mux, &mut mux.state.lock().unwrap(), surface).unwrap(); assert!(mux.terminal_host_connection_lost(PENDING_SURFACE, &identity)); - assert!(mux.terminal_host_reconnected( - PENDING_SURFACE, - &identity, - KittyGraphicsLimits::disabled(), - )); assert_eq!( mux.workspace_registry .lock() @@ -30176,19 +30189,8 @@ mod tests { .unwrap() .unwrap() .lifecycle, - TerminalLifecycle::Launching + TerminalLifecycle::Adopting ); - - mux.transition_terminal_lifecycle( - "terminal-adopting", - "test-adoption", - TERMINAL, - TerminalLifecycle::Adopting, - Some(INCARNATION), - None, - ) - .unwrap(); - assert!(mux.terminal_host_connection_lost(PENDING_SURFACE, &identity)); assert!(mux.terminal_host_reconnected( PENDING_SURFACE, &identity, @@ -30202,8 +30204,9 @@ mod tests { .unwrap() .unwrap() .lifecycle, - TerminalLifecycle::Adopting + TerminalLifecycle::Running ); + mux.shutdown().unwrap(); } #[test] From eaaea8aedd8e689864809ac2f12d8de56fcab6a2 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:49:40 -0700 Subject: [PATCH 487/521] test(tui): wait for websocket connection owners --- cmux-tui/crates/cmux-tui-core/src/server.rs | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 4094ec9d8f66..40c4fd0ab2d7 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -13206,6 +13206,41 @@ mod tests { assert!(server.thread.is_none(), "completed WebSocket listener retained its join handle"); } + #[test] + fn websocket_shutdown_waits_for_registered_connection_owner() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (tracked, _) = listener.accept().unwrap(); + let connections = Arc::new(Mutex::new(HashMap::from([(1, tracked)]))); + let (listener_finished, observed_listener_finished) = std::sync::mpsc::channel(); + let listener_thread = std::thread::spawn(move || { + listener_finished.send(()).unwrap(); + }); + observed_listener_finished + .recv_timeout(Duration::from_secs(1)) + .expect("synthetic WebSocket listener did not finish"); + let mut server = WebSocketServer { + local_addr: "127.0.0.1:0".parse().unwrap(), + shutdown: Arc::new(AtomicBool::new(false)), + connections: connections.clone(), + thread: Some(listener_thread), + accept_attempts: Arc::new(AtomicU64::new(0)), + active_connections: Arc::new(AtomicU64::new(0)), + #[cfg(target_os = "macos")] + accepted_stream_mode: Arc::new(AtomicU64::new(0)), + }; + + let first = server.shutdown_until(Instant::now() + Duration::from_millis(25)).unwrap(); + let retained = connections.lock().unwrap().contains_key(&1); + connections.lock().unwrap().remove(&1); + let second = server.shutdown_until(Instant::now() + Duration::from_secs(1)).unwrap(); + drop(client); + + assert!(!first, "registered WebSocket connection owner reported complete"); + assert!(retained, "shutdown discarded the registered connection owner"); + assert!(second, "WebSocket shutdown did not finish after owner release"); + } + #[test] fn websocket_worker_requires_a_registered_shutdown_owner() { let mux = Mux::new("websocket-tracking-clone-test", SurfaceOptions::default()); From 9b92bbba3cbdf2b78bda07601c918ff8053efdcd Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:51:17 -0700 Subject: [PATCH 488/521] fix(tui): await websocket shutdown owners --- cmux-tui/crates/cmux-tui-core/src/server.rs | 136 ++++++++++++++------ 1 file changed, 98 insertions(+), 38 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index 40c4fd0ab2d7..c039eba0f2a6 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4860,11 +4860,74 @@ pub fn serve_owned(mux: Arc, path: Option) -> anyhow::Result, + idle: Condvar, +} + +#[derive(Default)] +struct WebSocketConnectionState { + shutting_down: bool, + streams: HashMap, +} + +impl WebSocketConnectionRegistry { + fn register(&self, id: u64, stream: TcpStream) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if state.shutting_down { + let _ = stream.shutdown(Shutdown::Both); + return false; + } + state.streams.insert(id, stream); + true + } + + fn remove(&self, id: u64) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.streams.remove(&id); + if state.streams.is_empty() { + self.idle.notify_all(); + } + } + + fn shutdown_until(&self, deadline: Instant) -> bool { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.shutting_down = true; + for stream in state.streams.values() { + let _ = stream.shutdown(Shutdown::Both); + } + while !state.streams.is_empty() { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return false; + }; + let (next, timeout) = self + .idle + .wait_timeout(state, remaining) + .unwrap_or_else(std::sync::PoisonError::into_inner); + state = next; + if timeout.timed_out() && !state.streams.is_empty() { + return false; + } + } + true + } + + fn start_shutdown(&self) { + let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + state.shutting_down = true; + for stream in state.streams.values() { + let _ = stream.shutdown(Shutdown::Both); + } + } +} + pub struct WebSocketServer { local_addr: SocketAddr, shutdown: Arc, - connections: Arc>>, + connections: Arc, thread: Option>, + thread_finished: std::sync::mpsc::Receiver<()>, #[cfg(test)] accept_attempts: Arc, #[cfg(test)] @@ -4883,42 +4946,28 @@ impl WebSocketServer { /// Returns `Ok(false)` while the caller must retain this owner and retry. pub fn shutdown_until(&mut self, deadline: Instant) -> anyhow::Result { self.shutdown.store(true, Ordering::Release); - while !self.try_close_connections() { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - std::thread::sleep(remaining.min(Duration::from_millis(1))); + if !self.connections.shutdown_until(deadline) { + return Ok(false); } let Some(thread) = self.thread.as_ref() else { return Ok(true) }; - while !thread.is_finished() { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - std::thread::sleep(remaining.min(Duration::from_millis(1))); + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + match self.thread_finished.recv_timeout(remaining) { + Ok(()) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {} + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => return Ok(false), } let thread = self.thread.take().expect("finished WebSocket listener retained its owner"); thread.join().map_err(|_| anyhow::anyhow!("WebSocket listener thread panicked"))?; Ok(true) } - - fn try_close_connections(&self) -> bool { - let connections = match self.connections.try_lock() { - Ok(connections) => connections, - Err(std::sync::TryLockError::Poisoned(error)) => error.into_inner(), - Err(std::sync::TryLockError::WouldBlock) => return false, - }; - for stream in connections.values() { - let _ = stream.shutdown(Shutdown::Both); - } - true - } } impl Drop for WebSocketServer { fn drop(&mut self) { self.shutdown.store(true, Ordering::Release); - let _ = self.try_close_connections(); + self.connections.start_shutdown(); if let Some(thread) = self.thread.take() && thread.is_finished() { @@ -4971,7 +5020,7 @@ fn serve_websocket_with_tracking_cloner( listener.set_nonblocking(true)?; let local_addr = listener.local_addr()?; let shutdown = Arc::new(AtomicBool::new(false)); - let connections = Arc::new(Mutex::new(HashMap::new())); + let connections = Arc::new(WebSocketConnectionRegistry::default()); let next_connection = Arc::new(AtomicU64::new(1)); let active_connections = Arc::new(AtomicU64::new(0)); #[cfg(test)] @@ -4980,6 +5029,7 @@ fn serve_websocket_with_tracking_cloner( let accepted_stream_mode = Arc::new(AtomicU64::new(0)); let thread_shutdown = shutdown.clone(); let thread_connections = connections.clone(); + let (thread_finished_sender, thread_finished) = std::sync::mpsc::channel(); #[cfg(test)] let observed_active_connections = active_connections.clone(); #[cfg(test)] @@ -5023,7 +5073,9 @@ fn serve_websocket_with_tracking_cloner( Ok(tracked) => tracked, Err(_) => continue, }; - thread_connections.lock().unwrap().insert(id, tracked); + if !thread_connections.register(id, tracked) { + continue; + } let mux = mux.clone(); let token = token.clone(); let render_service = render_service.clone(); @@ -5040,24 +5092,22 @@ fn serve_websocket_with_tracking_cloner( render_service, Some(permit), ); - connections.lock().unwrap().remove(&id); + connections.remove(id); }) .is_err() { - cleanup_connections.lock().unwrap().remove(&id); + cleanup_connections.remove(id); } } - let connections = - thread_connections.lock().unwrap_or_else(std::sync::PoisonError::into_inner); - for stream in connections.values() { - let _ = stream.shutdown(Shutdown::Both); - } + thread_connections.start_shutdown(); + let _ = thread_finished_sender.send(()); })?; Ok(WebSocketServer { local_addr, shutdown, connections, thread: Some(thread), + thread_finished, #[cfg(test)] accept_attempts, #[cfg(test)] @@ -13148,15 +13198,18 @@ mod tests { fn websocket_drop_does_not_join_an_unfinished_listener() { let (release, blocked) = std::sync::mpsc::channel(); let (worker_finished, observed_worker) = std::sync::mpsc::channel(); + let (thread_finished_sender, thread_finished) = std::sync::mpsc::channel(); let listener_thread = std::thread::spawn(move || { let _ = blocked.recv(); let _ = worker_finished.send(()); + let _ = thread_finished_sender.send(()); }); let server = WebSocketServer { local_addr: "127.0.0.1:0".parse().unwrap(), shutdown: Arc::new(AtomicBool::new(false)), - connections: Arc::new(Mutex::new(HashMap::new())), + connections: Arc::new(WebSocketConnectionRegistry::default()), thread: Some(listener_thread), + thread_finished, accept_attempts: Arc::new(AtomicU64::new(0)), active_connections: Arc::new(AtomicU64::new(0)), #[cfg(target_os = "macos")] @@ -13181,14 +13234,17 @@ mod tests { #[test] fn websocket_shutdown_retains_an_unfinished_listener_for_retry() { let (release, blocked) = std::sync::mpsc::channel(); + let (thread_finished_sender, thread_finished) = std::sync::mpsc::channel(); let listener_thread = std::thread::spawn(move || { let _ = blocked.recv(); + let _ = thread_finished_sender.send(()); }); let mut server = WebSocketServer { local_addr: "127.0.0.1:0".parse().unwrap(), shutdown: Arc::new(AtomicBool::new(false)), - connections: Arc::new(Mutex::new(HashMap::new())), + connections: Arc::new(WebSocketConnectionRegistry::default()), thread: Some(listener_thread), + thread_finished, accept_attempts: Arc::new(AtomicU64::new(0)), active_connections: Arc::new(AtomicU64::new(0)), #[cfg(target_os = "macos")] @@ -13211,10 +13267,13 @@ mod tests { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let client = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); let (tracked, _) = listener.accept().unwrap(); - let connections = Arc::new(Mutex::new(HashMap::from([(1, tracked)]))); + let connections = Arc::new(WebSocketConnectionRegistry::default()); + assert!(connections.register(1, tracked)); let (listener_finished, observed_listener_finished) = std::sync::mpsc::channel(); + let (thread_finished_sender, thread_finished) = std::sync::mpsc::channel(); let listener_thread = std::thread::spawn(move || { listener_finished.send(()).unwrap(); + thread_finished_sender.send(()).unwrap(); }); observed_listener_finished .recv_timeout(Duration::from_secs(1)) @@ -13224,6 +13283,7 @@ mod tests { shutdown: Arc::new(AtomicBool::new(false)), connections: connections.clone(), thread: Some(listener_thread), + thread_finished, accept_attempts: Arc::new(AtomicU64::new(0)), active_connections: Arc::new(AtomicU64::new(0)), #[cfg(target_os = "macos")] @@ -13231,8 +13291,8 @@ mod tests { }; let first = server.shutdown_until(Instant::now() + Duration::from_millis(25)).unwrap(); - let retained = connections.lock().unwrap().contains_key(&1); - connections.lock().unwrap().remove(&1); + let retained = connections.state.lock().unwrap().streams.contains_key(&1); + connections.remove(1); let second = server.shutdown_until(Instant::now() + Duration::from_secs(1)).unwrap(); drop(client); From 9bdfa2b7bb131b11aad9b5886e7f9ee92126fcad Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:03:52 -0700 Subject: [PATCH 489/521] fix(tui): observe completed websocket shutdown --- cmux-tui/crates/cmux-tui-core/src/server.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index c039eba0f2a6..afca57a940c4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -4950,13 +4950,17 @@ impl WebSocketServer { return Ok(false); } - let Some(thread) = self.thread.as_ref() else { return Ok(true) }; - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - return Ok(false); - }; - match self.thread_finished.recv_timeout(remaining) { - Ok(()) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {} - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => return Ok(false), + if self.thread.is_none() { + return Ok(true); + } + if matches!(self.thread_finished.try_recv(), Err(std::sync::mpsc::TryRecvError::Empty)) { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Ok(false); + }; + match self.thread_finished.recv_timeout(remaining) { + Ok(()) | Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => {} + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => return Ok(false), + } } let thread = self.thread.take().expect("finished WebSocket listener retained its owner"); thread.join().map_err(|_| anyhow::anyhow!("WebSocket listener thread panicked"))?; @@ -13254,7 +13258,7 @@ mod tests { let first = server.shutdown_until(Instant::now() + Duration::from_millis(25)).unwrap(); let retained = server.thread.is_some(); let _ = release.send(()); - let second = server.shutdown_until(Instant::now() + Duration::from_secs(1)).unwrap(); + let second = server.shutdown_until(Instant::now()).unwrap(); assert!(!first, "unfinished WebSocket listener reported complete"); assert!(retained, "pending WebSocket shutdown discarded listener ownership"); From e5b788be31c0e195bee6001728695f39d186816d Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:08:20 -0700 Subject: [PATCH 490/521] test(tui): cover unavailable task-name identity --- .../cmux-tui-core/src/process_session.rs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 154ea1b2919e..ce1243f6db79 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -43,6 +43,27 @@ thread_local! { const { std::cell::Cell::new(false) }; static POST_SPAWN_FAILURE_MARKER: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; + #[cfg(target_os = "macos")] + static FORCE_TASK_NAME_FOR_PID_FAILURE: std::cell::Cell = + const { std::cell::Cell::new(false) }; +} + +#[cfg(all(test, target_os = "macos"))] +struct ForcedTaskNameForPidFailure { + previous: bool, +} + +#[cfg(all(test, target_os = "macos"))] +impl Drop for ForcedTaskNameForPidFailure { + fn drop(&mut self) { + FORCE_TASK_NAME_FOR_PID_FAILURE.set(self.previous); + } +} + +#[cfg(all(test, target_os = "macos"))] +fn force_task_name_for_pid_failure_for_test() -> ForcedTaskNameForPidFailure { + let previous = FORCE_TASK_NAME_FOR_PID_FAILURE.replace(true); + ForcedTaskNameForPidFailure { previous } } #[cfg(test)] @@ -1326,6 +1347,13 @@ impl StableProcessHandle { #[allow(deprecated)] let current_task = unsafe { libc::mach_task_self() }; // SAFETY: `task` points to writable storage for one task-name right. + #[cfg(test)] + let name_result = if FORCE_TASK_NAME_FOR_PID_FAILURE.get() { + libc::KERN_FAILURE + } else { + unsafe { task_name_for_pid(current_task, pid, &mut task) } + }; + #[cfg(not(test))] let name_result = unsafe { task_name_for_pid(current_task, pid, &mut task) }; if name_result != libc::KERN_SUCCESS { return if process_is_gone(pid)? { @@ -2173,6 +2201,18 @@ mod tests { true } + #[cfg(target_os = "macos")] + #[test] + fn macos_process_identity_survives_unavailable_task_name_access() { + let _failure = force_task_name_for_pid_failure_for_test(); + let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + + let process = StableProcessHandle::capture(pid).unwrap().unwrap(); + + assert!(process.matches_current().unwrap()); + assert!(process.signal(0).unwrap()); + } + #[cfg(target_os = "macos")] #[test] fn macos_session_query_is_scoped_to_the_owned_session() { From e14aaff1451e1f128027c802dc6b3ecbe3c7c91f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:16:32 -0700 Subject: [PATCH 491/521] test(tui): preserve exited terminal replay --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index 54a84f49b7de..e85f877c2e8a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -21106,13 +21106,21 @@ mod tests { assert!(after["notifications"].as_array().unwrap().contains(¬ification_value)); assert!(after["agents"].as_array().unwrap().contains(&agent_value)); assert!(after["frontend_projections"].as_array().unwrap().contains(&projection_value)); - assert_eq!(after["terminals"], serde_json::json!([])); + let restored_terminals = after["terminals"].as_array().unwrap(); + assert_eq!(restored_terminals.len(), 1); + let restored_terminal = &restored_terminals[0]; + assert_eq!(restored_terminal["id"], terminal_public_id.as_str()); + assert_eq!(restored_terminal["lifecycle"], "exited"); + assert_eq!(restored_terminal["running"], false); + assert_eq!(restored_terminal["tab_id"], serde_json::Value::Null); + assert_eq!(restored_terminal["tab_ids"], serde_json::json!([])); assert_eq!(reopened.resource_surface_for_terminal(&terminal_public_id), None); let exited = reopened.wait_for_terminal_exit(&terminal_public_id, Some(Duration::ZERO)).unwrap(); assert_eq!(exited["state"], "exited"); assert_eq!(exited["outcome"]["kind"], "unknown"); assert_eq!(exited["outcome"]["reason"], "missing-host-record"); + assert_eq!(restored_terminal["exit"]["outcome"], exited["outcome"]); let notifications = public_request( &reopened, "notifications", From 83e4070521170bf6fefe998281702a2aeee60c49 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:16:56 -0700 Subject: [PATCH 492/521] fix(tui): fall back to exact PID versions --- .../cmux-tui-core/src/process_session.rs | 57 ++++++++++++------- .../cmux-tui-core/src/unix_process_scope.rs | 6 ++ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index ce1243f6db79..bb8a95652720 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1327,7 +1327,7 @@ fn stable_process_in_session( /// An OS-backed reference to one process instance that cannot retarget after PID reuse. pub struct StableProcessHandle { pid: libc::pid_t, - audit_token: MacAuditToken, + pid_version: u32, } #[cfg(target_os = "macos")] @@ -1341,6 +1341,17 @@ struct MacAuditToken { impl StableProcessHandle { /// Capture the process currently using `pid`, or return `None` if it is already gone. pub fn capture(pid: libc::pid_t) -> io::Result> { + let pid_version = match Self::task_pid_version(pid)? { + Some(pid_version) => pid_version, + None => { + let Some(pid_version) = mac_process_pid_version(pid)? else { return Ok(None) }; + pid_version + } + }; + Ok(Some(Self { pid, pid_version })) + } + + fn task_pid_version(pid: libc::pid_t) -> io::Result> { const TASK_AUDIT_TOKEN: libc::task_flavor_t = 15; let mut task = 0; // SAFETY: mach_task_self returns the calling process's send right. @@ -1356,13 +1367,7 @@ impl StableProcessHandle { #[cfg(not(test))] let name_result = unsafe { task_name_for_pid(current_task, pid, &mut task) }; if name_result != libc::KERN_SUCCESS { - return if process_is_gone(pid)? { - Ok(None) - } else { - Err(io::Error::other(format!( - "cannot acquire stable process identity for PID {pid}: Mach error {name_result}" - ))) - }; + return Ok(None); } let mut audit_token = MacAuditToken { values: [0; 8] }; @@ -1381,13 +1386,7 @@ impl StableProcessHandle { // SAFETY: `task` is the send right returned by task_name_for_pid. let _ = unsafe { mach_port_deallocate(current_task, task) }; if info_result != libc::KERN_SUCCESS { - return if process_is_gone(pid)? { - Ok(None) - } else { - Err(io::Error::other(format!( - "cannot read stable process identity for PID {pid}: Mach error {info_result}" - ))) - }; + return Ok(None); } let expected_pid = u32::try_from(pid) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; @@ -1397,7 +1396,7 @@ impl StableProcessHandle { "stable process identity returned a mismatched PID", )); } - Ok(Some(Self { pid, audit_token })) + Ok(Some(audit_token.values[7])) } /// Return whether this exact process instance is still alive. @@ -1408,10 +1407,14 @@ impl StableProcessHandle { /// Signal this exact process instance, returning false if it is already gone. pub fn signal(&self, signal: libc::c_int) -> io::Result { let signal_process = resolve_proc_signal_with_audittoken()?; - let mut token = self.audit_token; + let expected_pid = u32::try_from(self.pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; + let mut token = MacAuditToken { values: [u32::MAX; 8] }; + token.values[5] = expected_pid; + token.values[7] = self.pid_version; loop { - // SAFETY: the audit token came from TASK_AUDIT_TOKEN for this - // process instance; libproc validates its PID generation. + // SAFETY: the PID and PID version came from kernel process + // metadata; libproc validates them together before signaling. let result = unsafe { signal_process(&mut token, signal) }; if result == 0 { #[cfg(test)] @@ -1430,6 +1433,22 @@ impl StableProcessHandle { } } +#[cfg(target_os = "macos")] +fn mac_process_pid_version(pid: libc::pid_t) -> io::Result> { + let expected_pid = u32::try_from(pid) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; + if let Some(pid_version) = crate::unix_process_scope::mac_process_pid_version(expected_pid) { + return Ok(Some(pid_version)); + } + if process_is_gone(pid)? { + Ok(None) + } else { + Err(io::Error::other(format!( + "cannot acquire stable process identity for PID {pid}: kernel PID version is unavailable" + ))) + } +} + #[cfg(target_os = "macos")] fn process_is_gone(pid: libc::pid_t) -> io::Result { // SAFETY: signal zero performs a non-mutating liveness/permission probe. diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index 579bb80272e6..b615d032526c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1412,6 +1412,12 @@ fn process_identity(pid: u32) -> Option { Some(mac_process_snapshot(pid)?.identity) } +/// Return the kernel PID version used for exact macOS process signaling. +#[cfg(target_os = "macos")] +pub(crate) fn mac_process_pid_version(pid: u32) -> Option { + Some(process_identity(pid)?.started as u32) +} + #[cfg(target_os = "macos")] fn mac_process_snapshot(pid: u32) -> Option { const PROC_PIDT_BSDINFOWITHUNIQID: libc::c_int = 18; From 54906b36cfd3557bd05816932823afcd0b1eb272 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:09:01 -0700 Subject: [PATCH 493/521] test(tui): use imported JSON null --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index e85f877c2e8a..bf83597b346b 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -21112,7 +21112,7 @@ mod tests { assert_eq!(restored_terminal["id"], terminal_public_id.as_str()); assert_eq!(restored_terminal["lifecycle"], "exited"); assert_eq!(restored_terminal["running"], false); - assert_eq!(restored_terminal["tab_id"], serde_json::Value::Null); + assert_eq!(restored_terminal["tab_id"], Value::Null); assert_eq!(restored_terminal["tab_ids"], serde_json::json!([])); assert_eq!(reopened.resource_surface_for_terminal(&terminal_public_id), None); let exited = From 803d135013b84fab3797dea4c22dc27259ab4624 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:16:39 -0700 Subject: [PATCH 494/521] fix(tui): retain resource replay revision --- .../src/mux/resource_topology.rs | 19 +++++++++++++++- .../cmux-tui-core/src/workspace_registry.rs | 22 ++++++++++--------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs index c9a0cf452a22..6dff76181da0 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux/resource_topology.rs @@ -2436,7 +2436,24 @@ impl Mux { &projection.changes, )?; let (terminal, resource) = match committed { - TerminalResourceCloseCommit::Replay(terminal) => { + TerminalResourceCloseCommit::TerminalReplay(terminal) => { + let result = TerminalCloseResult { + surface: None, + terminal_id: terminal_id.to_string(), + terminal_incarnation: terminal.result["incarnation"] + .as_str() + .map(str::to_string), + already_closed: terminal.result["already_closed"].as_bool().unwrap_or(false), + terminal_revision: terminal.revision, + }; + drop(state); + drop(registry); + drop(_creation_fence); + drop(_creation_handoff); + return Ok(Some(result)); + } + TerminalResourceCloseCommit::ResourceReplay { terminal, resource } => { + state.resource_revision = state.resource_revision.max(resource.revision); let result = TerminalCloseResult { surface: None, terminal_id: terminal_id.to_string(), diff --git a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs index 588c046a9f7b..cd265115103a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs +++ b/cmux-tui/crates/cmux-tui-core/src/workspace_registry.rs @@ -359,10 +359,12 @@ pub struct TerminalRegistryCommit { pub replayed: bool, } -/// A host mutation replay cannot acquire a public-resource side effect that -/// was not part of its original transaction. +/// A replay cannot acquire close effects that were not part of its original +/// transaction. Resource replays still carry their committed revision so the +/// in-memory projection can catch up without terminating a live runtime. pub(crate) enum TerminalResourceCloseCommit { - Replay(TerminalRegistryCommit), + TerminalReplay(TerminalRegistryCommit), + ResourceReplay { terminal: TerminalRegistryCommit, resource: ResourcePatchCommit }, Committed { terminal: TerminalRegistryCommit, resource: ResourcePatchCommit }, } @@ -3066,9 +3068,10 @@ impl WorkspaceRegistry { let tx = self.connection.transaction()?; if let Some(terminal) = terminal_replay(&tx, mutation, &fingerprint)? { tx.commit()?; - return Ok(TerminalResourceCloseCommit::Replay(terminal)); + return Ok(TerminalResourceCloseCommit::TerminalReplay(terminal)); } - if resource_store::resource_patch_replay(&tx, mutation, OPERATION, &fingerprint)?.is_some() + if let Some(resource) = + resource_store::resource_patch_replay(&tx, mutation, OPERATION, &fingerprint)? { let terminal = read_terminal(&tx, terminal_id)?.context("terminal close state is unavailable")?; @@ -3084,11 +3087,10 @@ impl WorkspaceRegistry { "already_closed": true, }); tx.commit()?; - return Ok(TerminalResourceCloseCommit::Replay(TerminalRegistryCommit { - revision, - result, - replayed: true, - })); + return Ok(TerminalResourceCloseCommit::ResourceReplay { + terminal: TerminalRegistryCommit { revision, result, replayed: true }, + resource, + }); } let terminal = close_terminal_in_transaction( &tx, From 4bdf0247e77feca2432f0b8637dbe25a140012c1 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:23:59 -0700 Subject: [PATCH 495/521] test(tui): preserve process identity across exec --- .../cmux-tui-core/src/process_session.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index bb8a95652720..b4a578f5be76 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2232,6 +2232,38 @@ mod tests { assert!(process.signal(0).unwrap()); } + #[cfg(target_os = "macos")] + #[test] + fn macos_process_identity_survives_exec() { + use std::io::{BufRead, BufReader, Write}; + + let mut child = Command::new("/bin/sh") + .arg("-c") + .arg("read _; exec /bin/sh -c 'printf \"ready\\n\"; read _'") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = libc::pid_t::try_from(child.id()).unwrap(); + let process = StableProcessHandle::capture(pid).unwrap().unwrap(); + let mut input = child.stdin.take().unwrap(); + let mut output = BufReader::new(child.stdout.take().unwrap()); + + writeln!(input, "continue").unwrap(); + input.flush().unwrap(); + let mut ready = String::new(); + output.read_line(&mut ready).unwrap(); + assert_eq!(ready, "ready\n"); + + assert!(process.matches_current().unwrap()); + assert!(process.signal(0).unwrap()); + + writeln!(input, "finish").unwrap(); + input.flush().unwrap(); + assert!(child.wait().unwrap().success()); + } + #[cfg(target_os = "macos")] #[test] fn macos_session_query_is_scoped_to_the_owned_session() { From a531585a4254b5c29c877b2aa57ec0c3f0e04f44 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:25:02 -0700 Subject: [PATCH 496/521] test(tui): cover persisted close and child identity --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 8 ++++---- cmux-tui/crates/cmux-tui-core/src/process_session.rs | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bf83597b346b..6c30d9255150 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -29791,10 +29791,10 @@ mod tests { drop(mux); let reopened = Mux::open_persistent("recover-exited", options, &root).unwrap(); - let exited = reopened.resolve_terminal(TERMINAL).unwrap().unwrap(); - assert_eq!(exited.surface, None); - assert_eq!(exited.terminal.lifecycle, TerminalLifecycle::Exited); - assert_eq!(exited.terminal.exit.unwrap()["outcome"]["reason"], "persisted-exit"); + let tombstoned = reopened.resolve_terminal(TERMINAL).unwrap().unwrap(); + assert_eq!(tombstoned.surface, None); + assert_eq!(tombstoned.terminal.lifecycle, TerminalLifecycle::Tombstoned); + assert_eq!(tombstoned.terminal.exit.unwrap()["outcome"]["reason"], "persisted-exit"); let _ = reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index bb8a95652720..59146d92f3e4 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2222,14 +2222,17 @@ mod tests { #[cfg(target_os = "macos")] #[test] - fn macos_process_identity_survives_unavailable_task_name_access() { + fn macos_child_process_identity_survives_unavailable_task_name_access() { + let mut child = spawn_session_child(); let _failure = force_task_name_for_pid_failure_for_test(); - let pid = libc::pid_t::try_from(std::process::id()).unwrap(); + let pid = libc::pid_t::try_from(child.id()).unwrap(); let process = StableProcessHandle::capture(pid).unwrap().unwrap(); assert!(process.matches_current().unwrap()); assert!(process.signal(0).unwrap()); + child.kill().unwrap(); + child.wait().unwrap(); } #[cfg(target_os = "macos")] From f24608dcfaf28d30363a56225140d137d4cf7ac4 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:25:27 -0700 Subject: [PATCH 497/521] fix(tui): read child PID versions directly --- .../cmux-tui-core/src/unix_process_scope.rs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index b615d032526c..e68460db6231 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1415,7 +1415,26 @@ fn process_identity(pid: u32) -> Option { /// Return the kernel PID version used for exact macOS process signaling. #[cfg(target_os = "macos")] pub(crate) fn mac_process_pid_version(pid: u32) -> Option { - Some(process_identity(pid)?.started as u32) + const PROC_PIDUNIQIDENTIFIERINFO: libc::c_int = 17; + let pid_int = libc::c_int::try_from(pid).ok()?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = libc::c_int::try_from(size_of::()).ok()?; + let written = unsafe { + libc::proc_pidinfo( + pid_int, + PROC_PIDUNIQIDENTIFIERINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if written != size { + return None; + } + // SAFETY: proc_pidinfo initialized the full structure. This narrow flavor + // exposes the kernel PID version without the BSD metadata access check. + let info = unsafe { info.assume_init() }; + Some(info.id_version as u32) } #[cfg(target_os = "macos")] From 44e505d73e67e9bbb07a1137fe28efe010d07958 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:26:21 -0700 Subject: [PATCH 498/521] fix(tui): retain process ownership across exec --- .../cmux-tui-core/src/process_session.rs | 140 ++++-------------- .../cmux-tui-core/src/unix_process_scope.rs | 81 ++++++++-- 2 files changed, 100 insertions(+), 121 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index b4a578f5be76..0847bf611eb3 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -43,27 +43,6 @@ thread_local! { const { std::cell::Cell::new(false) }; static POST_SPAWN_FAILURE_MARKER: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; - #[cfg(target_os = "macos")] - static FORCE_TASK_NAME_FOR_PID_FAILURE: std::cell::Cell = - const { std::cell::Cell::new(false) }; -} - -#[cfg(all(test, target_os = "macos"))] -struct ForcedTaskNameForPidFailure { - previous: bool, -} - -#[cfg(all(test, target_os = "macos"))] -impl Drop for ForcedTaskNameForPidFailure { - fn drop(&mut self) { - FORCE_TASK_NAME_FOR_PID_FAILURE.set(self.previous); - } -} - -#[cfg(all(test, target_os = "macos"))] -fn force_task_name_for_pid_failure_for_test() -> ForcedTaskNameForPidFailure { - let previous = FORCE_TASK_NAME_FOR_PID_FAILURE.replace(true); - ForcedTaskNameForPidFailure { previous } } #[cfg(test)] @@ -1327,7 +1306,7 @@ fn stable_process_in_session( /// An OS-backed reference to one process instance that cannot retarget after PID reuse. pub struct StableProcessHandle { pid: libc::pid_t, - pid_version: u32, + birth: crate::unix_process_scope::MacProcessBirthIdentity, } #[cfg(target_os = "macos")] @@ -1341,67 +1320,13 @@ struct MacAuditToken { impl StableProcessHandle { /// Capture the process currently using `pid`, or return `None` if it is already gone. pub fn capture(pid: libc::pid_t) -> io::Result> { - let pid_version = match Self::task_pid_version(pid)? { - Some(pid_version) => pid_version, - None => { - let Some(pid_version) = mac_process_pid_version(pid)? else { return Ok(None) }; - pid_version - } - }; - Ok(Some(Self { pid, pid_version })) - } - - fn task_pid_version(pid: libc::pid_t) -> io::Result> { - const TASK_AUDIT_TOKEN: libc::task_flavor_t = 15; - let mut task = 0; - // SAFETY: mach_task_self returns the calling process's send right. - #[allow(deprecated)] - let current_task = unsafe { libc::mach_task_self() }; - // SAFETY: `task` points to writable storage for one task-name right. - #[cfg(test)] - let name_result = if FORCE_TASK_NAME_FOR_PID_FAILURE.get() { - libc::KERN_FAILURE - } else { - unsafe { task_name_for_pid(current_task, pid, &mut task) } - }; - #[cfg(not(test))] - let name_result = unsafe { task_name_for_pid(current_task, pid, &mut task) }; - if name_result != libc::KERN_SUCCESS { - return Ok(None); - } - - let mut audit_token = MacAuditToken { values: [0; 8] }; - let mut count = u32::try_from(size_of::() / size_of::()) - .expect("audit token count fits mach_msg_type_number_t"); - // SAFETY: the token buffer contains `count` natural_t values and the - // returned task-name right remains owned until the call completes. - let info_result = unsafe { - libc::task_info( - task, - TASK_AUDIT_TOKEN, - audit_token.values.as_mut_ptr().cast::(), - &mut count, - ) - }; - // SAFETY: `task` is the send right returned by task_name_for_pid. - let _ = unsafe { mach_port_deallocate(current_task, task) }; - if info_result != libc::KERN_SUCCESS { - return Ok(None); - } - let expected_pid = u32::try_from(pid) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; - if count != 8 || audit_token.values[5] != expected_pid { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "stable process identity returned a mismatched PID", - )); - } - Ok(Some(audit_token.values[7])) + let Some(birth) = mac_process_birth_identity(pid)? else { return Ok(None) }; + Ok(Some(Self { pid, birth })) } /// Return whether this exact process instance is still alive. pub fn matches_current(&self) -> io::Result { - Ok(Self::capture(self.pid)?.is_some_and(|current| current == *self)) + Ok(mac_process_birth_identity(self.pid)?.is_some_and(|birth| birth == self.birth)) } /// Signal this exact process instance, returning false if it is already gone. @@ -1409,9 +1334,23 @@ impl StableProcessHandle { let signal_process = resolve_proc_signal_with_audittoken()?; let expected_pid = u32::try_from(self.pid) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; + let Some(current) = crate::unix_process_scope::mac_process_signal_identity(expected_pid) + else { + return match mac_process_birth_identity(self.pid)? { + None => Ok(false), + Some(birth) if birth != self.birth => Ok(false), + Some(_) => Err(io::Error::other(format!( + "cannot acquire exact signal identity for PID {}", + self.pid + ))), + }; + }; + if current.birth != self.birth { + return Ok(false); + } let mut token = MacAuditToken { values: [u32::MAX; 8] }; token.values[5] = expected_pid; - token.values[7] = self.pid_version; + token.values[7] = current.pid_version; loop { // SAFETY: the PID and PID version came from kernel process // metadata; libproc validates them together before signaling. @@ -1434,17 +1373,19 @@ impl StableProcessHandle { } #[cfg(target_os = "macos")] -fn mac_process_pid_version(pid: libc::pid_t) -> io::Result> { +fn mac_process_birth_identity( + pid: libc::pid_t, +) -> io::Result> { let expected_pid = u32::try_from(pid) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; - if let Some(pid_version) = crate::unix_process_scope::mac_process_pid_version(expected_pid) { - return Ok(Some(pid_version)); + if let Some(birth) = crate::unix_process_scope::mac_process_birth_identity(expected_pid) { + return Ok(Some(birth)); } if process_is_gone(pid)? { Ok(None) } else { Err(io::Error::other(format!( - "cannot acquire stable process identity for PID {pid}: kernel PID version is unavailable" + "cannot acquire stable process identity for PID {pid}: kernel process birth is unavailable" ))) } } @@ -1463,19 +1404,6 @@ fn process_is_gone(pid: libc::pid_t) -> io::Result { } } -#[cfg(target_os = "macos")] -unsafe extern "C" { - fn task_name_for_pid( - target_task: libc::mach_port_t, - pid: libc::pid_t, - task: *mut libc::mach_port_t, - ) -> libc::kern_return_t; - fn mach_port_deallocate( - task: libc::mach_port_t, - name: libc::mach_port_t, - ) -> libc::kern_return_t; -} - #[cfg(target_os = "macos")] type ProcSignalWithAuditToken = unsafe extern "C" fn(*mut MacAuditToken, libc::c_int) -> libc::c_int; @@ -2220,18 +2148,6 @@ mod tests { true } - #[cfg(target_os = "macos")] - #[test] - fn macos_process_identity_survives_unavailable_task_name_access() { - let _failure = force_task_name_for_pid_failure_for_test(); - let pid = libc::pid_t::try_from(std::process::id()).unwrap(); - - let process = StableProcessHandle::capture(pid).unwrap().unwrap(); - - assert!(process.matches_current().unwrap()); - assert!(process.signal(0).unwrap()); - } - #[cfg(target_os = "macos")] #[test] fn macos_process_identity_survives_exec() { @@ -2256,12 +2172,14 @@ mod tests { output.read_line(&mut ready).unwrap(); assert_eq!(ready, "ready\n"); - assert!(process.matches_current().unwrap()); - assert!(process.signal(0).unwrap()); + let matches_after_exec = process.matches_current(); + let signal_after_exec = process.signal(0); writeln!(input, "finish").unwrap(); input.flush().unwrap(); assert!(child.wait().unwrap().success()); + assert!(matches_after_exec.unwrap()); + assert!(signal_after_exec.unwrap()); } #[cfg(target_os = "macos")] diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index b615d032526c..1a1f73e46750 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1248,6 +1248,20 @@ struct MacBsdInfoWithUniqueId { unique: MacProcessUniqueInfo, } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct MacProcessBirthIdentity { + pub(crate) seconds: u64, + pub(crate) microseconds: u64, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct MacProcessSignalIdentity { + pub(crate) birth: MacProcessBirthIdentity, + pub(crate) pid_version: u32, +} + #[cfg(target_os = "macos")] #[repr(C)] struct MacAuditToken { @@ -1412,14 +1426,51 @@ fn process_identity(pid: u32) -> Option { Some(mac_process_snapshot(pid)?.identity) } -/// Return the kernel PID version used for exact macOS process signaling. +/// Return the process birth identity, which remains stable across exec. +#[cfg(target_os = "macos")] +pub(crate) fn mac_process_birth_identity(pid: u32) -> Option { + if let Some(identity) = mac_process_signal_identity(pid) { + return Some(identity.birth); + } + let bsd = mac_process_bsd_info(pid)?; + if bsd.pbi_status == libc::SZOMB { + return None; + } + Some(MacProcessBirthIdentity { + seconds: bsd.pbi_start_tvsec, + microseconds: bsd.pbi_start_tvusec, + }) +} + +/// Return one atomic birth-identity and PID-version record for exact signaling. #[cfg(target_os = "macos")] -pub(crate) fn mac_process_pid_version(pid: u32) -> Option { - Some(process_identity(pid)?.started as u32) +pub(crate) fn mac_process_signal_identity(pid: u32) -> Option { + let info = mac_process_info_with_unique_id(pid)?; + if info.bsd.pbi_status == libc::SZOMB { + return None; + } + Some(MacProcessSignalIdentity { + birth: MacProcessBirthIdentity { + seconds: info.bsd.pbi_start_tvsec, + microseconds: info.bsd.pbi_start_tvusec, + }, + pid_version: info.unique.id_version as u32, + }) } #[cfg(target_os = "macos")] fn mac_process_snapshot(pid: u32) -> Option { + let info = mac_process_info_with_unique_id(pid)?; + // Keep chronological start time in the high bits for scan filtering and + // the kernel PID version in the low bits for race-free signal delivery. + let started = (u128::from(info.bsd.pbi_start_tvsec) << 64) + | (u128::from(info.bsd.pbi_start_tvusec) << 32) + | u128::from(info.unique.id_version as u32); + Some(ProcessSnapshot { identity: ProcessIdentity { pid, started }, parent: info.bsd.pbi_ppid }) +} + +#[cfg(target_os = "macos")] +fn mac_process_info_with_unique_id(pid: u32) -> Option { const PROC_PIDT_BSDINFOWITHUNIQID: libc::c_int = 18; let pid_int = libc::c_int::try_from(pid).ok()?; let mut info = std::mem::MaybeUninit::::zeroed(); @@ -1431,13 +1482,23 @@ fn mac_process_snapshot(pid: u32) -> Option { return None; } // SAFETY: proc_pidinfo initialized the full structure. - let info = unsafe { info.assume_init() }; - // Keep chronological start time in the high bits for scan filtering and - // the kernel PID version in the low bits for race-free signal delivery. - let started = (u128::from(info.bsd.pbi_start_tvsec) << 64) - | (u128::from(info.bsd.pbi_start_tvusec) << 32) - | u128::from(info.unique.id_version as u32); - Some(ProcessSnapshot { identity: ProcessIdentity { pid, started }, parent: info.bsd.pbi_ppid }) + Some(unsafe { info.assume_init() }) +} + +#[cfg(target_os = "macos")] +fn mac_process_bsd_info(pid: u32) -> Option { + let pid_int = libc::c_int::try_from(pid).ok()?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = libc::c_int::try_from(size_of::()).ok()?; + // SAFETY: info points to writable storage of exactly the requested size. + let written = unsafe { + libc::proc_pidinfo(pid_int, libc::PROC_PIDTBSDINFO, 0, info.as_mut_ptr().cast(), size) + }; + if written != size { + return None; + } + // SAFETY: proc_pidinfo initialized the full structure. + Some(unsafe { info.assume_init() }) } #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] From 3a319f8e1d5fdb1c31ae41469fd0a195f017e03f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:26:38 -0700 Subject: [PATCH 499/521] test(tui): preserve terminal tombstone on restart --- cmux-tui/crates/cmux-tui-core/src/mux.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/mux.rs b/cmux-tui/crates/cmux-tui-core/src/mux.rs index bf83597b346b..6c30d9255150 100644 --- a/cmux-tui/crates/cmux-tui-core/src/mux.rs +++ b/cmux-tui/crates/cmux-tui-core/src/mux.rs @@ -29791,10 +29791,10 @@ mod tests { drop(mux); let reopened = Mux::open_persistent("recover-exited", options, &root).unwrap(); - let exited = reopened.resolve_terminal(TERMINAL).unwrap().unwrap(); - assert_eq!(exited.surface, None); - assert_eq!(exited.terminal.lifecycle, TerminalLifecycle::Exited); - assert_eq!(exited.terminal.exit.unwrap()["outcome"]["reason"], "persisted-exit"); + let tombstoned = reopened.resolve_terminal(TERMINAL).unwrap().unwrap(); + assert_eq!(tombstoned.surface, None); + assert_eq!(tombstoned.terminal.lifecycle, TerminalLifecycle::Tombstoned); + assert_eq!(tombstoned.terminal.exit.unwrap()["outcome"]["reason"], "persisted-exit"); let _ = reopened.shutdown(); drop(reopened); std::fs::remove_dir_all(root).unwrap(); From c065d4136589d4bcf81aac80fbb67b7aa8d8e575 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:27:03 -0700 Subject: [PATCH 500/521] fix(tui): close failed renderer writers --- cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs index f68816b11b10..6a5365b3713f 100644 --- a/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs +++ b/cmux-tui/crates/cmux-tui-core/src/terminal_host_runtime.rs @@ -6930,6 +6930,10 @@ mod unix { break; } } + // This writer loop is the final owner of the connection. Closing the + // socket here also wakes the cloned input reader when a write timeout + // ends the loop before byte-queue overflow can close the tap. + tap.close(); host.remove_client(client); Ok(()) } From fc002c2613c5a541b7c1c0debdf678c77150e63f Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:56:01 -0700 Subject: [PATCH 501/521] test(tui): cover restricted child signaling --- .../cmux-tui-core/src/process_session.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 0847bf611eb3..2e5c6dc7742a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2182,6 +2182,25 @@ mod tests { assert!(signal_after_exec.unwrap()); } + #[cfg(target_os = "macos")] + #[test] + fn macos_child_process_has_an_exact_signal_identity() { + let mut child = spawn_session_child(); + let pid = libc::pid_t::try_from(child.id()).unwrap(); + let result = (|| -> io::Result<(bool, bool)> { + let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { + io::Error::new(io::ErrorKind::NotFound, "child exited before identity capture") + })?; + Ok((process.matches_current()?, process.signal(0)?)) + })(); + let _ = child.kill(); + let _ = child.wait(); + + let (matches, signaled) = result.unwrap(); + assert!(matches); + assert!(signaled); + } + #[cfg(target_os = "macos")] #[test] fn macos_session_query_is_scoped_to_the_owned_session() { From 0630967f6b341face44a9ffe8a750e5eeabf84d5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:57:36 -0700 Subject: [PATCH 502/521] fix(tui): bracket restricted process identity --- .../cmux-tui-core/src/unix_process_scope.rs | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index 1a1f73e46750..9c9845d3a57d 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1433,27 +1433,15 @@ pub(crate) fn mac_process_birth_identity(pid: u32) -> Option Option { let info = mac_process_info_with_unique_id(pid)?; - if info.bsd.pbi_status == libc::SZOMB { - return None; - } Some(MacProcessSignalIdentity { - birth: MacProcessBirthIdentity { - seconds: info.bsd.pbi_start_tvsec, - microseconds: info.bsd.pbi_start_tvusec, - }, + birth: mac_process_birth_from_bsd(&info.bsd)?, pid_version: info.unique.id_version as u32, }) } @@ -1471,6 +1459,25 @@ fn mac_process_snapshot(pid: u32) -> Option { #[cfg(target_os = "macos")] fn mac_process_info_with_unique_id(pid: u32) -> Option { + if let Some(info) = mac_process_combined_info_with_unique_id(pid) { + return Some(info); + } + + // Some macOS process-security contexts deny the combined BSD and unique + // record. Bracket the unrestricted unique record with process births so a + // recycled PID can never pair a prior owner with a replacement's version. + let before = mac_process_bsd_info(pid)?; + let before_birth = mac_process_birth_from_bsd(&before)?; + let unique = mac_process_unique_info(pid)?; + let after = mac_process_bsd_info(pid)?; + if before_birth != mac_process_birth_from_bsd(&after)? { + return None; + } + Some(MacBsdInfoWithUniqueId { bsd: after, unique }) +} + +#[cfg(target_os = "macos")] +fn mac_process_combined_info_with_unique_id(pid: u32) -> Option { const PROC_PIDT_BSDINFOWITHUNIQID: libc::c_int = 18; let pid_int = libc::c_int::try_from(pid).ok()?; let mut info = std::mem::MaybeUninit::::zeroed(); @@ -1485,6 +1492,28 @@ fn mac_process_info_with_unique_id(pid: u32) -> Option { Some(unsafe { info.assume_init() }) } +#[cfg(target_os = "macos")] +fn mac_process_unique_info(pid: u32) -> Option { + const PROC_PIDUNIQIDENTIFIERINFO: libc::c_int = 17; + let pid_int = libc::c_int::try_from(pid).ok()?; + let mut info = std::mem::MaybeUninit::::zeroed(); + let size = libc::c_int::try_from(size_of::()).ok()?; + let written = unsafe { + libc::proc_pidinfo( + pid_int, + PROC_PIDUNIQIDENTIFIERINFO, + 0, + info.as_mut_ptr().cast(), + size, + ) + }; + if written != size { + return None; + } + // SAFETY: proc_pidinfo initialized the full structure. + Some(unsafe { info.assume_init() }) +} + #[cfg(target_os = "macos")] fn mac_process_bsd_info(pid: u32) -> Option { let pid_int = libc::c_int::try_from(pid).ok()?; @@ -1501,6 +1530,17 @@ fn mac_process_bsd_info(pid: u32) -> Option { Some(unsafe { info.assume_init() }) } +#[cfg(target_os = "macos")] +fn mac_process_birth_from_bsd(info: &libc::proc_bsdinfo) -> Option { + if info.pbi_status == libc::SZOMB { + return None; + } + Some(MacProcessBirthIdentity { + seconds: info.pbi_start_tvsec, + microseconds: info.pbi_start_tvusec, + }) +} + #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] fn scan_registered_processes( _scopes: &[ScopeRegistration], From 2f5674672345a6309510a2184190de06a4073c21 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:37:51 -0700 Subject: [PATCH 503/521] test(tui): use valid exact child signal --- cmux-tui/crates/cmux-tui-core/src/process_session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 2e5c6dc7742a..43c8bad5d80e 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2191,7 +2191,7 @@ mod tests { let process = StableProcessHandle::capture(pid)?.ok_or_else(|| { io::Error::new(io::ErrorKind::NotFound, "child exited before identity capture") })?; - Ok((process.matches_current()?, process.signal(0)?)) + Ok((process.matches_current()?, process.signal(libc::SIGCONT)?)) })(); let _ = child.kill(); let _ = child.wait(); From 6b9e4686eb62cd638a1e93f9978a75f36eb99944 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:38:19 -0700 Subject: [PATCH 504/521] test(tui): use supported audit-token signals --- cmux-tui/crates/cmux-tui-core/src/process_session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 43c8bad5d80e..49d90ba8876a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -2173,7 +2173,7 @@ mod tests { assert_eq!(ready, "ready\n"); let matches_after_exec = process.matches_current(); - let signal_after_exec = process.signal(0); + let signal_after_exec = process.signal(libc::SIGCONT); writeln!(input, "finish").unwrap(); input.flush().unwrap(); From 4f9bb7ea44eddc6a9bb573e5ae78f9486435fcf8 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:41:00 -0700 Subject: [PATCH 505/521] fix(tui): own macOS processes by unique ID --- .../cmux-tui-core/src/process_session.rs | 29 +++++----- .../cmux-tui-core/src/unix_process_scope.rs | 58 ++++++++----------- 2 files changed, 39 insertions(+), 48 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index 49d90ba8876a..bd1e0d83cecd 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1287,7 +1287,7 @@ fn stable_process_in_session( session: libc::pid_t, ) -> io::Result> { let Some(process) = StableProcessHandle::capture(pid)? else { return Ok(None) }; - // Bracket the PID-based session query with a stable process identity. If + // Bracket the PID-based session query with a stable process instance. If // the PID is recycled during getsid(2), the final identity check fails // closed and no signal is sent. let current_session = unsafe { libc::getsid(pid) }; @@ -1306,7 +1306,7 @@ fn stable_process_in_session( /// An OS-backed reference to one process instance that cannot retarget after PID reuse. pub struct StableProcessHandle { pid: libc::pid_t, - birth: crate::unix_process_scope::MacProcessBirthIdentity, + instance: crate::unix_process_scope::MacProcessInstanceIdentity, } #[cfg(target_os = "macos")] @@ -1320,13 +1320,14 @@ struct MacAuditToken { impl StableProcessHandle { /// Capture the process currently using `pid`, or return `None` if it is already gone. pub fn capture(pid: libc::pid_t) -> io::Result> { - let Some(birth) = mac_process_birth_identity(pid)? else { return Ok(None) }; - Ok(Some(Self { pid, birth })) + let Some(instance) = mac_process_instance_identity(pid)? else { return Ok(None) }; + Ok(Some(Self { pid, instance })) } /// Return whether this exact process instance is still alive. pub fn matches_current(&self) -> io::Result { - Ok(mac_process_birth_identity(self.pid)?.is_some_and(|birth| birth == self.birth)) + Ok(mac_process_instance_identity(self.pid)? + .is_some_and(|instance| instance == self.instance)) } /// Signal this exact process instance, returning false if it is already gone. @@ -1336,16 +1337,16 @@ impl StableProcessHandle { .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; let Some(current) = crate::unix_process_scope::mac_process_signal_identity(expected_pid) else { - return match mac_process_birth_identity(self.pid)? { + return match mac_process_instance_identity(self.pid)? { None => Ok(false), - Some(birth) if birth != self.birth => Ok(false), + Some(instance) if instance != self.instance => Ok(false), Some(_) => Err(io::Error::other(format!( "cannot acquire exact signal identity for PID {}", self.pid ))), }; }; - if current.birth != self.birth { + if current.instance != self.instance { return Ok(false); } let mut token = MacAuditToken { values: [u32::MAX; 8] }; @@ -1373,19 +1374,21 @@ impl StableProcessHandle { } #[cfg(target_os = "macos")] -fn mac_process_birth_identity( +fn mac_process_instance_identity( pid: libc::pid_t, -) -> io::Result> { +) -> io::Result> { let expected_pid = u32::try_from(pid) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; - if let Some(birth) = crate::unix_process_scope::mac_process_birth_identity(expected_pid) { - return Ok(Some(birth)); + if let Some(instance) = + crate::unix_process_scope::mac_process_instance_identity(expected_pid) + { + return Ok(Some(instance)); } if process_is_gone(pid)? { Ok(None) } else { Err(io::Error::other(format!( - "cannot acquire stable process identity for PID {pid}: kernel process birth is unavailable" + "cannot acquire stable process identity for PID {pid}: kernel process instance is unavailable" ))) } } diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index 9c9845d3a57d..8b659c88ca70 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1233,7 +1233,7 @@ struct ProcFileInfo { #[repr(C)] struct MacProcessUniqueInfo { _uuid: [u8; 16], - _unique_id: u64, + unique_id: u64, _parent_unique_id: u64, id_version: i32, _original_parent_id_version: i32, @@ -1250,15 +1250,14 @@ struct MacBsdInfoWithUniqueId { #[cfg(target_os = "macos")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct MacProcessBirthIdentity { - pub(crate) seconds: u64, - pub(crate) microseconds: u64, +pub(crate) struct MacProcessInstanceIdentity { + unique_id: u64, } #[cfg(target_os = "macos")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct MacProcessSignalIdentity { - pub(crate) birth: MacProcessBirthIdentity, + pub(crate) instance: MacProcessInstanceIdentity, pub(crate) pid_version: u32, } @@ -1426,23 +1425,20 @@ fn process_identity(pid: u32) -> Option { Some(mac_process_snapshot(pid)?.identity) } -/// Return the process birth identity, which remains stable across exec. +/// Return the kernel process instance identity, which remains stable across exec. #[cfg(target_os = "macos")] -pub(crate) fn mac_process_birth_identity(pid: u32) -> Option { - if let Some(identity) = mac_process_signal_identity(pid) { - return Some(identity.birth); - } - let bsd = mac_process_bsd_info(pid)?; - mac_process_birth_from_bsd(&bsd) +pub(crate) fn mac_process_instance_identity(pid: u32) -> Option { + let unique = mac_process_unique_info(pid)?; + Some(mac_process_instance_from_unique(&unique)) } -/// Return one birth-identity and PID-version record safe for exact signaling. +/// Return one instance-identity and PID-version record for exact signaling. #[cfg(target_os = "macos")] pub(crate) fn mac_process_signal_identity(pid: u32) -> Option { - let info = mac_process_info_with_unique_id(pid)?; + let unique = mac_process_unique_info(pid)?; Some(MacProcessSignalIdentity { - birth: mac_process_birth_from_bsd(&info.bsd)?, - pid_version: info.unique.id_version as u32, + instance: mac_process_instance_from_unique(&unique), + pid_version: unique.id_version as u32, }) } @@ -1450,10 +1446,9 @@ pub(crate) fn mac_process_signal_identity(pid: u32) -> Option Option { let info = mac_process_info_with_unique_id(pid)?; // Keep chronological start time in the high bits for scan filtering and - // the kernel PID version in the low bits for race-free signal delivery. + // the exec-stable process unique ID in the low bits for exact ownership. let started = (u128::from(info.bsd.pbi_start_tvsec) << 64) - | (u128::from(info.bsd.pbi_start_tvusec) << 32) - | u128::from(info.unique.id_version as u32); + | u128::from(info.unique.unique_id); Some(ProcessSnapshot { identity: ProcessIdentity { pid, started }, parent: info.bsd.pbi_ppid }) } @@ -1464,16 +1459,15 @@ fn mac_process_info_with_unique_id(pid: u32) -> Option { } // Some macOS process-security contexts deny the combined BSD and unique - // record. Bracket the unrestricted unique record with process births so a - // recycled PID can never pair a prior owner with a replacement's version. - let before = mac_process_bsd_info(pid)?; - let before_birth = mac_process_birth_from_bsd(&before)?; - let unique = mac_process_unique_info(pid)?; - let after = mac_process_bsd_info(pid)?; - if before_birth != mac_process_birth_from_bsd(&after)? { + // record. Bracket BSD metadata with the exec-stable unique ID so a + // recycled PID can never publish a replacement process as the prior owner. + let before = mac_process_unique_info(pid)?; + let bsd = mac_process_bsd_info(pid)?; + let after = mac_process_unique_info(pid)?; + if before.unique_id != after.unique_id { return None; } - Some(MacBsdInfoWithUniqueId { bsd: after, unique }) + Some(MacBsdInfoWithUniqueId { bsd, unique: after }) } #[cfg(target_os = "macos")] @@ -1531,14 +1525,8 @@ fn mac_process_bsd_info(pid: u32) -> Option { } #[cfg(target_os = "macos")] -fn mac_process_birth_from_bsd(info: &libc::proc_bsdinfo) -> Option { - if info.pbi_status == libc::SZOMB { - return None; - } - Some(MacProcessBirthIdentity { - seconds: info.pbi_start_tvsec, - microseconds: info.pbi_start_tvusec, - }) +fn mac_process_instance_from_unique(info: &MacProcessUniqueInfo) -> MacProcessInstanceIdentity { + MacProcessInstanceIdentity { unique_id: info.unique_id } } #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] From 505c8363c433489fe753445b04339548163bd80e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:46:15 -0700 Subject: [PATCH 506/521] test(tui): cover unique scope signaling --- cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index 8b659c88ca70..8f47d3dc2221 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1571,6 +1571,15 @@ mod tests { assert!(!mac_environment_contains(&argv_only, expected)); } + #[cfg(target_os = "macos")] + #[test] + fn mac_process_scope_signals_the_current_unique_identity() { + let pid = std::process::id(); + let identity = process_identity(pid).unwrap(); + + assert!(signal_process_with(identity, libc::SIGCONT)); + } + #[cfg(target_os = "macos")] #[test] fn mac_process_scope_launcher_stops_before_the_requested_program() { From 11b2648656ad5e49f9bafbd723db0a7ff9d5f066 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:47:05 -0700 Subject: [PATCH 507/521] test(tui): cover cross-platform scope signaling --- .../cmux-tui-core/src/unix_process_scope.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index 8f47d3dc2221..cd092b9e303a 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1571,13 +1571,20 @@ mod tests { assert!(!mac_environment_contains(&argv_only, expected)); } - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "linux", target_os = "macos"))] #[test] - fn mac_process_scope_signals_the_current_unique_identity() { + fn process_scope_signals_the_current_exact_identity() { let pid = std::process::id(); - let identity = process_identity(pid).unwrap(); - - assert!(signal_process_with(identity, libc::SIGCONT)); + #[cfg(target_os = "macos")] + { + let identity = process_identity(pid).unwrap(); + assert!(signal_process_with(identity, libc::SIGCONT)); + } + #[cfg(target_os = "linux")] + { + let pidfd = pidfd_open(pid).unwrap(); + pidfd_send_signal(&pidfd, libc::SIGCONT).unwrap(); + } } #[cfg(target_os = "macos")] From 95c9cf0d119724cfab2982db2ee0de110e6e8dd7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:53:23 -0700 Subject: [PATCH 508/521] style(tui): apply hosted rustfmt --- cmux-tui/crates/cmux-tui-core/src/process_session.rs | 4 +--- .../crates/cmux-tui-core/src/unix_process_scope.rs | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/process_session.rs b/cmux-tui/crates/cmux-tui-core/src/process_session.rs index bd1e0d83cecd..e69f68f245a9 100644 --- a/cmux-tui/crates/cmux-tui-core/src/process_session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/process_session.rs @@ -1379,9 +1379,7 @@ fn mac_process_instance_identity( ) -> io::Result> { let expected_pid = u32::try_from(pid) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid process id"))?; - if let Some(instance) = - crate::unix_process_scope::mac_process_instance_identity(expected_pid) - { + if let Some(instance) = crate::unix_process_scope::mac_process_instance_identity(expected_pid) { return Ok(Some(instance)); } if process_is_gone(pid)? { diff --git a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs index cd092b9e303a..5c78a037dc84 100644 --- a/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs +++ b/cmux-tui/crates/cmux-tui-core/src/unix_process_scope.rs @@ -1447,8 +1447,7 @@ fn mac_process_snapshot(pid: u32) -> Option { let info = mac_process_info_with_unique_id(pid)?; // Keep chronological start time in the high bits for scan filtering and // the exec-stable process unique ID in the low bits for exact ownership. - let started = (u128::from(info.bsd.pbi_start_tvsec) << 64) - | u128::from(info.unique.unique_id); + let started = (u128::from(info.bsd.pbi_start_tvsec) << 64) | u128::from(info.unique.unique_id); Some(ProcessSnapshot { identity: ProcessIdentity { pid, started }, parent: info.bsd.pbi_ppid }) } @@ -1493,13 +1492,7 @@ fn mac_process_unique_info(pid: u32) -> Option { let mut info = std::mem::MaybeUninit::::zeroed(); let size = libc::c_int::try_from(size_of::()).ok()?; let written = unsafe { - libc::proc_pidinfo( - pid_int, - PROC_PIDUNIQIDENTIFIERINFO, - 0, - info.as_mut_ptr().cast(), - size, - ) + libc::proc_pidinfo(pid_int, PROC_PIDUNIQIDENTIFIERINFO, 0, info.as_mut_ptr().cast(), size) }; if written != size { return None; From f860e5f25bc52c96e345d996d9351dd29498aa7a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:52:37 -0700 Subject: [PATCH 509/521] fix(tui): reconcile hosted compile owners --- cmux-tui/crates/cmux-tui-core/src/server.rs | 18 ++++++- cmux-tui/crates/cmux-tui/src/cli.rs | 41 ++++++++-------- cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs | 24 +++++++++ cmux-tui/crates/cmux-tui/src/cli/wire.rs | 1 + cmux-tui/crates/cmux-tui/src/main.rs | 28 +++++------ cmux-tui/crates/cmux-tui/tests/cli.rs | 49 ++++++++----------- 6 files changed, 97 insertions(+), 64 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/server.rs b/cmux-tui/crates/cmux-tui-core/src/server.rs index e32e399cd395..ba7e64f80265 100644 --- a/cmux-tui/crates/cmux-tui-core/src/server.rs +++ b/cmux-tui/crates/cmux-tui-core/src/server.rs @@ -14810,6 +14810,14 @@ mod tests { self.outbound.push_control(text) } + fn send_control_confirmed( + &self, + text: Arc, + _timeout: Duration, + ) -> std::io::Result<()> { + self.send_control(text) + } + fn send_terminal( &self, text: Arc, @@ -14873,6 +14881,14 @@ mod tests { self.outbound.push_control(text) } + fn send_control_confirmed( + &self, + text: Arc, + _timeout: Duration, + ) -> std::io::Result<()> { + self.send_control(text) + } + fn send_terminal( &self, text: Arc, @@ -20882,7 +20898,7 @@ mod tests { assert_eq!(response["ok"], false); assert!(!mux.daemon_shutdown_requested()); assert!( - !mux.daemon_handoff_pending.load(Ordering::Acquire), + !mux.control_clients.daemon_handoff_pending(), "failed preflight left daemon handoff admission fenced" ); } diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 871af9852113..0e4d5637c8ab 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -108,26 +108,28 @@ pub fn run(args: &[String], startup_usage: &str) -> i32 { } CommandPlan::RawCommand(command) => raw::run(global, command), }, - Err(failure) => { - let message = if matches!(failure.output, OutputMode::Quiet | OutputMode::Human) { - format!("cmux: {}", failure.error) - } else { - failure.error.to_string() - }; - wire::print_local_error( - &serde_json::json!({ - "code":"usage.invalid", - "message":message, - "details":{}, - "retryable":false, - }), - failure.output, - 2, - ) - } + Err(failure) => print_usage_error(failure.error, failure.output), } } +fn print_usage_error(error: UsageError, output: OutputMode) -> i32 { + let message = if matches!(output, OutputMode::Quiet | OutputMode::Human) { + format!("cmux: {error}") + } else { + error.to_string() + }; + wire::print_local_error( + &serde_json::json!({ + "code":"usage.invalid", + "message":message, + "details":{}, + "retryable":false, + }), + output, + 2, + ) +} + fn parse(args: &[String]) -> Result { let (global, command_args) = parse_globals(args).map_err(|(error, output)| ParseFailure { error, output })?; @@ -610,10 +612,9 @@ escape for the legacy control protocol and provides no compatibility promise. fn run_server_lifecycle_if_requested(args: &[String]) -> Option { let (global, command) = match parse_globals(args) { Ok(parsed) => parsed, - Err(error) => { + Err((error, output)) => { if args.iter().any(|value| value == "server") { - eprintln!("cmux: {error}"); - return Some(2); + return Some(print_usage_error(error, output)); } return None; } diff --git a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs index ac3cf67edecb..6ed6dc6c2839 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs @@ -438,6 +438,14 @@ mod tests { Ok(()) } + fn read_timeout(&self) -> io::Result> { + Ok(None) + } + + fn write_timeout(&self) -> io::Result> { + Ok(None) + } + fn shutdown(&self, _: Shutdown) -> io::Result<()> { Ok(()) } @@ -482,6 +490,14 @@ mod tests { Ok(()) } + fn read_timeout(&self) -> io::Result> { + Ok(*self.read_timeout.lock().unwrap()) + } + + fn write_timeout(&self) -> io::Result> { + Ok(*self.write_timeout.lock().unwrap()) + } + fn shutdown(&self, _: Shutdown) -> io::Result<()> { Ok(()) } @@ -518,6 +534,14 @@ mod tests { Ok(()) } + fn read_timeout(&self) -> io::Result> { + Ok(None) + } + + fn write_timeout(&self) -> io::Result> { + Ok(None) + } + fn shutdown(&self, _: Shutdown) -> io::Result<()> { Ok(()) } diff --git a/cmux-tui/crates/cmux-tui/src/cli/wire.rs b/cmux-tui/crates/cmux-tui/src/cli/wire.rs index 9a88082d19e5..d712f6223299 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/wire.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/wire.rs @@ -1,6 +1,7 @@ use std::io::{self, BufRead, BufReader, Read, Write}; #[cfg(unix)] use std::net::Shutdown; +use std::path::PathBuf; use std::time::Duration; use cmux_tui_core::platform::transport; diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 867e6a9e9f50..9145239705c7 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -3339,7 +3339,7 @@ mod tests { let _ = returned.send(()); }); - let result = observed.recv_timeout(std::time::Duration::from_millis(350)); + let result = observed.recv_timeout(Duration::from_millis(350)); if result.is_ok() { worker.join().unwrap(); } @@ -3383,19 +3383,19 @@ mod tests { cmux_tui_core::server::serve_owned(mux.clone(), Some(socket_path.clone())).unwrap(); let cleanup = Arc::new(ServerShutdownCleanup::new_with_process_owners(mux, process_owners)); cleanup.set_shutdown_attempt_for_test(|| Ok(())); - cleanup.set_process_owner_timeout_for_test(std::time::Duration::from_millis(50)); + cleanup.set_process_owner_timeout_for_test(Duration::from_millis(50)); let worker_cleanup = cleanup.clone(); let (returned, observed) = std::sync::mpsc::channel(); let worker = std::thread::spawn(move || { let _ = returned.send(worker_cleanup.run_until_complete()); }); - let bounded = observed.recv_timeout(std::time::Duration::from_millis(350)); + let bounded = observed.recv_timeout(Duration::from_millis(350)); let returned_in_time = bounded.is_ok(); let returned_incomplete = matches!(&bounded, Ok(Err(_))); let retained_remote_owner = !cleanup.process_owners.remote_is_finished(); let mut stream = UnixStream::connect(&socket_path).unwrap(); - stream.set_read_timeout(Some(std::time::Duration::from_secs(1))).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); stream.write_all(b"{\"id\":1,\"cmd\":\"identify\"}\n").unwrap(); stream.flush().unwrap(); let mut response = String::new(); @@ -3449,19 +3449,19 @@ mod tests { let _ = returned.send(result); }); - let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + let first_deadline = std::time::Instant::now() + Duration::from_millis(350); while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { std::thread::yield_now(); } assert_eq!(attempts.load(Ordering::SeqCst), 1); assert!( - observed.recv_timeout(std::time::Duration::from_millis(75)).is_err(), + observed.recv_timeout(Duration::from_millis(75)).is_err(), "a failed cleanup released the server's process owners" ); mux.request_shutdown(); observed - .recv_timeout(std::time::Duration::from_secs(1)) + .recv_timeout(Duration::from_secs(1)) .expect("an explicit shutdown retry did not release the server") .unwrap(); worker.join().unwrap(); @@ -3497,21 +3497,21 @@ mod tests { let _ = returned.send(result); }); - let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + let first_deadline = std::time::Instant::now() + Duration::from_millis(350); while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { std::thread::yield_now(); } assert_eq!(attempts.load(Ordering::SeqCst), 1); assert!( - observed.recv_timeout(std::time::Duration::from_millis(75)).is_err(), + observed.recv_timeout(Duration::from_millis(75)).is_err(), "a panicked cleanup released the server's process owners" ); mux.request_shutdown(); - let retried = observed.recv_timeout(std::time::Duration::from_secs(1)); + let retried = observed.recv_timeout(Duration::from_secs(1)); if retried.is_err() { cleanup.abandon(); - let _ = observed.recv_timeout(std::time::Duration::from_secs(1)); + let _ = observed.recv_timeout(Duration::from_secs(1)); } worker.join().unwrap(); assert!(retried.is_ok(), "an explicit shutdown retry did not recover from a panic"); @@ -3540,15 +3540,15 @@ mod tests { let _ = returned.send(result); }); - let first_deadline = std::time::Instant::now() + std::time::Duration::from_millis(350); + let first_deadline = std::time::Instant::now() + Duration::from_millis(350); while attempts.load(Ordering::SeqCst) == 0 && std::time::Instant::now() < first_deadline { std::thread::yield_now(); } assert_eq!(attempts.load(Ordering::SeqCst), 1); - let early = observed.recv_timeout(std::time::Duration::from_millis(75)); + let early = observed.recv_timeout(Duration::from_millis(75)); if early.is_err() { mux.request_shutdown(); - let _ = observed.recv_timeout(std::time::Duration::from_secs(1)); + let _ = observed.recv_timeout(Duration::from_secs(1)); } worker.join().unwrap(); diff --git a/cmux-tui/crates/cmux-tui/tests/cli.rs b/cmux-tui/crates/cmux-tui/tests/cli.rs index d0c2fb602e35..a0cf93d4d3d1 100644 --- a/cmux-tui/crates/cmux-tui/tests/cli.rs +++ b/cmux-tui/crates/cmux-tui/tests/cli.rs @@ -92,12 +92,6 @@ fn wait_for_child_exit( child.wait().unwrap() } -#[cfg(unix)] -fn native_pty_child(child: &mut (dyn cmux_pty::Child + Send + Sync)) -> &mut Child { - let child: &mut dyn cmux_pty::Child = child; - child.downcast_mut::().expect("cmux-pty Unix test child is a native process") -} - struct HeadlessServer { child: Child, socket: PathBuf, @@ -2024,7 +2018,12 @@ fn named_server_status_stop_alias_and_durable_restart_preserve_topology() { socket.to_str().unwrap(), ]); assert_success(&stop_alias); - assert!(wait_for_child_exit(&mut first, Duration::from_secs(10))); + let status = wait_for_child_exit( + &mut first, + Duration::from_secs(10), + "server remained alive after session stop", + ); + assert!(status.success(), "server exited with {status}"); let mut restarted = spawn(); wait_for_socket_path(&socket); @@ -3579,7 +3578,7 @@ fn plain_launch_attaches_to_existing_local_session() { ); ready.receive(Duration::from_secs(10)); - if let Some(status) = tui.child.try_wait().unwrap() { + if let Some(status) = tui.child.as_mut().unwrap().try_wait().unwrap() { panic!("plain launch exited instead of attaching: {status}"); } let clients = json_cli(&server, &["client", "list"]); @@ -3979,11 +3978,9 @@ fn server_shutdown_exits_when_the_interactive_driver_cannot_progress() { let response = json_socket_request(&socket, serde_json::json!({"id": 1, "cmd": "shutdown"})); assert_eq!(response, serde_json::json!({})); - let status = wait_for_child_exit( - native_pty_child(server.child.as_mut()), - Duration::from_secs(3), - "server remained alive after acknowledging shutdown", - ); + let status = server + .wait_for_exit(Duration::from_secs(3)) + .expect("server remained alive after acknowledging shutdown"); assert!(status.success(), "server exited with {status}"); assert!(transport::connect(&socket).is_err()); drop(server); @@ -4038,17 +4035,15 @@ fn forced_exit_waits_for_remote_runtime_cleanup() { assert_eq!(response, serde_json::json!({})); remote_shutdown_started.receive(Duration::from_secs(3)); assert!( - server.child.try_wait().unwrap().is_none(), + server.child.as_mut().unwrap().try_wait().unwrap().is_none(), "server forced exit before remote runtime cleanup completed" ); remote_shutdown_release.send(); remote_shutdown_complete.receive(Duration::from_secs(5)); - let status = wait_for_child_exit( - native_pty_child(server.child.as_mut()), - Duration::from_secs(5), - "server remained alive after remote runtime cleanup completed", - ); + let status = server + .wait_for_exit(Duration::from_secs(5)) + .expect("server remained alive after remote runtime cleanup completed"); assert!(status.success(), "server exited with {status}"); drop(server); fs::remove_dir_all(dir).unwrap(); @@ -4103,11 +4098,9 @@ fn daemon_handoff_cleans_local_ptys_before_forcing_a_blocked_interactive_driver_ ); assert_eq!(response["accepted"], true); - let status = wait_for_child_exit( - native_pty_child(server.child.as_mut()), - Duration::from_secs(3), - "server remained alive after acknowledging daemon handoff", - ); + let status = server + .wait_for_exit(Duration::from_secs(3)) + .expect("server remained alive after acknowledging daemon handoff"); assert!(status.success(), "server exited with {status}"); assert!(transport::connect(&socket).is_err()); let cleaned = !process_is_active(local_pid) && !process_group_is_active(local_pid); @@ -5108,11 +5101,9 @@ fn configured_websocket_server_does_not_attach_to_existing_session() { &["--socket", server.socket.to_str().unwrap()], &[("CMUX_TUI_CONFIG", config.as_os_str())], ); - let status = wait_for_child_exit( - native_pty_child(tui.child.as_mut()), - Duration::from_secs(10), - "configured WebSocket server attached instead of preserving server mode", - ); + let status = tui + .wait_for_exit(Duration::from_secs(10)) + .expect("configured WebSocket server attached instead of preserving server mode"); assert!(!status.success(), "server launch unexpectedly succeeded"); } From 7fe82bae60ba337eb029316683813f412fcc6f05 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:01:08 -0700 Subject: [PATCH 510/521] fix(ci): verify PyPI artifact version --- .github/workflows/cmux-tui-build-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 5a0a0c021310..89879646bbad 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -750,7 +750,7 @@ jobs: /tmp/cmux-tui-wheel-smoke/bin/python -m pip install --no-index --find-links dist/pypi-wheels cmux=="$PYPI_VERSION" python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ --binary /tmp/cmux-tui-wheel-smoke/bin/cmux \ - --version "$NPM_VERSION" \ + --version "$PYPI_VERSION" \ --build-commit "$(git rev-parse HEAD)" \ --ghostty-commit "$(git rev-parse HEAD:ghostty)" From 84ebf7e0e741b339a80b693ce77803d45418272a Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:06:59 -0700 Subject: [PATCH 511/521] fix(tui): align merged shutdown checks --- cmux-tui/crates/cmux-tui-core/src/resource_router/session.rs | 2 +- cmux-tui/crates/cmux-tui/src/app.rs | 2 ++ cmux-tui/crates/cmux-tui/src/main.rs | 4 ++-- cmux-tui/crates/cmux-tui/src/session/mod.rs | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-tui-core/src/resource_router/session.rs b/cmux-tui/crates/cmux-tui-core/src/resource_router/session.rs index 9d2f7a92aa17..e7290c6f968c 100644 --- a/cmux-tui/crates/cmux-tui-core/src/resource_router/session.rs +++ b/cmux-tui/crates/cmux-tui-core/src/resource_router/session.rs @@ -315,7 +315,7 @@ mod tests { #[test] fn stopped_owner_reload_replays_its_known_failure_receipt() { let mux = Mux::new_for_test("session-reload-failure", SurfaceOptions::default()); - mux.shutdown(); + mux.shutdown().unwrap(); let reload = || { dispatch( &mux, diff --git a/cmux-tui/crates/cmux-tui/src/app.rs b/cmux-tui/crates/cmux-tui/src/app.rs index 7b17432a5f09..de113a1a3051 100644 --- a/cmux-tui/crates/cmux-tui/src/app.rs +++ b/cmux-tui/crates/cmux-tui/src/app.rs @@ -2199,6 +2199,7 @@ impl OrderedSession { self.inner.begin_shutdown(); } + #[cfg(test)] fn daemon_shutdown_requested(&self) -> bool { self.inner.daemon_shutdown_requested() } @@ -8614,6 +8615,7 @@ impl App { } } + #[cfg(test)] fn owner_shutdown_requested(&self) -> bool { self.owner_mux.as_ref().is_some_and(|mux| mux.daemon_shutdown_requested()) } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 9145239705c7..60d43b4062cb 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -2771,13 +2771,13 @@ fn run_server( run_headless(&mux, &socket_path, || false) } } else if let Some(runtime) = machine_runtime { - run_machine_client(runtime, mux.clone()) + run_machine_client(runtime, mux) } else { match RemoteSession::connect(&socket_path) .context("connect the interactive client to its session server") { Ok(remote) => { - run_tui_with_owner(Session::Remote(remote), args.session, None, Some(mux.clone())) + run_tui_with_owner(Session::Remote(remote), args.session, None, Some(mux)) } Err(error) => Err(error), } diff --git a/cmux-tui/crates/cmux-tui/src/session/mod.rs b/cmux-tui/crates/cmux-tui/src/session/mod.rs index 07aa8c536d0f..1b99eb87cab7 100644 --- a/cmux-tui/crates/cmux-tui/src/session/mod.rs +++ b/cmux-tui/crates/cmux-tui/src/session/mod.rs @@ -485,6 +485,7 @@ impl Session { } } + #[cfg(test)] pub fn daemon_shutdown_requested(&self) -> bool { match self { Session::Local(mux) => mux.daemon_shutdown_requested(), From 32285489c94939121b34be11317fdc3313a723a9 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:24:21 -0700 Subject: [PATCH 512/521] test(macOS): release fake runtime pointers through override --- .../TerminalSurfaceRestoreSpawnSchedulerTests.swift | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 0d95b6cbd172..2660df9eeb68 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -736,7 +736,12 @@ import CmuxTerminalCore scheduler: RecordingRestoreSpawnScheduler(), runtimeTeardown: TerminalSurfaceRuntimeTeardownCoordinator() ) - TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { _ in } + let freedPointerBits = OSAllocatedUnfairLock(initialState: [UInt]()) + TerminalSurface.runtimeSurfaceFreeOverrideForTesting = { pointer in + freedPointerBits.withLock { + $0.append(UInt(bitPattern: pointer)) + } + } defer { fixture.surface.releaseSurfaceForTesting() TerminalSurface.runtimeSurfaceFreeOverrideForTesting = nil @@ -754,6 +759,8 @@ import CmuxTerminalCore #expect( fixture.paneHost.activeRuntimeSurfaceCreationFailureMessage == nil ) + fixture.surface.releaseSurfaceForTesting() + #expect(freedPointerBits.withLock { $0 } == [UInt(0x7542)]) } @Test func stalledCloseWorkersFailDeferredCreationAndRecoverSafely() async throws { From bcab6ccb4908b936e540d0d61f24eba823c7510e Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:25:11 -0700 Subject: [PATCH 513/521] fix(macOS): honor test surface free overrides --- .../CmuxTerminal/Surface/TerminalSurface+Debug.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift index caa6bfc8c2fb..fbdf1c6547d6 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Surface/TerminalSurface+Debug.swift @@ -198,7 +198,11 @@ extension TerminalSurface { registry.unregisterRuntimeSurface(surfaceToFree, ownerId: id) surface = nil - ghostty_surface_free(surfaceToFree) + if let freeSurface = Self.runtimeSurfaceFreeOverrideForTesting { + freeSurface(surfaceToFree) + } else { + ghostty_surface_free(surfaceToFree) + } callbackContext?.release() if let runtimeOwnershipReservation { runtimeTeardown.cancelRuntimeSurfaceOwnership( From 19507d4656d3ab2e65b93cda5239e97e3caed021 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:34:48 -0700 Subject: [PATCH 514/521] fix(tui): restore canonical lifecycle CLI routing --- .github/workflows/cmux-tui-build-package.yml | 12 +- cmux-tui/crates/cmux-tui/src/cli.rs | 166 ------------------- cmux-tui/crates/cmux-tui/src/main.rs | 4 + 3 files changed, 15 insertions(+), 167 deletions(-) diff --git a/.github/workflows/cmux-tui-build-package.yml b/.github/workflows/cmux-tui-build-package.yml index 89879646bbad..e3f8dab58ca6 100644 --- a/.github/workflows/cmux-tui-build-package.yml +++ b/.github/workflows/cmux-tui-build-package.yml @@ -578,6 +578,7 @@ jobs: permissions: contents: read env: + BINARY_VERSION: ${{ inputs.version }} NPM_VERSION: ${{ inputs.version }} PYPI_VERSION: ${{ inputs.pypi_version != '' && inputs.pypi_version || inputs.version }} steps: @@ -748,9 +749,18 @@ jobs: dist/binaries/cmux-tui-x86_64-unknown-linux-musl --help >/tmp/cmux-tui-version.txt 2>&1 python3 -m venv /tmp/cmux-tui-wheel-smoke /tmp/cmux-tui-wheel-smoke/bin/python -m pip install --no-index --find-links dist/pypi-wheels cmux=="$PYPI_VERSION" + /tmp/cmux-tui-wheel-smoke/bin/python - <<'PY' + import importlib.metadata + import os + + expected = os.environ["PYPI_VERSION"] + actual = importlib.metadata.version("cmux") + if actual != expected: + raise SystemExit(f"installed wheel version {actual!r} != {expected!r}") + PY python3 cmux-tui/dist/scripts/verify_artifact_identity.py \ --binary /tmp/cmux-tui-wheel-smoke/bin/cmux \ - --version "$PYPI_VERSION" \ + --version "$BINARY_VERSION" \ --build-commit "$(git rev-parse HEAD)" \ --ghostty-commit "$(git rev-parse HEAD:ghostty)" diff --git a/cmux-tui/crates/cmux-tui/src/cli.rs b/cmux-tui/crates/cmux-tui/src/cli.rs index 0e4d5637c8ab..8eae674a163b 100644 --- a/cmux-tui/crates/cmux-tui/src/cli.rs +++ b/cmux-tui/crates/cmux-tui/src/cli.rs @@ -13,10 +13,7 @@ use std::borrow::Cow; use std::io::{self, Write}; use std::path::PathBuf; -use cmux_tui_core::release::ReleaseIdentity; -use cmux_tui_core::server::PROTOCOL_VERSION; use command::{CommandPlan, ParsedCommand}; -use serde_json::json; const PUBLIC_SCOPES: &[&str] = &[ "machine", @@ -83,9 +80,6 @@ pub fn is_public_scope(value: &str) -> bool { } pub fn run(args: &[String], startup_usage: &str) -> i32 { - if let Some(result) = run_server_lifecycle_if_requested(args) { - return result; - } match parse(args) { Ok(ParsedCommand::Help(scope)) => { if scope.as_deref() == Some("start") { @@ -609,166 +603,6 @@ USAGE escape for the legacy control protocol and provides no compatibility promise. "; -fn run_server_lifecycle_if_requested(args: &[String]) -> Option { - let (global, command) = match parse_globals(args) { - Ok(parsed) => parsed, - Err((error, output)) => { - if args.iter().any(|value| value == "server") { - return Some(print_usage_error(error, output)); - } - return None; - } - }; - if command.first().map(String::as_str) != Some("server") { - return None; - } - let actions = &command[1..]; - if actions.iter().any(|value| matches!(value.as_str(), "-h" | "--help" | "help")) { - let messages = &crate::localization::catalog().server; - let mut stdout = io::stdout().lock(); - let _ = stdout.write_all(messages.help.as_bytes()); - let _ = stdout.flush(); - return Some(0); - } - Some(run_server_lifecycle(&global, actions)) -} - -fn run_server_lifecycle(global: &GlobalArgs, actions: &[String]) -> i32 { - let messages = &crate::localization::catalog().server; - let action = match actions { - [action] if matches!(action.as_str(), "status" | "stop") => action.as_str(), - [] => { - eprintln!("cmux: {}", messages.missing_action); - return 2; - } - [action] => { - eprintln!("cmux: {} {action:?}", messages.unknown_action); - return 2; - } - _ => { - eprintln!("cmux: {} {actions:?}", messages.unknown_action); - return 2; - } - }; - if global.machine.is_some() { - eprintln!("cmux: {}", messages.machine_scope_unsupported); - return 2; - } - let socket_path = resolve_server_socket(global); - let lifecycle = match crate::server_lifecycle::ServerLifecycle::connect(socket_path) { - Ok(lifecycle) => lifecycle, - Err(error) => { - eprintln!("{error}"); - return 3; - } - }; - match action { - "status" => print_server_status(global.output, lifecycle.probe()), - "stop" => { - eprintln!("cmux: {}", messages.stopping_exits_panes); - match lifecycle.stop() { - Ok(()) => { - match global.output { - OutputMode::Json | OutputMode::JsonLines => { - println!("{}", json!({"stopped": true})); - } - OutputMode::Human => println!("{}", messages.stopped), - OutputMode::Quiet => {} - } - 0 - } - Err(error) => { - eprintln!("{error}"); - 1 - } - } - } - _ => unreachable!("server action was validated before connecting"), - } -} - -fn print_server_status(output: OutputMode, probe: &crate::server_lifecycle::ServerProbe) -> i32 { - let messages = &crate::localization::catalog().server; - let server = &probe.identity; - let client = ReleaseIdentity::current(PROTOCOL_VERSION); - let mismatches = probe.mismatches(); - let compatible = mismatches.is_empty(); - match output { - OutputMode::Json | OutputMode::JsonLines => { - let mismatch_reasons = - mismatches.iter().map(|mismatch| mismatch.code()).collect::>(); - let value = json!({ - "running": true, - "compatible": compatible, - "mismatch_reasons": mismatch_reasons, - "server": { - "version": server.release.version, - "protocol": server.release.protocol, - "shutdown_cleanup": { - "pending": server.shutdown_cleanup.pending, - "retrying": server.shutdown_cleanup.retrying, - "degraded": server.shutdown_cleanup.degraded, - }, - }, - "client": {"version": client.version, "protocol": client.protocol}, - }); - let mut stdout = io::stdout(); - match serde_json::to_writer(&mut stdout, &value) - .map_err(io::Error::other) - .and_then(|_| stdout.write_all(b"\n")) - { - Ok(()) => 0, - Err(error) => { - eprintln!("stdout error: {error}"); - 3 - } - } - } - OutputMode::Quiet => 0, - OutputMode::Human => { - println!( - "{}: v{} {} {}", - messages.server_label, - server.release.version, - messages.protocol_label, - server.release.protocol, - ); - println!( - "{}: v{} {} {}", - messages.client_label, client.version, messages.protocol_label, client.protocol, - ); - println!( - "{}: {}", - messages.status_label, - if compatible { messages.compatible } else { messages.incompatible }, - ); - if !mismatches.is_empty() { - let reasons = mismatches - .into_iter() - .map(|mismatch| mismatch.message(messages)) - .collect::>() - .join(messages.reason_separator); - println!("{}: {}", messages.reason_label, reasons); - } - if server.shutdown_cleanup.pending != 0 { - let state = if server.shutdown_cleanup.degraded { - messages.cleanup_degraded - } else { - messages.cleanup_retrying - }; - println!( - "{}: {} ({}: {})", - messages.cleanup_label, - state, - messages.cleanup_pending_label, - server.shutdown_cleanup.pending, - ); - } - 0 - } - } -} - fn resolve_server_socket(global: &GlobalArgs) -> PathBuf { resolve_server_socket_with(global, |name| std::env::var_os(name).map(PathBuf::from)) } diff --git a/cmux-tui/crates/cmux-tui/src/main.rs b/cmux-tui/crates/cmux-tui/src/main.rs index 60d43b4062cb..6ebf78ac1c56 100644 --- a/cmux-tui/crates/cmux-tui/src/main.rs +++ b/cmux-tui/crates/cmux-tui/src/main.rs @@ -2745,6 +2745,10 @@ fn run_server( #[cfg(debug_assertions)] if !args.headless && std::env::var_os("CMUX_TUI_TEST_BLOCK_INTERACTIVE_DRIVER").is_some() { + // This hook models a driver that stops after the mux has completed + // shutdown-owner publication. A lifecycle request must still be able + // to close those owners and force this blocked process to exit. + mux.mark_server_lifecycle_ready(); loop { std::thread::park(); } From a4ab577e135dcc97b430aa0ec4d764fcd44dc955 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:03:21 -0700 Subject: [PATCH 515/521] test(macOS): reject stale teardown failures --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 2660df9eeb68..13c628d5fdab 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -952,6 +952,107 @@ import CmuxTerminalCore #expect(coordinator.debugRuntimeSurfaceOwnerCount == 0) } + @Test @MainActor func recoveredCloseWorkerCancelsQueuedCapacityFailures() async throws { + let clock = ManualTerminalSurfaceRuntimeTeardownClock() + let stalledSlots = AsyncStream.makeStream() + let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( + maximumRuntimeSurfaceOwnerCount: 2, + closeTeardownTimeout: .seconds(5), + closeTeardownClock: clock, + closeTeardownStalledObserver: { slot in + stalledSlots.continuation.yield(slot) + } + ) + let pointers = (0..<2).map { _ in + UnsafeMutableRawPointer.allocate(byteCount: 8, alignment: 8) + } + let freeStarted = AsyncStream.makeStream() + let releaseFrees = (0..<2).map { _ in DispatchSemaphore(value: 0) } + defer { + for releaseFree in releaseFrees { + releaseFree.signal() + } + for pointer in pointers { + pointer.deallocate() + } + freeStarted.continuation.finish() + stalledSlots.continuation.finish() + } + let reservations = try (0..<2).map { _ in + try #require(coordinator.reserveRuntimeSurfaceOwnership()) + } + let tickets = try pointers.enumerated().map { index, pointer in + try #require( + coordinator.enqueueRuntimeTeardown( + id: UUID(), + workspaceId: UUID(), + reason: "test.cancelStaleCapacityFailure.\(index)", + surface: pointer, + callbackContext: nil, + manualIOContext: nil, + byteTeeLease: nil, + runtimeOwnershipReservation: reservations[index], + freeSurface: { _ in + freeStarted.continuation.yield(index) + releaseFrees[index].wait() + } + ) + ) + } + var freeStartedIterator = freeStarted.stream.makeAsyncIterator() + _ = try #require(await freeStartedIterator.next()) + _ = try #require(await freeStartedIterator.next()) + var registrationIterator = clock.registrations.makeAsyncIterator() + let firstWatchdog = try #require(await registrationIterator.next()) + let secondWatchdog = try #require(await registrationIterator.next()) + + let registry = FakeSurfaceRegistry() + let scheduler = RecordingRestoreSpawnScheduler() + let deferredFixtures = (0..<3).map { _ in + makeSurfaceFixture( + registry: registry, + scheduler: scheduler, + runtimeTeardown: coordinator + ) + } + for (index, fixture) in deferredFixtures.enumerated() { + fixture.surface.createSurface(for: fixture.nativeView) + scheduler.runScheduledOperation(at: index) + } + + var stalledSlotIterator = stalledSlots.stream.makeAsyncIterator() + clock.fire(firstWatchdog) + _ = try #require(await stalledSlotIterator.next()) + clock.fire(secondWatchdog) + _ = try #require(await stalledSlotIterator.next()) + + // Keep MainActor failure delivery queued while the coordinator actor + // receives the native-free completion and reopens admission. + let freeCompletion = DispatchSemaphore(value: 0) + Task.detached { + _ = await tickets[0].wait(timeout: nil) + freeCompletion.signal() + } + releaseFrees[0].signal() + freeCompletion.wait() + #expect(!coordinator.debugCloseTeardownAllStalled) + + await waitForMainActorQueueBarrier() + #expect( + deferredFixtures.allSatisfy { + $0.paneHost.runtimeSurfaceCreationFailureMessages.isEmpty + } + ) + + releaseFrees[1].signal() + #expect(await tickets[1].wait(timeout: nil)) + for fixture in deferredFixtures { + fixture.surface.beginPortalCloseLifecycle( + reason: "test.cancelStaleCapacityFailure.cleanup" + ) + } + } + @Test func repeatedStalledRecoveryUsesLatestFailureDelivery() throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 From f3c2300e6af85285374d5e28c152b8fbd9e10add Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:05:19 -0700 Subject: [PATCH 516/521] fix(macOS): cancel stale teardown failures --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 28 +++++++++++--- ...RuntimeOwnershipRecoveryFailureDrain.swift | 6 +-- ...timeOwnershipRecoveryRescanScheduler.swift | 37 +++++++++++++++---- ...nalSurfaceRuntimeTeardownCoordinator.swift | 1 + 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index ada586428264..7d2e67740ada 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -335,12 +335,9 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { let output = state.withLock { state in let capacityWasOpen = recoveryCapacityIsOpen(in: state) for delivery in completedDeliveries { - precondition( - state.pendingRecoveryFailureDeliveryIDs.remove( - delivery.deliveryID - ) != nil, - "completed stalled recovery failure must be owned" - ) + guard state.pendingRecoveryFailureDeliveryIDs.remove( + delivery.deliveryID + ) != nil else { continue } if state.recoveryEntriesByID[delivery.recoveryID]? .pendingFailureDeliveryID == delivery.deliveryID { state.recoveryEntriesByID[delivery.recoveryID]? @@ -362,6 +359,18 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } } + func ownsStalledCloseFailure( + _ delivery: TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery + ) -> Bool { + state.withLock { state in + state.pendingRecoveryFailureDeliveryIDs.contains( + delivery.deliveryID + ) + && state.recoveryEntriesByID[delivery.recoveryID]? + .pendingFailureDeliveryID == delivery.deliveryID + } + } + func clearAllStalledCloseTeardowns() { let grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant? grant = state.withLock { state in @@ -369,6 +378,13 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { return nil } state.closeTeardownAllStalled = false + for recoveryID in Array(state.recoveryEntriesByID.keys) { + guard let deliveryID = state.recoveryEntriesByID[recoveryID]? + .pendingFailureDeliveryID else { continue } + state.recoveryEntriesByID[recoveryID]? + .pendingFailureDeliveryID = nil + state.pendingRecoveryFailureDeliveryIDs.remove(deliveryID) + } let grant = takeNextRecoveryGrant(from: &state) _ = takeRecoveryRescanRequest(from: &state) return grant diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift index bc135b121ff9..8a394e8cb42c 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift @@ -11,9 +11,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: private static let maximumBatchCount = 32 private struct State { - var failures: [ - TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery - ] = [] + var failures: [TerminalSurfaceRuntimeOwnershipRecoveryFailureDelivery] = [] var nextFailureIndex = 0 var task: Task? } @@ -85,7 +83,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: state.nextFailureIndex = endIndex return batch } - for failure in batch { + for failure in batch where admission.ownsStalledCloseFailure(failure) { failure() } admission.completeStalledCloseRecoveryFailures(batch) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift index 77dde854d549..5476dd941d40 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler.swift @@ -43,6 +43,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var tailID: UUID? var failureThroughSequence: UInt64? var failureCursorID: UUID? + var failureCancellationGeneration: UInt64 = 0 var rescanRequested = false var rescanTask: Task? } @@ -174,6 +175,22 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: startGate?.start() } + /// Invalidates failure delivery that was scheduled for a fully stalled + /// close pool. A recovered close slot owns this transition before it opens + /// admission again, so a queued MainActor batch cannot publish a stale + /// capacity error after native ownership capacity has returned. + internal func cancelPendingOverflowFailures() { + state.withLockUnchecked { state in + state.failureThroughSequence = nil + state.failureCursorID = nil + precondition(state.failureCancellationGeneration < UInt64.max) + state.failureCancellationGeneration += 1 + for entry in state.entriesByID.values { + entry.failureReported = false + } + } + } + deinit { state.withLockUnchecked { state in state.rescanTask?.cancel() @@ -203,13 +220,19 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: var processedCount = 0 if shouldRun { while processedCount < Self.maximumBatchCount { - if let failedEntry = state.withLockUnchecked({ state in + if let failure = state.withLockUnchecked({ state in takeNextFailureEntry(from: &state) }) { - failedEntry.surface? - .failRuntimeSurfaceCreationForTeardownCapacity( - preservingRecoveryOwner: true - ) + let failureIsCurrent = state.withLockUnchecked { state in + state.failureThroughSequence != nil + && state.failureCancellationGeneration == failure.generation + } + if failureIsCurrent { + failure.entry.surface? + .failRuntimeSurfaceCreationForTeardownCapacity( + preservingRecoveryOwner: true + ) + } processedCount += 1 continue } @@ -319,7 +342,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: private func takeNextFailureEntry( from state: inout State - ) -> OverflowEntry? { + ) -> (entry: OverflowEntry, generation: UInt64)? { guard let cutoff = state.failureThroughSequence else { return nil } while let currentID = state.failureCursorID, let entry = state.entriesByID[currentID] @@ -328,7 +351,7 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryRescanScheduler: state.failureCursorID = entry.nextID guard !entry.failureReported else { continue } entry.failureReported = true - return entry + return (entry, state.failureCancellationGeneration) } state.failureThroughSequence = nil state.failureCursorID = nil diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 04ff47c28724..1214941106cc 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -606,6 +606,7 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { if stalledCloseExecutionSlots.count < Self.maximumConcurrentCloseTeardownCount { runtimeOwnershipAdmission.clearAllStalledCloseTeardowns() + recoveryRescanScheduler.cancelPendingOverflowFailures() } runtimeOwnershipAdmission.setCloseTeardownDegraded( activeCloseTeardownsBySlot.count From 6c76dc1d912ef9a0357e92967507f13d7e8baecb Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:05:32 -0700 Subject: [PATCH 517/521] fix(tui): connect legacy lifecycle ownership --- cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs | 72 +++-- .../crates/cmux-tui/src/server_lifecycle.rs | 267 ++---------------- 2 files changed, 69 insertions(+), 270 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs index 6ed6dc6c2839..ee35f4e5ea59 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs @@ -2,6 +2,7 @@ use std::io::{BufRead, BufReader, Read, Write}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; +use cmux_tui_core::server::PROTOCOL_VERSION; use serde_json::{Value, json}; use super::{GlobalArgs, OutputMode}; @@ -119,14 +120,6 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { 3, ); }; - let Some(generation) = identity["generation"].as_str().filter(|value| !value.is_empty()) else { - return local_error( - "server.invalid_identity", - crate::localization::catalog().local_server.invalid_identity, - global.output, - 3, - ); - }; if expected_session.as_deref().is_some_and(|expected| actual_session != expected) { return local_error( "server.different_session", @@ -135,6 +128,38 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { 1, ); } + let generation = identity["generation"].as_str().filter(|value| !value.is_empty()); + if generation.is_none() + && identity["protocol"].as_u64().unwrap_or_default() < u64::from(PROTOCOL_VERSION) + && let ServerAction::Stop { force } = &plan.action + { + if *force { + return local_error( + "server.force_unsupported", + crate::localization::catalog().local_server.force_unsupported, + global.output, + 1, + ); + } + let result = + crate::server_lifecycle::ServerLifecycle::from_connected(socket, &identity, connection) + .and_then(crate::server_lifecycle::ServerLifecycle::stop_legacy); + return match result { + Ok(()) => print_stop_success(global.output, actual_session, pid, None), + Err(error) => { + let message = error.to_string(); + local_error("server.stop_failed", &message, global.output, 1) + } + }; + } + let Some(generation) = generation else { + return local_error( + "server.invalid_identity", + crate::localization::catalog().local_server.invalid_identity, + global.output, + 3, + ); + }; match plan.action { ServerAction::Status => print_success( @@ -240,21 +265,30 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { if let Err(message) = wait_for_close(&mut connection, deadline) { return local_error("server.stop_incomplete", &message, global.output, 3); } - print_success( - json!({ - "status":"stopped", - "accepted":true, - "session":actual_session, - "pid":pid, - "generation":generation, - "message":crate::localization::catalog().local_server.stopped, - }), - global.output, - ) + print_stop_success(global.output, actual_session, pid, Some(generation)) } } } +fn print_stop_success( + output: OutputMode, + session: &str, + pid: u32, + generation: Option<&str>, +) -> i32 { + print_success( + json!({ + "status":"stopped", + "accepted":true, + "session":session, + "pid":pid, + "generation":generation, + "message":crate::localization::catalog().local_server.stopped, + }), + output, + ) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExchangeError { Transport, diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 8d1d0bc0a2e2..e34a39560423 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -17,7 +17,7 @@ use cmux_tui_core::platform::transport; use cmux_tui_core::release::{LAUNCHER_COMMAND_ENV, ReleaseIdentity}; use cmux_tui_core::server::{ LOCAL_SOCKET_CONNECT_TIMEOUT, PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, - SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, + SERVER_SHUTDOWN_TIMEOUT, }; use serde_json::{Value, json}; @@ -32,20 +32,17 @@ use legacy_process::{ }; const PROBE_REQUEST_ID: u64 = 0; -const SHUTDOWN_REQUEST_ID: u64 = 1; #[cfg(unix)] const LEGACY_LIST_REQUEST_ID: u64 = 2; #[cfg(unix)] const LEGACY_STABLE_EMPTY_SCANS: usize = 2; #[cfg(unix)] const LEGACY_MAX_SCAN_ROUNDS: usize = 64; -const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); const MAX_LIFECYCLE_RESPONSE_BYTES: usize = 16 * 1024 * 1024; const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); -const SHUTDOWN_RESPONSE_TIMEOUT: Duration = - SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); #[cfg(unix)] -const LEGACY_SHUTDOWN_TIMEOUT: Duration = SHUTDOWN_RESPONSE_TIMEOUT; +const LEGACY_SHUTDOWN_TIMEOUT: Duration = + SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); #[cfg(unix)] const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); #[cfg(unix)] @@ -560,15 +557,6 @@ pub(crate) enum ReleaseMismatch { } impl ReleaseMismatch { - pub(crate) fn code(self) -> &'static str { - match self { - Self::DistributionVersion => "distribution-version", - Self::SourceBuild => "source-build", - Self::TerminalEngine => "terminal-engine", - Self::Protocol => "protocol", - } - } - pub(crate) fn message(self, messages: &crate::localization::ServerMessages) -> &'static str { match self { Self::DistributionVersion => messages.reason_version, @@ -601,14 +589,6 @@ pub(crate) struct ServerIdentity { pub release: ReleaseIdentity, pub pid: u32, capabilities: HashSet, - pub(crate) shutdown_cleanup: ShutdownCleanupStatus, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct ShutdownCleanupStatus { - pub(crate) pending: u64, - pub(crate) retrying: bool, - pub(crate) degraded: bool, } impl ServerIdentity { @@ -629,26 +609,7 @@ impl ServerIdentity { .filter_map(Value::as_str) .map(str::to_string) .collect(); - let shutdown_cleanup = match data.get("shutdown_cleanup") { - None => ShutdownCleanupStatus::default(), - Some(cleanup) => ShutdownCleanupStatus { - pending: cleanup.get("pending").and_then(Value::as_u64).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - retrying: cleanup.get("retrying").and_then(Value::as_bool).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - degraded: cleanup.get("degraded").and_then(Value::as_bool).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - }, - }; - Ok(Self { - release: ReleaseIdentity::from_protocol_data(data), - pid, - capabilities, - shutdown_cleanup, - }) + Ok(Self { release: ReleaseIdentity::from_protocol_data(data), pid, capabilities }) } pub(crate) fn supports(&self, capability: &str) -> bool { @@ -706,10 +667,6 @@ impl ServerProbe { } } - pub(crate) fn connect(path: &Path) -> anyhow::Result<(Self, TransportReader, Option)> { - Self::connect_until(path, Instant::now() + RESPONSE_TIMEOUT) - } - fn connect_until( path: &Path, deadline: Instant, @@ -761,41 +718,27 @@ pub(crate) struct ServerLifecycle { } impl ServerLifecycle { - pub(crate) fn connect(path: PathBuf) -> anyhow::Result { - let (probe, reader, peer_process_id) = ServerProbe::connect(&path)?; - Ok(Self { path, probe, reader, peer_process_id }) - } - #[cfg(unix)] fn connect_until(path: PathBuf, deadline: Instant) -> anyhow::Result { let (probe, reader, peer_process_id) = ServerProbe::connect_until(&path, deadline)?; Ok(Self { path, probe, reader, peer_process_id }) } - pub(crate) fn probe(&self) -> &ServerProbe { - &self.probe + pub(crate) fn from_connected( + path: PathBuf, + identity: &Value, + reader: TransportReader, + ) -> anyhow::Result { + let peer_process_id = reader.get_ref().peer_process_id().ok().flatten(); + let probe = ServerProbe { identity: ServerIdentity::from_protocol_data(identity)? }; + Ok(Self { path, probe, reader, peer_process_id }) } - pub(crate) fn stop(mut self) -> anyhow::Result<()> { - if !self.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { - return self.stop_legacy_server(); - } - - self.reader - .get_mut() - .set_read_timeout(Some(SHUTDOWN_RESPONSE_TIMEOUT)) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - write_json_line( - self.reader.get_mut(), - &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), - ) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - - let response = read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID)?; - if response.get("ok").and_then(Value::as_bool) == Some(true) { - return wait_for_disconnect(&mut self.reader, &self.path); + pub(crate) fn stop_legacy(self) -> anyhow::Result<()> { + if self.probe.identity.release.protocol >= PROTOCOL_VERSION { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - anyhow::bail!(localized_shutdown_error(response.get("error").and_then(Value::as_str))) + self.stop_legacy_server() } #[cfg(unix)] @@ -1065,14 +1008,6 @@ impl ServerLifecycle { } } -fn localized_shutdown_error(error: Option<&str>) -> &'static str { - let messages = &crate::localization::catalog().server; - match error { - Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR) => messages.shutdown_cleanup_incomplete, - Some(_) | None => messages.shutdown_failed, - } -} - #[cfg(unix)] fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { let request_id = *next_request_id; @@ -1510,24 +1445,6 @@ fn read_response_until( read_matching_response_until(reader, request_id, false, deadline) } -fn read_shutdown_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { - read_matching_response_with_timeout(reader, request_id, true, SHUTDOWN_RESPONSE_TIMEOUT) -} - -fn read_matching_response_with_timeout( - reader: &mut TransportReader, - request_id: u64, - accept_unidentified_error: bool, - timeout: Duration, -) -> anyhow::Result { - read_matching_response_until( - reader, - request_id, - accept_unidentified_error, - Instant::now() + timeout, - ) -} - fn read_matching_response_until( reader: &mut TransportReader, request_id: u64, @@ -1676,10 +1593,6 @@ fn legacy_surface_ids(data: &Value) -> Vec { legacy_surfaces(data).unwrap().into_iter().map(|surface| surface.id).collect() } -fn wait_for_disconnect(reader: &mut TransportReader, path: &Path) -> anyhow::Result<()> { - wait_for_disconnect_until(reader, path, Instant::now() + RESPONSE_TIMEOUT) -} - fn wait_for_disconnect_until( reader: &mut TransportReader, path: &Path, @@ -1898,7 +1811,6 @@ mod tests { }, pid: 42, capabilities: HashSet::new(), - shutdown_cleanup: ShutdownCleanupStatus::default(), }; let message = incompatible_server_message(&identity, Path::new("/tmp/test socket")); @@ -1921,44 +1833,6 @@ mod tests { assert!(message.contains("Stopping exits pane processes.")); } - #[test] - fn legacy_identity_may_omit_shutdown_cleanup_as_a_whole() { - let identity = ServerIdentity::from_protocol_data(&json!({ - "app": "cmux-tui", - "pid": 42, - "protocol": PROTOCOL_VERSION, - })) - .unwrap(); - - assert_eq!(identity.shutdown_cleanup, ShutdownCleanupStatus::default()); - } - - #[test] - fn present_shutdown_cleanup_requires_every_typed_member() { - let malformed = [ - json!(null), - json!({}), - json!({"pending": 1, "retrying": true}), - json!({"pending": "1", "retrying": true, "degraded": false}), - json!({"pending": 1, "retrying": "true", "degraded": false}), - json!({"pending": 1, "retrying": true, "degraded": 0}), - ]; - - for shutdown_cleanup in malformed { - let identity = json!({ - "app": "cmux-tui", - "pid": 42, - "protocol": PROTOCOL_VERSION, - "shutdown_cleanup": shutdown_cleanup, - }); - assert!( - ServerIdentity::from_protocol_data(&identity).is_err(), - "accepted malformed shutdown cleanup: {}", - identity["shutdown_cleanup"] - ); - } - } - #[cfg(unix)] #[test] fn shell_quote_preserves_non_utf8_socket_paths() { @@ -2017,115 +1891,6 @@ mod tests { ); } - #[cfg(unix)] - #[test] - fn modern_shutdown_requires_a_success_response_before_disconnect() { - let path = PathBuf::from("/tmp").join(format!( - "cmux-tui-shutdown-disconnect-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() - )); - let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let identify: Value = serde_json::from_str(&line).unwrap(); - assert_eq!(identify["cmd"], "identify"); - write_json_line( - &mut stream, - &json!({ - "id": identify["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "version": "test", - "protocol": PROTOCOL_VERSION, - "pid": std::process::id(), - "capabilities": [SERVER_SHUTDOWN_CAPABILITY], - }, - }), - ) - .unwrap(); - line.clear(); - reader.read_line(&mut line).unwrap(); - let shutdown: Value = serde_json::from_str(&line).unwrap(); - assert_eq!(shutdown["cmd"], "shutdown"); - }); - - let result = ServerLifecycle::connect(path.clone()).unwrap().stop(); - - server.join().unwrap(); - std::fs::remove_file(path).unwrap(); - let error = result.expect_err("disconnect without a success response was accepted"); - assert!( - error.to_string().contains(crate::localization::catalog().server.response_closed) - || error - .to_string() - .contains(crate::localization::catalog().server.transport_failed) - ); - } - - #[cfg(unix)] - #[test] - fn modern_shutdown_does_not_expose_unlocalized_server_details() { - let path = PathBuf::from("/tmp").join(format!( - "cmux-tui-shutdown-localization-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() - )); - let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let identify: Value = serde_json::from_str(&line).unwrap(); - write_json_line( - &mut stream, - &json!({ - "id": identify["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "version": "test", - "protocol": PROTOCOL_VERSION, - "pid": std::process::id(), - "capabilities": [SERVER_SHUTDOWN_CAPABILITY], - }, - }), - ) - .unwrap(); - line.clear(); - reader.read_line(&mut line).unwrap(); - let shutdown: Value = serde_json::from_str(&line).unwrap(); - write_json_line( - &mut stream, - &json!({ - "id": shutdown["id"], - "ok": false, - "error": "raw internal shutdown detail", - }), - ) - .unwrap(); - }); - - let error = ServerLifecycle::connect(path.clone()).unwrap().stop().unwrap_err(); - - server.join().unwrap(); - std::fs::remove_file(path).unwrap(); - assert_eq!(error.to_string(), crate::localization::catalog().server.shutdown_failed); - } - - #[test] - fn modern_shutdown_localizes_stable_cleanup_error_code() { - assert_eq!( - localized_shutdown_error(Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR)), - crate::localization::catalog().server.shutdown_cleanup_incomplete - ); - } - #[cfg(unix)] #[test] fn legacy_helper_timeout_retains_reap_ownership() { From f3be93e8be0522b169604941d6ff3a7384ec6ea7 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:05:32 -0700 Subject: [PATCH 518/521] fix(tui): connect legacy lifecycle ownership --- ...inalSurfaceRuntimeOwnershipAdmission.swift | 19 +- ...nalSurfaceRuntimeTeardownCoordinator.swift | 6 +- cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs | 72 +++-- .../crates/cmux-tui/src/server_lifecycle.rs | 267 ++---------------- 4 files changed, 86 insertions(+), 278 deletions(-) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift index 7d2e67740ada..49ce6b6819a3 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipAdmission.swift @@ -372,10 +372,14 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } func clearAllStalledCloseTeardowns() { - let grant: TerminalSurfaceRuntimeOwnershipRecoveryGrant? - grant = state.withLock { state in + _ = clearAllStalledCloseTeardownsIfNeeded() + } + + func clearAllStalledCloseTeardownsIfNeeded() -> Bool { + let output: (TerminalSurfaceRuntimeOwnershipRecoveryGrant?, Bool) + output = state.withLock { state in guard state.closeTeardownAllStalled else { - return nil + return (nil, false) } state.closeTeardownAllStalled = false for recoveryID in Array(state.recoveryEntriesByID.keys) { @@ -387,10 +391,13 @@ final class TerminalSurfaceRuntimeOwnershipAdmission: @unchecked Sendable { } let grant = takeNextRecoveryGrant(from: &state) _ = takeRecoveryRescanRequest(from: &state) - return grant + return (grant, true) } - schedule(grant) - recoveryRescanScheduler.requestRescan() + schedule(output.0) + if output.1 { + recoveryRescanScheduler.requestRescan() + } + return output.1 } func cancelRecovery(_ recoveryID: UUID) { diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift index 1214941106cc..6514d0b814a4 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeTeardownCoordinator.swift @@ -605,8 +605,10 @@ public actor TerminalSurfaceRuntimeTeardownCoordinator { private func updateCloseTeardownAdmission() { if stalledCloseExecutionSlots.count < Self.maximumConcurrentCloseTeardownCount { - runtimeOwnershipAdmission.clearAllStalledCloseTeardowns() - recoveryRescanScheduler.cancelPendingOverflowFailures() + if runtimeOwnershipAdmission + .clearAllStalledCloseTeardownsIfNeeded() { + recoveryRescanScheduler.cancelPendingOverflowFailures() + } } runtimeOwnershipAdmission.setCloseTeardownDegraded( activeCloseTeardownsBySlot.count diff --git a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs index 6ed6dc6c2839..ee35f4e5ea59 100644 --- a/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/cli/lifecycle.rs @@ -2,6 +2,7 @@ use std::io::{BufRead, BufReader, Read, Write}; use std::time::{Duration, Instant}; use cmux_tui_core::platform::transport; +use cmux_tui_core::server::PROTOCOL_VERSION; use serde_json::{Value, json}; use super::{GlobalArgs, OutputMode}; @@ -119,14 +120,6 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { 3, ); }; - let Some(generation) = identity["generation"].as_str().filter(|value| !value.is_empty()) else { - return local_error( - "server.invalid_identity", - crate::localization::catalog().local_server.invalid_identity, - global.output, - 3, - ); - }; if expected_session.as_deref().is_some_and(|expected| actual_session != expected) { return local_error( "server.different_session", @@ -135,6 +128,38 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { 1, ); } + let generation = identity["generation"].as_str().filter(|value| !value.is_empty()); + if generation.is_none() + && identity["protocol"].as_u64().unwrap_or_default() < u64::from(PROTOCOL_VERSION) + && let ServerAction::Stop { force } = &plan.action + { + if *force { + return local_error( + "server.force_unsupported", + crate::localization::catalog().local_server.force_unsupported, + global.output, + 1, + ); + } + let result = + crate::server_lifecycle::ServerLifecycle::from_connected(socket, &identity, connection) + .and_then(crate::server_lifecycle::ServerLifecycle::stop_legacy); + return match result { + Ok(()) => print_stop_success(global.output, actual_session, pid, None), + Err(error) => { + let message = error.to_string(); + local_error("server.stop_failed", &message, global.output, 1) + } + }; + } + let Some(generation) = generation else { + return local_error( + "server.invalid_identity", + crate::localization::catalog().local_server.invalid_identity, + global.output, + 3, + ); + }; match plan.action { ServerAction::Status => print_success( @@ -240,21 +265,30 @@ pub(super) fn run(mut global: GlobalArgs, plan: ServerPlan) -> i32 { if let Err(message) = wait_for_close(&mut connection, deadline) { return local_error("server.stop_incomplete", &message, global.output, 3); } - print_success( - json!({ - "status":"stopped", - "accepted":true, - "session":actual_session, - "pid":pid, - "generation":generation, - "message":crate::localization::catalog().local_server.stopped, - }), - global.output, - ) + print_stop_success(global.output, actual_session, pid, Some(generation)) } } } +fn print_stop_success( + output: OutputMode, + session: &str, + pid: u32, + generation: Option<&str>, +) -> i32 { + print_success( + json!({ + "status":"stopped", + "accepted":true, + "session":session, + "pid":pid, + "generation":generation, + "message":crate::localization::catalog().local_server.stopped, + }), + output, + ) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExchangeError { Transport, diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index 8d1d0bc0a2e2..e34a39560423 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -17,7 +17,7 @@ use cmux_tui_core::platform::transport; use cmux_tui_core::release::{LAUNCHER_COMMAND_ENV, ReleaseIdentity}; use cmux_tui_core::server::{ LOCAL_SOCKET_CONNECT_TIMEOUT, PROTOCOL_VERSION, SERVER_SHUTDOWN_CAPABILITY, - SERVER_SHUTDOWN_INCOMPLETE_ERROR, SERVER_SHUTDOWN_TIMEOUT, + SERVER_SHUTDOWN_TIMEOUT, }; use serde_json::{Value, json}; @@ -32,20 +32,17 @@ use legacy_process::{ }; const PROBE_REQUEST_ID: u64 = 0; -const SHUTDOWN_REQUEST_ID: u64 = 1; #[cfg(unix)] const LEGACY_LIST_REQUEST_ID: u64 = 2; #[cfg(unix)] const LEGACY_STABLE_EMPTY_SCANS: usize = 2; #[cfg(unix)] const LEGACY_MAX_SCAN_ROUNDS: usize = 64; -const RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); const MAX_LIFECYCLE_RESPONSE_BYTES: usize = 16 * 1024 * 1024; const SHUTDOWN_TRANSPORT_MARGIN: Duration = Duration::from_secs(5); -const SHUTDOWN_RESPONSE_TIMEOUT: Duration = - SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); #[cfg(unix)] -const LEGACY_SHUTDOWN_TIMEOUT: Duration = SHUTDOWN_RESPONSE_TIMEOUT; +const LEGACY_SHUTDOWN_TIMEOUT: Duration = + SERVER_SHUTDOWN_TIMEOUT.saturating_add(SHUTDOWN_TRANSPORT_MARGIN); #[cfg(unix)] const LEGACY_HELPER_WAIT_MARGIN: Duration = Duration::from_millis(250); #[cfg(unix)] @@ -560,15 +557,6 @@ pub(crate) enum ReleaseMismatch { } impl ReleaseMismatch { - pub(crate) fn code(self) -> &'static str { - match self { - Self::DistributionVersion => "distribution-version", - Self::SourceBuild => "source-build", - Self::TerminalEngine => "terminal-engine", - Self::Protocol => "protocol", - } - } - pub(crate) fn message(self, messages: &crate::localization::ServerMessages) -> &'static str { match self { Self::DistributionVersion => messages.reason_version, @@ -601,14 +589,6 @@ pub(crate) struct ServerIdentity { pub release: ReleaseIdentity, pub pid: u32, capabilities: HashSet, - pub(crate) shutdown_cleanup: ShutdownCleanupStatus, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(crate) struct ShutdownCleanupStatus { - pub(crate) pending: u64, - pub(crate) retrying: bool, - pub(crate) degraded: bool, } impl ServerIdentity { @@ -629,26 +609,7 @@ impl ServerIdentity { .filter_map(Value::as_str) .map(str::to_string) .collect(); - let shutdown_cleanup = match data.get("shutdown_cleanup") { - None => ShutdownCleanupStatus::default(), - Some(cleanup) => ShutdownCleanupStatus { - pending: cleanup.get("pending").and_then(Value::as_u64).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - retrying: cleanup.get("retrying").and_then(Value::as_bool).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - degraded: cleanup.get("degraded").and_then(Value::as_bool).ok_or_else(|| { - anyhow::anyhow!(crate::localization::catalog().server.endpoint_invalid) - })?, - }, - }; - Ok(Self { - release: ReleaseIdentity::from_protocol_data(data), - pid, - capabilities, - shutdown_cleanup, - }) + Ok(Self { release: ReleaseIdentity::from_protocol_data(data), pid, capabilities }) } pub(crate) fn supports(&self, capability: &str) -> bool { @@ -706,10 +667,6 @@ impl ServerProbe { } } - pub(crate) fn connect(path: &Path) -> anyhow::Result<(Self, TransportReader, Option)> { - Self::connect_until(path, Instant::now() + RESPONSE_TIMEOUT) - } - fn connect_until( path: &Path, deadline: Instant, @@ -761,41 +718,27 @@ pub(crate) struct ServerLifecycle { } impl ServerLifecycle { - pub(crate) fn connect(path: PathBuf) -> anyhow::Result { - let (probe, reader, peer_process_id) = ServerProbe::connect(&path)?; - Ok(Self { path, probe, reader, peer_process_id }) - } - #[cfg(unix)] fn connect_until(path: PathBuf, deadline: Instant) -> anyhow::Result { let (probe, reader, peer_process_id) = ServerProbe::connect_until(&path, deadline)?; Ok(Self { path, probe, reader, peer_process_id }) } - pub(crate) fn probe(&self) -> &ServerProbe { - &self.probe + pub(crate) fn from_connected( + path: PathBuf, + identity: &Value, + reader: TransportReader, + ) -> anyhow::Result { + let peer_process_id = reader.get_ref().peer_process_id().ok().flatten(); + let probe = ServerProbe { identity: ServerIdentity::from_protocol_data(identity)? }; + Ok(Self { path, probe, reader, peer_process_id }) } - pub(crate) fn stop(mut self) -> anyhow::Result<()> { - if !self.probe.identity.supports(SERVER_SHUTDOWN_CAPABILITY) { - return self.stop_legacy_server(); - } - - self.reader - .get_mut() - .set_read_timeout(Some(SHUTDOWN_RESPONSE_TIMEOUT)) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - write_json_line( - self.reader.get_mut(), - &json!({"id": SHUTDOWN_REQUEST_ID, "cmd": "shutdown"}), - ) - .map_err(|_| anyhow::anyhow!(crate::localization::catalog().server.transport_failed))?; - - let response = read_shutdown_response(&mut self.reader, SHUTDOWN_REQUEST_ID)?; - if response.get("ok").and_then(Value::as_bool) == Some(true) { - return wait_for_disconnect(&mut self.reader, &self.path); + pub(crate) fn stop_legacy(self) -> anyhow::Result<()> { + if self.probe.identity.release.protocol >= PROTOCOL_VERSION { + anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); } - anyhow::bail!(localized_shutdown_error(response.get("error").and_then(Value::as_str))) + self.stop_legacy_server() } #[cfg(unix)] @@ -1065,14 +1008,6 @@ impl ServerLifecycle { } } -fn localized_shutdown_error(error: Option<&str>) -> &'static str { - let messages = &crate::localization::catalog().server; - match error { - Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR) => messages.shutdown_cleanup_incomplete, - Some(_) | None => messages.shutdown_failed, - } -} - #[cfg(unix)] fn take_legacy_request_id(next_request_id: &mut u64) -> anyhow::Result { let request_id = *next_request_id; @@ -1510,24 +1445,6 @@ fn read_response_until( read_matching_response_until(reader, request_id, false, deadline) } -fn read_shutdown_response(reader: &mut TransportReader, request_id: u64) -> anyhow::Result { - read_matching_response_with_timeout(reader, request_id, true, SHUTDOWN_RESPONSE_TIMEOUT) -} - -fn read_matching_response_with_timeout( - reader: &mut TransportReader, - request_id: u64, - accept_unidentified_error: bool, - timeout: Duration, -) -> anyhow::Result { - read_matching_response_until( - reader, - request_id, - accept_unidentified_error, - Instant::now() + timeout, - ) -} - fn read_matching_response_until( reader: &mut TransportReader, request_id: u64, @@ -1676,10 +1593,6 @@ fn legacy_surface_ids(data: &Value) -> Vec { legacy_surfaces(data).unwrap().into_iter().map(|surface| surface.id).collect() } -fn wait_for_disconnect(reader: &mut TransportReader, path: &Path) -> anyhow::Result<()> { - wait_for_disconnect_until(reader, path, Instant::now() + RESPONSE_TIMEOUT) -} - fn wait_for_disconnect_until( reader: &mut TransportReader, path: &Path, @@ -1898,7 +1811,6 @@ mod tests { }, pid: 42, capabilities: HashSet::new(), - shutdown_cleanup: ShutdownCleanupStatus::default(), }; let message = incompatible_server_message(&identity, Path::new("/tmp/test socket")); @@ -1921,44 +1833,6 @@ mod tests { assert!(message.contains("Stopping exits pane processes.")); } - #[test] - fn legacy_identity_may_omit_shutdown_cleanup_as_a_whole() { - let identity = ServerIdentity::from_protocol_data(&json!({ - "app": "cmux-tui", - "pid": 42, - "protocol": PROTOCOL_VERSION, - })) - .unwrap(); - - assert_eq!(identity.shutdown_cleanup, ShutdownCleanupStatus::default()); - } - - #[test] - fn present_shutdown_cleanup_requires_every_typed_member() { - let malformed = [ - json!(null), - json!({}), - json!({"pending": 1, "retrying": true}), - json!({"pending": "1", "retrying": true, "degraded": false}), - json!({"pending": 1, "retrying": "true", "degraded": false}), - json!({"pending": 1, "retrying": true, "degraded": 0}), - ]; - - for shutdown_cleanup in malformed { - let identity = json!({ - "app": "cmux-tui", - "pid": 42, - "protocol": PROTOCOL_VERSION, - "shutdown_cleanup": shutdown_cleanup, - }); - assert!( - ServerIdentity::from_protocol_data(&identity).is_err(), - "accepted malformed shutdown cleanup: {}", - identity["shutdown_cleanup"] - ); - } - } - #[cfg(unix)] #[test] fn shell_quote_preserves_non_utf8_socket_paths() { @@ -2017,115 +1891,6 @@ mod tests { ); } - #[cfg(unix)] - #[test] - fn modern_shutdown_requires_a_success_response_before_disconnect() { - let path = PathBuf::from("/tmp").join(format!( - "cmux-tui-shutdown-disconnect-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() - )); - let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let identify: Value = serde_json::from_str(&line).unwrap(); - assert_eq!(identify["cmd"], "identify"); - write_json_line( - &mut stream, - &json!({ - "id": identify["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "version": "test", - "protocol": PROTOCOL_VERSION, - "pid": std::process::id(), - "capabilities": [SERVER_SHUTDOWN_CAPABILITY], - }, - }), - ) - .unwrap(); - line.clear(); - reader.read_line(&mut line).unwrap(); - let shutdown: Value = serde_json::from_str(&line).unwrap(); - assert_eq!(shutdown["cmd"], "shutdown"); - }); - - let result = ServerLifecycle::connect(path.clone()).unwrap().stop(); - - server.join().unwrap(); - std::fs::remove_file(path).unwrap(); - let error = result.expect_err("disconnect without a success response was accepted"); - assert!( - error.to_string().contains(crate::localization::catalog().server.response_closed) - || error - .to_string() - .contains(crate::localization::catalog().server.transport_failed) - ); - } - - #[cfg(unix)] - #[test] - fn modern_shutdown_does_not_expose_unlocalized_server_details() { - let path = PathBuf::from("/tmp").join(format!( - "cmux-tui-shutdown-localization-{}-{}.sock", - std::process::id(), - std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos() - )); - let listener = std::os::unix::net::UnixListener::bind(&path).unwrap(); - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut reader = BufReader::new(stream.try_clone().unwrap()); - let mut line = String::new(); - reader.read_line(&mut line).unwrap(); - let identify: Value = serde_json::from_str(&line).unwrap(); - write_json_line( - &mut stream, - &json!({ - "id": identify["id"], - "ok": true, - "data": { - "app": "cmux-tui", - "version": "test", - "protocol": PROTOCOL_VERSION, - "pid": std::process::id(), - "capabilities": [SERVER_SHUTDOWN_CAPABILITY], - }, - }), - ) - .unwrap(); - line.clear(); - reader.read_line(&mut line).unwrap(); - let shutdown: Value = serde_json::from_str(&line).unwrap(); - write_json_line( - &mut stream, - &json!({ - "id": shutdown["id"], - "ok": false, - "error": "raw internal shutdown detail", - }), - ) - .unwrap(); - }); - - let error = ServerLifecycle::connect(path.clone()).unwrap().stop().unwrap_err(); - - server.join().unwrap(); - std::fs::remove_file(path).unwrap(); - assert_eq!(error.to_string(), crate::localization::catalog().server.shutdown_failed); - } - - #[test] - fn modern_shutdown_localizes_stable_cleanup_error_code() { - assert_eq!( - localized_shutdown_error(Some(SERVER_SHUTDOWN_INCOMPLETE_ERROR)), - crate::localization::catalog().server.shutdown_cleanup_incomplete - ); - } - #[cfg(unix)] #[test] fn legacy_helper_timeout_retains_reap_ownership() { From 851a0e4947a6cba3327c9ebb676fd4cc0a2f91a6 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:26:00 -0700 Subject: [PATCH 519/521] fix(tui): compile lifecycle compatibility tests --- .../crates/cmux-tui/src/server_lifecycle.rs | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs index e34a39560423..3347e1087670 100644 --- a/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs +++ b/cmux-tui/crates/cmux-tui/src/server_lifecycle.rs @@ -736,7 +736,7 @@ impl ServerLifecycle { pub(crate) fn stop_legacy(self) -> anyhow::Result<()> { if self.probe.identity.release.protocol >= PROTOCOL_VERSION { - anyhow::bail!(crate::localization::catalog().server.legacy_peer_mismatch); + anyhow::bail!(crate::localization::catalog().server.shutdown_unsupported); } self.stop_legacy_server() } @@ -2131,9 +2131,13 @@ mod tests { let mut reader = BufReader::new(Box::new(client) as Box); let started = Instant::now(); - let error = - read_matching_response_with_timeout(&mut reader, 7, false, Duration::from_millis(50)) - .unwrap_err(); + let error = read_matching_response_until( + &mut reader, + 7, + false, + Instant::now() + Duration::from_millis(50), + ) + .unwrap_err(); assert!( started.elapsed() < Duration::from_millis(250), @@ -2151,9 +2155,13 @@ mod tests { let mut reader = BufReader::with_capacity(CHUNK_BYTES, Box::new(stream) as Box); - let error = - read_matching_response_with_timeout(&mut reader, 7, false, Duration::from_secs(1)) - .unwrap_err(); + let error = read_matching_response_until( + &mut reader, + 7, + false, + Instant::now() + Duration::from_secs(1), + ) + .unwrap_err(); assert_eq!(error.to_string(), crate::localization::catalog().server.response_invalid); assert!( From f0c86bb0252163cdd66773ba17bc830587d2e6a5 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:37:49 -0700 Subject: [PATCH 520/521] test(macOS): bound repeated failure delivery --- ...nalSurfaceRestoreSpawnSchedulerTests.swift | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift index 13c628d5fdab..cc124abc2502 100644 --- a/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift +++ b/Packages/macOS/CmuxTerminal/Tests/CmuxTerminalTests/TerminalSurfaceRestoreSpawnSchedulerTests.swift @@ -1096,6 +1096,53 @@ import CmuxTerminalCore } } + @Test func repeatedStallReplacesQueuedFailureDeliveriesWithinCapacity() async throws { + let admission = TerminalSurfaceRuntimeOwnershipAdmission( + maximumOwnerCount: 2 + ) + let failureDrain = TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain( + maximumFailureCount: 2, + admission: admission + ) + let owners = try (0..<2).map { _ in + try #require(admission.reserve()) + } + let recoveryIDs = (0..<2).map { _ in UUID() } + var failureCount = 0 + defer { + for recoveryID in recoveryIDs { + admission.cancelRecovery(recoveryID) + } + admission.clearAllStalledCloseTeardowns() + for owner in owners { + admission.release(owner) + } + } + + for recoveryID in recoveryIDs { + #expect( + admission.reserve( + recoveryID: recoveryID, + onRecovery: { reservation in + admission.release(reservation) + }, + onFailure: { failureCount += 1 } + ) == .deferred + ) + } + failureDrain.enqueue( + admission.failRecoveriesForAllStalledCloseTeardowns() + ) + + admission.clearAllStalledCloseTeardowns() + failureDrain.enqueue( + admission.failRecoveriesForAllStalledCloseTeardowns() + ) + await waitForMainActorQueueBarrier() + + #expect(failureCount == recoveryIDs.count) + } + @Test func newRecoveryDefersBehindScheduledOlderGrant() async throws { let coordinator = TerminalSurfaceRuntimeTeardownCoordinator( maximumRuntimeSurfaceOwnerCount: 2 From b9ad9dfc326057a5427eb444cf995d8790136268 Mon Sep 17 00:00:00 2001 From: Lawrence Chen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:39:10 -0700 Subject: [PATCH 521/521] fix(macOS): compact stale failure deliveries --- ...erminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift index 8a394e8cb42c..c639c0c2043f 100644 --- a/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift +++ b/Packages/macOS/CmuxTerminal/Sources/CmuxTerminal/Lifecycle/TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain.swift @@ -37,6 +37,12 @@ internal final class TerminalSurfaceRuntimeOwnershipRecoveryFailureDrain: guard !failures.isEmpty else { return } var startGate: TerminalSurfaceRuntimeTeardownStartGate? state.withLockUnchecked { state in + if state.nextFailureIndex < state.failures.count { + state.failures = state.failures[state.nextFailureIndex...].filter { + admission.ownsStalledCloseFailure($0) + } + state.nextFailureIndex = 0 + } let pendingCount = state.failures.count - state.nextFailureIndex precondition( pendingCount + failures.count <= maximumFailureCount,