diff --git a/docs/content/docs/reference/cua-driver/cli-reference.mdx b/docs/content/docs/reference/cua-driver/cli-reference.mdx index 2128605236..bffda4b0b1 100644 --- a/docs/content/docs/reference/cua-driver/cli-reference.mdx +++ b/docs/content/docs/reference/cua-driver/cli-reference.mdx @@ -40,9 +40,9 @@ Print a tool's full description and JSON input schema. ### `cua-driver call` -Invoke an MCP tool directly from the shell. +Invoke an MCP tool through the running daemon. -Sends the tool request to the required Cua Driver daemon. JSON arguments may be passed as a positional JSON object or through stdin. If the daemon is unavailable, the command fails; it never executes the tool in the CLI process. +Requires a Cua Driver daemon. JSON arguments may be passed as a positional JSON object or through stdin. **Arguments:** @@ -62,9 +62,9 @@ Sends the tool request to the required Cua Driver daemon. JSON arguments may be ### `cua-driver mcp` -Run the stdio MCP server. +Run the daemon-backed stdio MCP proxy. -Every MCP process is a stdio proxy to a Cua Driver daemon. On macOS it can auto-launch the CuaDriver.app daemon so TCC grants attach to the bundle. On Windows and Linux, the daemon must already be running. +Every MCP tool call is forwarded to a Cua Driver daemon. On macOS the proxy can auto-launch CuaDriver.app; on Windows and Linux the daemon must already be running. **Options:** @@ -140,7 +140,7 @@ Supported clients include claude, codex, cursor, antigravity, openclaw, opencode Control trajectory recording on a running daemon. -Recording state lives in the daemon and is shared across daemon-backed clients. +Recording state lives in the required daemon and survives client reconnects. **Options:** diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx index 0e3725b503..214ebcf287 100644 --- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx +++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx @@ -524,7 +524,7 @@ Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restar **Windows + Linux use an ffmpeg subprocess** (`gdigrab` / `x11grab` + libx264). Requires ffmpeg on PATH (winget install Gyan.FFmpeg / apt install ffmpeg); when ffmpeg is missing or fails on startup the per-turn capture (screenshots + action.json) still runs and the session's `last_error` field carries the diagnostic. -State persists for the life of the daemon / MCP session; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4. +State persists for the life of the daemon; a restart resets to disabled with no on-disk state. Call `stop_recording` to disable + finalize the mp4. **Arguments:** diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/page.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/page.rs index f9fdd0c498..731c248a50 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/page.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/page.rs @@ -51,14 +51,14 @@ pub struct ClickElementResult { pub trait PageBackend: Send + Sync { /// Returns the visible text of the page (rough analog of /// `document.body.innerText`). - async fn get_text(&self, pid: i32, window_id: u32) -> anyhow::Result; + async fn get_text(&self, pid: i32, window_id: u64) -> anyhow::Result; /// Find elements matching `css_selector` and return a formatted-text /// response (same human-readable shape macOS already emits). async fn query_dom( &self, pid: i32, - window_id: u32, + window_id: u64, css_selector: &str, attributes: &[String], ) -> anyhow::Result; @@ -68,7 +68,7 @@ pub trait PageBackend: Send + Sync { async fn execute_javascript( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, ) -> anyhow::Result; @@ -80,7 +80,7 @@ pub trait PageBackend: Send + Sync { async fn execute_javascript_targeted( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -122,7 +122,7 @@ pub trait PageBackend: Send + Sync { async fn click_element( &self, _pid: i32, - _window_id: u32, + _window_id: u64, _selector: &str, ) -> anyhow::Result { anyhow::bail!( @@ -162,7 +162,7 @@ pub trait PageBackend: Send + Sync { async fn type_keystrokes( &self, _pid: i32, - _window_id: u32, + _window_id: u64, _text: &str, _cdp_port: Option, _target_url_contains: Option<&str>, @@ -191,7 +191,7 @@ pub trait PageBackend: Send + Sync { async fn insert_text( &self, _pid: i32, - _window_id: u32, + _window_id: u64, _text: &str, _cdp_port: Option, _target_url_contains: Option<&str>, @@ -327,9 +327,9 @@ impl Tool for PageTool { // `pid` / `window_id` are resolved per-action: every action except // `enable_javascript_apple_events` needs both. We resolve once here // so each arm can `?` on the Result and we get matching error text. - // Narrowing casts use `TryFrom` so out-of-range JSON numbers fail - // with an actionable error instead of silently truncating to the - // wrong process / window. + // PID narrowing uses `TryFrom` so out-of-range JSON numbers fail + // instead of silently truncating. Window IDs remain u64 because + // native Wayland accessibility providers can legitimately exceed u32. let resolve_pid = |args: &Value| -> Result { let raw = args .get("pid") @@ -337,17 +337,14 @@ impl Tool for PageTool { .ok_or_else(|| "Missing required parameter: pid".to_owned())?; i32::try_from(raw).map_err(|_| format!("Invalid parameter: pid {raw} out of i32 range")) }; - let resolve_window_id = |args: &Value| -> Result { - let raw = args - .get("window_id") + let resolve_window_id = |args: &Value| -> Result { + args.get("window_id") .and_then(|v| v.as_u64()) - .ok_or_else(|| "Missing required parameter: window_id".to_owned())?; - u32::try_from(raw) - .map_err(|_| format!("Invalid parameter: window_id {raw} out of u32 range")) + .ok_or_else(|| "Missing required parameter: window_id".to_owned()) }; let (pid, window_id) = if action == "enable_javascript_apple_events" { - (0i32, 0u32) // unused + (0i32, 0u64) // unused } else { let pid = match resolve_pid(&args) { Ok(v) => v, @@ -533,7 +530,7 @@ mod tests { use super::*; use std::sync::Mutex; - type TargetedCall = (i32, u32, String, Option, Option); + type TargetedCall = (i32, u64, String, Option, Option); #[derive(Default)] struct RecordingBackend { @@ -542,14 +539,14 @@ mod tests { #[async_trait] impl PageBackend for RecordingBackend { - async fn get_text(&self, _pid: i32, _window_id: u32) -> anyhow::Result { + async fn get_text(&self, _pid: i32, _window_id: u64) -> anyhow::Result { Ok(String::new()) } async fn query_dom( &self, _pid: i32, - _window_id: u32, + _window_id: u64, _css_selector: &str, _attributes: &[String], ) -> anyhow::Result { @@ -559,7 +556,7 @@ mod tests { async fn execute_javascript( &self, _pid: i32, - _window_id: u32, + _window_id: u64, _javascript: &str, ) -> anyhow::Result { anyhow::bail!("untargeted execute must not be used") @@ -568,7 +565,7 @@ mod tests { async fn execute_javascript_targeted( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -588,11 +585,12 @@ mod tests { async fn execute_javascript_forwards_explicit_page_target() { let backend = Arc::new(RecordingBackend::default()); let tool = PageTool::new(backend.clone()); + let synthetic_wayland_window_id = u64::from(u32::MAX) + 0x1234; let result = tool .invoke(serde_json::json!({ "pid": 42, - "window_id": 7, + "window_id": synthetic_wayland_window_id, "action": "execute_javascript", "javascript": "document.title", "cdp_port": 9333, @@ -605,7 +603,7 @@ mod tests { *backend.targeted.lock().unwrap(), Some(( 42, - 7, + synthetic_wayland_window_id, "document.title".to_owned(), Some(9333), Some("#window-b".to_owned()), diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/daemon.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/daemon.rs index 9fa5c6c196..1ba512b4ce 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/daemon.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/daemon.rs @@ -98,7 +98,44 @@ fn daemon_is_listening(_binary: &Path, socket: &str) -> bool { std::os::unix::net::UnixStream::connect(socket).is_ok() } -#[cfg(not(unix))] +#[cfg(target_os = "windows")] +fn daemon_is_listening(_binary: &Path, socket: &str) -> bool { + use std::io::{BufRead, BufReader, Write}; + + // Exercise the real named-pipe protocol instead of spawning `status`. + // A finite CLI command is wrapped by the telemetry completion observer, + // which makes it an unnecessarily heavy and timing-sensitive readiness + // probe on hosted Windows runners. Completing `list` also proves that the + // server has progressed past pipe creation and can service the connection. + let Ok(pipe) = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(socket) + else { + return false; + }; + let Ok(mut writer) = pipe.try_clone() else { + return false; + }; + if writer + .write_all(b"{\"method\":\"list\"}\n") + .and_then(|()| writer.flush()) + .is_err() + { + return false; + } + + let mut response = String::new(); + if BufReader::new(pipe).read_line(&mut response).is_err() { + return false; + } + serde_json::from_str::(&response) + .ok() + .and_then(|value| value.get("ok").and_then(serde_json::Value::as_bool)) + == Some(true) +} + +#[cfg(not(any(unix, target_os = "windows")))] fn daemon_is_listening(binary: &Path, socket: &str) -> bool { Command::new(binary) .args(["status", "--socket", socket]) diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs index efba0fa669..6c4f5324d3 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs @@ -109,6 +109,7 @@ pub enum Scope { #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] #[serde(rename_all = "snake_case")] pub enum DriverRoute { + CaptureScopeGate, AxRead, WindowState, UiaInvoke, @@ -1361,7 +1362,7 @@ impl CatalogPolicy { fn case_requires_action_turn(case: &CaseSpec) -> bool { !matches!( case.driver_route, - DriverRoute::AxRead | DriverRoute::WindowState + DriverRoute::CaptureScopeGate | DriverRoute::AxRead | DriverRoute::WindowState ) && case.action != "screenshot" } @@ -2016,6 +2017,22 @@ mod tests { .any(|error| error.contains("missing turn evidence"))); } + #[test] + fn strict_capture_scope_gate_does_not_invent_an_action_turn() { + let case = CaseSpec::delivered( + "window-scope-gate", + "desktop", + "x11", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::CaptureScopeGate, + vec![OracleKind::Protocol], + ); + assert!(!case_requires_action_turn(&case)); + } + #[test] fn validator_exposes_missing_legacy_modal_images() { let (root, case, result, turn) = complete_turn_fixture(); diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs index ddb4e5ce62..684e2b6888 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs @@ -20,21 +20,45 @@ pub struct ForegroundSentinel { impl ForegroundSentinel { pub fn launch(driver: &mut impl Driver) -> Self { - let electron = electron_fixture(); - assert!( - electron.path.exists(), - "Electron sentinel fixture is missing at {}", - electron.path.display() + let mut last_error = None; + for attempt in 1..=2 { + match Self::try_launch(driver) { + Ok(sentinel) => return sentinel, + Err(error) => { + eprintln!( + "[testkit] foreground sentinel launch attempt {attempt}/2 failed: {error}" + ); + last_error = Some(error); + if attempt < 2 { + std::thread::sleep(Duration::from_millis(300)); + } + } + } + } + panic!( + "could not launch foreground sentinel after bounded retry: {}", + last_error.unwrap_or_else(|| "unknown launch error".to_owned()) ); + } + + fn try_launch(driver: &mut impl Driver) -> Result { + let electron = electron_fixture(); + if !electron.path.exists() { + return Err(format!( + "Electron sentinel fixture is missing at {}", + electron.path.display() + )); + } let user_data = tempfile::Builder::new() .prefix("cua-e2e-sentinel-") .tempdir() - .expect("create sentinel user-data directory"); + .map_err(|error| format!("create sentinel user-data directory: {error}"))?; let journal_path = user_data.path().join("sentinel-events.jsonl"); - fs::write(&journal_path, "").expect("initialize sentinel event journal"); + fs::write(&journal_path, "") + .map_err(|error| format!("initialize sentinel event journal: {error}"))?; let cdp_port = TcpListener::bind(("127.0.0.1", 0)) .and_then(|listener| listener.local_addr()) - .expect("allocate sentinel CDP port") + .map_err(|error| format!("allocate sentinel CDP port: {error}"))? .port(); let mut command = Command::new(&electron.path); command @@ -45,7 +69,8 @@ impl ForegroundSentinel { .env("CUA_ELECTRON_CDP_PORT", cdp_port.to_string()) .stdout(Stdio::null()) .stderr(Stdio::null()); - let child = spawn_in_job(&mut command).expect("launch foreground sentinel"); + let child = spawn_in_job(&mut command) + .map_err(|error| format!("launch foreground sentinel: {error}"))?; let launched_pid = child.id(); let mut reaper = ChildReaper::new(); reaper.push(child); @@ -73,10 +98,9 @@ impl ForegroundSentinel { ); break target; } - assert!( - Instant::now() < window_deadline, - "foreground sentinel window did not appear" - ); + if Instant::now() >= window_deadline { + return Err("foreground sentinel window did not appear".to_owned()); + } std::thread::sleep(Duration::from_millis(100)); }; reaper.track_pid(target.pid); @@ -84,24 +108,25 @@ impl ForegroundSentinel { let focus_deadline = Instant::now() + Duration::from_secs(10); if is_wayland_session() { wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready"); - activate_native_foreground(driver, target); + try_activate_native_foreground(driver, target)?; // Electron may already be focused before its preload listener is ready. // The compositor observation is the authoritative Wayland focus gate. wait_for_native_focus_stable(target); } else { wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready"); - activate_native_foreground(driver, target); + try_activate_native_foreground(driver, target)?; wait_for_native_focus_stable(target); wait_for_journal(&journal_path, focus_deadline, r#""kind":"focus""#, "focus"); } - fs::write(&journal_path, "").expect("reset focused sentinel journal"); + fs::write(&journal_path, "") + .map_err(|error| format!("reset focused sentinel journal: {error}"))?; - Self { + Ok(Self { journal_path, target, _reaper: reaper, _user_data: user_data, - } + }) } pub fn observe(&self) -> (Vec, Vec) { @@ -173,12 +198,22 @@ impl ForegroundSentinel { wait_for_event(&self.journal_path, "heartbeat", Duration::from_secs(2))?; reset_journal(&self.journal_path)?; + let canary_key = if std::env::var("CUA_E2E_WAYLAND_SESSION") + .is_ok_and(|session| session == "cua-compositor") + { + // The intentionally small injection protocol exposes navigation + // and control keys, not printable letters. Space still exercises + // the renderer keydown leak detector without broadening it. + "space" + } else { + "a" + }; let leaked_key = driver.call( "press_key", serde_json::json!({ "pid": self.target.pid, "window_id": self.target.native_id, - "key": "a", + "key": canary_key, "delivery_mode": "foreground", }), ); @@ -282,6 +317,13 @@ impl ForegroundSentinel { wait_for_native_focus_stable(self.target); std::thread::sleep(Duration::from_millis(100)); reset_journal(&self.journal_path)?; + // Windows establishes focus with a physical click. Its DOM `click` + // can arrive after the native focus transition and the first journal + // reset, falsely attributing setup input to the background action. + // A later heartbeat is an event-loop barrier: once observed, clear the + // journal again so the action boundary starts from a quiet sentinel. + wait_for_event(&self.journal_path, "heartbeat", Duration::from_secs(2))?; + reset_journal(&self.journal_path)?; self.assert_background_posture(target) } @@ -417,6 +459,14 @@ fn is_wayland_session() -> bool { } fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) { + try_activate_native_foreground(driver, target) + .unwrap_or_else(|error| panic!("could not activate foreground sentinel: {error}")); +} + +fn try_activate_native_foreground( + driver: &mut impl Driver, + target: TargetWindow, +) -> Result<(), String> { let response = driver.call( "bring_to_front", serde_json::json!({ @@ -424,16 +474,16 @@ fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) { "window_id": target.native_id, }), ); - assert!( - !response.is_error(), - "could not activate foreground sentinel: {}", - response.text() - ); + if response.is_error() { + return Err(response.text().to_owned()); + } #[cfg(target_os = "linux")] - focus_sway_target(driver, target) - .expect("could not focus foreground sentinel through Sway IPC"); + focus_sway_target(driver, target).map_err(|error| { + format!("could not focus foreground sentinel through Sway IPC: {error}") + })?; #[cfg(target_os = "windows")] physically_focus_windows_sentinel(target); + Ok(()) } #[cfg(target_os = "linux")] diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index fd466c6973..8726556ae6 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -801,15 +801,7 @@ fn build_registry( #[cfg(target_os = "linux")] { cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(xid) = window_id { - platform_linux::wayland::screenshot_dispatch(xid).ok() - } else if let Some(p) = pid { - let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32)); - wins.first() - .and_then(|w| platform_linux::wayland::screenshot_dispatch(w.xid).ok()) - } else { - platform_linux::capture::screenshot_display_bytes().ok() - } + platform_linux::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() @@ -907,15 +899,7 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { tracing::warn!("could not activate the persistent AT-SPI listener: {error}"); } cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(xid) = window_id { - platform_linux::wayland::screenshot_dispatch(xid).ok() - } else if let Some(p) = pid { - let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32)); - wins.first() - .and_then(|w| platform_linux::wayland::screenshot_dispatch(w.xid).ok()) - } else { - platform_linux::capture::screenshot_display_bytes().ok() - } + platform_linux::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() diff --git a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs index 0c48b97b95..60dbdb98e1 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs @@ -471,12 +471,10 @@ async fn invoke_daemon_tool( } } - if !known_tool { - observe_daemon_error(observation, 64); - return DaemonResponse::err(format!("Unknown tool: {tool_name}"), 64); - } - // Policy enforcement — defense-in-depth for direct daemon socket connections. + // Evaluate before registry lookup so a deny-by-default policy does not leak + // whether an unapproved name happens to be registered. This also preserves + // the MCP policy contract now that every call passes through the daemon. match cua_driver_core::policy::configured_policy() { Ok(Some(policy)) => match policy.evaluate(&tool_name, &args) { cua_driver_core::policy::PolicyDecision::Allow => {} @@ -496,6 +494,11 @@ async fn invoke_daemon_tool( } } + if !known_tool { + observe_daemon_error(observation, 64); + return DaemonResponse::err(format!("Unknown tool: {tool_name}"), 64); + } + inject_browser_approvals(&tool_name, &mut args, req.session_id.as_deref()); let session_context = observation_transport.and_then(|transport| { diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs index 98065d2c2d..e5f9624753 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs @@ -258,7 +258,7 @@ fn window_scope_rejects_windowless_click() { Targeting::Px, Delivery::NotApplicable, Scope::Window, - DriverRoute::Composite, + DriverRoute::CaptureScopeGate, vec![OracleKind::Protocol], ); execute_case(case, |evidence| { diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs index 13860aa49e..cdeb25d64a 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs @@ -264,7 +264,7 @@ fn window_scope_rejects_windowless_click() { Targeting::Px, Delivery::NotApplicable, Scope::Window, - DriverRoute::Composite, + DriverRoute::CaptureScopeGate, vec![OracleKind::Protocol], ); execute_case(case, |evidence| { diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs index a2a9029dc7..2948a1077b 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs @@ -304,7 +304,7 @@ fn window_scope_rejects_windowless_click() { Targeting::Px, Delivery::NotApplicable, Scope::Window, - DriverRoute::Composite, + DriverRoute::CaptureScopeGate, vec![OracleKind::Protocol], ); execute_case(case, |evidence| { diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs index 0e481992a5..ad92e3c32b 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs @@ -328,12 +328,17 @@ fn run_preflight() { }; driver.reaper().push(child.into_child()); - driver.start_behavior_recording(); let target = TargetWindow { pid: pid as u32, native_id: window_id, }; let sentinel = ForegroundSentinel::launch(&mut driver); + // Sentinel activation is preflight setup, not behavior under test. Starting + // the recorder before launch turns its bring_to_front call into a captured + // action; nested Wayland then blocks on a setup-only full-display preimage. + // Keep the deliberate guard canaries in the recording while excluding the + // activation that establishes their baseline. + driver.start_behavior_recording(); sentinel .assert_guard_canaries(&mut driver, target) .expect("foreground sentinel guard canaries failed"); diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index 97987e4910..76b74c9cab 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -169,6 +169,15 @@ fn window_bounds(driver: &mut McpDriver, pid: u32, wid: u64) -> (f64, f64, f64, } fn pixel_center(state: &ToolResponse, target_id: &str, window: (f64, f64, f64, f64)) -> (f64, f64) { + let (x, y, width, height) = pixel_frame(state, target_id, window); + (x + width / 2.0, y + height / 2.0) +} + +fn pixel_frame( + state: &ToolResponse, + target_id: &str, + window: (f64, f64, f64, f64), +) -> (f64, f64, f64, f64) { let target_index = ax::element_index_by_id(state.text(), target_id) .unwrap_or_else(|| panic!("missing PX target {target_id:?}: {}", state.text())); let elements = state.structured()["elements"] @@ -196,11 +205,15 @@ fn pixel_center(state: &ToolResponse, target_id: &str, window: (f64, f64, f64, f let scale_y = screenshot_h / window_h; let x = (target["x"].as_f64().unwrap_or(0.0) + target_w / 2.0 - window_x) * scale_x; let y = (target["y"].as_f64().unwrap_or(0.0) + target_h / 2.0 - window_y) * scale_y; + let width = target_w * scale_x; + let height = target_h * scale_y; + let x = x - width / 2.0; + let y = y - height / 2.0; assert!( - x >= 0.0 && x < screenshot_w && y >= 0.0 && y < screenshot_h, - "WPF PX target center ({x:.1}, {y:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" + x >= 0.0 && x + width <= screenshot_w && y >= 0.0 && y + height <= screenshot_h, + "WPF PX target frame ({x:.1}, {y:.1}, {width:.1}, {height:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" ); - (x, y) + (x, y, width, height) } fn wait_for_fixture_file_text(path: &std::path::Path, id: &str, expected: &str) { @@ -1116,19 +1129,17 @@ fn harness_wpf_slider_drag() { pre.text().contains("slider_value=0"), "initial slider_value=0 missing" ); + let (x, y, width, height) = + pixel_frame(&pre, "sld-value", window_bounds(driver, pid, wid)); let resp = driver.call( "drag", serde_json::json!({ "pid": pid as i64, "window_id": wid, - // Window-local coords along the slider TRACK. The track row sits at - // window-local y≈304 (verified on the VM: y=275 landed ~29px above - // it, on empty GroupBox space, so the thumb never moved); the thumb - // rests at the left (x≈44) at value=0. Dragging left→right advances - // the value. (TODO: derive these from the `sld-value` element frame - // in get_window_state for DPI/placement independence.) - "from_x": 44.0, "from_y": 304.0, - "to_x": 330.0, "to_y": 304.0, + // Resolve the live UIA frame so fixture reordering and DPI + // scaling cannot silently move this drag onto another control. + "from_x": x + width * 0.05, "from_y": y + height / 2.0, + "to_x": x + width * 0.90, "to_y": y + height / 2.0, "duration_ms": 700, "steps": 40, "delivery_mode": "foreground" }), diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_element_token_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_element_token_test.rs index 21a31d045e..79b62f8650 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_element_token_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_element_token_test.rs @@ -12,61 +12,21 @@ // then-unused tools/list helpers. #![cfg(any(target_os = "macos", target_os = "linux"))] -use std::io::{BufRead, BufReader, Write}; -use std::process::{Command, Stdio}; - -use cua_driver_testkit::driver_binary; - -fn send_request(stdin: &mut impl Write, request: &serde_json::Value) { - let line = serde_json::to_string(request).unwrap(); - writeln!(stdin, "{}", line).unwrap(); -} - -fn read_response(reader: &mut impl BufRead) -> serde_json::Value { - let mut line = String::new(); - reader.read_line(&mut line).expect("read line"); - serde_json::from_str(line.trim()).expect("parse JSON") -} +use cua_driver_testkit::RawDriver; /// Spawn the driver, send initialize + tools/list, return the parsed /// `tools/list` response. Skips the test silently if the binary hasn't /// been built (CI builds it separately). fn fetch_tools_list() -> Option { - let binary = driver_binary(); - if !binary.exists() { - eprintln!("Binary not found at {:?} — run `cargo build` first", binary); - return None; - } - - let mut child = Command::new(&binary) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn cua-driver"); - - { - let stdin = child.stdin.as_mut().unwrap(); - let mut stdout = BufReader::new(child.stdout.as_mut().unwrap()); - - send_request( - stdin, - &serde_json::json!({ - "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} - }), - ); - let _ = read_response(&mut stdout); - - send_request( - stdin, - &serde_json::json!({ - "jsonrpc": "2.0", "id": 2, "method": "tools/list" - }), - ); - let resp = read_response(&mut stdout); - child.kill().ok(); - return Some(resp); - } + let mut driver = RawDriver::spawn()?; + driver.send(&serde_json::json!({ + "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} + })); + driver.recv(); + driver.send(&serde_json::json!({ + "jsonrpc": "2.0", "id": 2, "method": "tools/list" + })); + Some(driver.recv()) } /// Surface 6: Every tool that accepts the opaque `element_token` arg diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs index cda1e0927a..9ef2bd029f 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs @@ -52,6 +52,31 @@ pub fn walk_tree(pid: u32, xid: u64, query: Option<&str>) -> AtspiTreeResult { walk_tree_bounded(pid, xid, query, None, None) } +/// Best-effort accessibility snapshot for synchronous trajectory evidence. +/// +/// Recording brackets an action with before/after captures, so it must use a +/// smaller budget than the transport's tool-call deadline. Unlike the +/// interactive tree walker, evidence capture makes one attempt and accepts an +/// unavailable tree when the target renderer is blocked. +pub(crate) fn walk_tree_for_recording( + pid: u32, + xid: u64, + timeout: std::time::Duration, +) -> AtspiTreeResult { + if let Ok(Some((tree_markdown, nodes, bounds))) = + native::walk_tree_bounded_with_timeout(pid, xid, None, None, timeout) + { + if !tree_markdown.is_empty() { + return AtspiTreeResult { + tree_markdown, + nodes, + bounds, + }; + } + } + walk_via_x11_properties(xid, None) +} + /// Walk the AT-SPI tree with caller-supplied caps. `None` for either cap /// means "use the walker's built-in default" (5 000 nodes; unlimited depth). /// Issue #22865: caps protect against Electron / large web apps that diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs index 937e09dac3..7818534644 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs @@ -137,6 +137,10 @@ struct Visited<'a> { /// True when an ancestor is a web document (e.g. role "document web"), /// i.e. this node is page content rather than browser chrome. in_web_doc: bool, + /// True when this node is exported by a separate WebKit WebProcess bus. + /// Chromium keeps its document on the application's ordinary AT-SPI bus, + /// where descendant Window extents already include the document origin. + on_web_process_bus: bool, acc: AccessibleProxy<'a>, } @@ -574,6 +578,7 @@ async fn collect_visited_bounded<'a>( has_component, focused, in_web_doc, + on_web_process_bus: is_web_process_bus(&oref.name), acc, }); } @@ -703,13 +708,23 @@ pub fn walk_tree_bounded( xid: u64, max_elements: Option, max_depth: Option, +) -> Result, Vec<(usize, i32, i32, u32, u32)>)>> { + walk_tree_bounded_with_timeout(pid, xid, max_elements, max_depth, OP_TIMEOUT) +} + +pub(super) fn walk_tree_bounded_with_timeout( + pid: u32, + xid: u64, + max_elements: Option, + max_depth: Option, + timeout: Duration, ) -> Result, Vec<(usize, i32, i32, u32, u32)>)>> { runtime().block_on(async { let walk = async { let conn = shared_connection().await?; collect_visited_bounded(conn, pid, max_elements, max_depth).await }; - let visited = match tokio::time::timeout(OP_TIMEOUT, walk).await { + let visited = match tokio::time::timeout(timeout, walk).await { Ok(result) => result?, Err(_) => { dlog!("walk_tree timed out for pid {pid}"); @@ -1921,7 +1936,15 @@ fn prefer_authoritative_wayland_origin( fn combine_wayland_content_offsets( compositor: Option<(i32, i32)>, document: Option<(i32, i32)>, + document_is_separate: bool, ) -> Option<(i32, i32)> { + if !document_is_separate { + // Chromium descendants' CoordType::Window extents are already rooted + // below the document accessible. Adding that document's own (x,y) + // double-counts its renderer inset. WebKitGTK exports page content on + // a distinct WebProcess bus, so only that bridge needs the extra hop. + return compositor; + } match (compositor, document) { (Some((cx, cy)), Some((dx, dy))) => Some((cx + dx, cy + dy)), (Some(offset), None) | (None, Some(offset)) => Some(offset), @@ -1974,9 +1997,10 @@ async fn web_document_origin_for_visited(visited: &[Visited<'_>], pid: u32) -> O } else { None }; - let combined = combine_wayland_content_offsets(compositor, document); + let document_is_separate = visited.iter().any(|node| node.on_web_process_bus); + let combined = combine_wayland_content_offsets(compositor, document, document_is_separate); dlog!( - "Wayland web content offset: compositor={compositor:?} document={document:?} combined={combined:?}" + "Wayland web content offset: compositor={compositor:?} document={document:?} separate_process={document_is_separate} combined={combined:?}" ); combined } @@ -2249,18 +2273,30 @@ mod coord_tests { #[test] fn wayland_compositor_and_document_offsets_are_additive() { assert_eq!( - combine_wayland_content_offsets(Some((0, 47)), Some((0, 0))), + combine_wayland_content_offsets(Some((0, 47)), Some((0, 0)), true), Some((0, 47)) ); assert_eq!( - combine_wayland_content_offsets(Some((2, 20)), Some((0, 47))), + combine_wayland_content_offsets(Some((2, 20)), Some((0, 47)), true), Some((2, 67)) ); assert_eq!( - combine_wayland_content_offsets(None, Some((0, 47))), + combine_wayland_content_offsets(None, Some((0, 47)), true), Some((0, 47)) ); - assert_eq!(combine_wayland_content_offsets(None, None), None); + assert_eq!(combine_wayland_content_offsets(None, None, true), None); + } + + #[test] + fn chromium_window_extents_do_not_double_count_document_origin() { + assert_eq!( + combine_wayland_content_offsets(Some((2, 20)), Some((22, 55)), false), + Some((2, 20)) + ); + assert_eq!( + combine_wayland_content_offsets(None, Some((22, 55)), false), + None + ); } #[test] diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs index b1aff9e976..22cdbe2eda 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_consent_ui.rs @@ -159,6 +159,55 @@ fn prove_window_owner(pid: u32, window_id: u64) -> Result<(), BrowserRefusal> { Ok(()) } +fn with_target_foreground( + pid: u32, + window_id: u64, + body: impl FnOnce() -> anyhow::Result, +) -> anyhow::Result { + if std::env::var_os("WAYLAND_DISPLAY").is_some() { + let window = crate::wayland::sway_ipc::window_for_id(window_id) + .filter(|window| window.pid == pid) + .ok_or_else(|| anyhow::anyhow!("no exact Sway container owns the approved window"))?; + crate::wayland::sway_ipc::with_focused_container(window.id, body) + } else { + crate::input::with_x11_foreground(window_id, 80, body) + } +} + +fn exact_button_center( + bounds: &[(usize, i32, i32, u32, u32)], + element_index: usize, +) -> anyhow::Result<(i32, i32)> { + let (_, x, y, width, height) = bounds + .iter() + .find(|(index, _, _, width, height)| *index == element_index && *width > 1 && *height > 1) + .ok_or_else(|| anyhow::anyhow!("the exact Allow action had empty screen bounds"))?; + let center_x = x + .checked_add(i32::try_from(width / 2)?) + .ok_or_else(|| anyhow::anyhow!("Allow button center x overflowed"))?; + let center_y = y + .checked_add(i32::try_from(height / 2)?) + .ok_or_else(|| anyhow::anyhow!("Allow button center y overflowed"))?; + Ok((center_x, center_y)) +} + +fn trusted_allow_click(pid: u32, window_id: u64) -> anyhow::Result<()> { + with_target_foreground(pid, window_id, || { + let tree = crate::atspi::walk_tree(pid, window_id, None); + let index = exact_allow_button(&tree.nodes, &tree.bounds) + .map_err(|error| anyhow::anyhow!(error.message))? + .ok_or_else(|| { + anyhow::anyhow!("the exact Chromium remote-debugging consent action became stale") + })?; + let (center_x, center_y) = exact_button_center(&tree.bounds, index)?; + if std::env::var_os("WAYLAND_DISPLAY").is_some() { + crate::wayland::click_desktop(center_x, center_y, 1, 1) + } else { + crate::input::send_click_xtest_desktop(center_x, center_y, 1, 1) + } + }) +} + pub async fn handle( request: BrowserConsentRequest, ) -> Result { @@ -171,6 +220,8 @@ pub async fn handle( prove_window_owner(pid, request.window_id)?; let deadline = Instant::now() + Duration::from_secs(4); let mut saw_prompt = false; + let mut accessibility_action_at = None; + let mut trusted_click_attempted = false; loop { prove_window_owner(pid, request.window_id)?; let window_id = request.window_id; @@ -186,7 +237,7 @@ pub async fn handle( let prompt_present = remote_debugging_prompt_present(&tree.nodes); saw_prompt |= prompt_present; match exact_allow_button(&tree.nodes, &tree.bounds)? { - Some(index) => { + Some(index) if accessibility_action_at.is_none() => { tokio::task::spawn_blocking(move || crate::atspi::perform_action(pid, index)) .await .map_err(|error| { @@ -201,9 +252,37 @@ pub async fn handle( format!("the exact browser consent action failed: {error}"), ) })?; - return Ok(BrowserConsentOutcome::Accepted); + accessibility_action_at = Some(Instant::now()); + } + Some(_) + if !trusted_click_attempted + && accessibility_action_at.is_some_and(|attempted| { + attempted.elapsed() >= Duration::from_millis(150) + }) => + { + let window_id = request.window_id; + tokio::task::spawn_blocking(move || trusted_allow_click(pid, window_id)) + .await + .map_err(|error| { + refusal( + BrowserRefusalCode::BrowserRouteUnavailable, + format!( + "could not dispatch the trusted browser consent click: {error}" + ), + ) + })? + .map_err(|error| { + refusal( + BrowserRefusalCode::BrowserWrongTargetRefused, + format!("the trusted browser consent click failed: {error}"), + ) + })?; + trusted_click_attempted = true; } None if saw_prompt && !prompt_present => { + if accessibility_action_at.is_some() { + return Ok(BrowserConsentOutcome::Accepted); + } return Err(refusal( BrowserRefusalCode::BrowserConsentRevoked, "the person dismissed the browser consent prompt", @@ -218,8 +297,9 @@ pub async fn handle( ), )); } - None => tokio::time::sleep(Duration::from_millis(100)).await, + _ => {} } + tokio::time::sleep(Duration::from_millis(50)).await; } } @@ -301,4 +381,14 @@ mod tests { } assert_eq!(exact_allow_button(&nodes, &[]).unwrap(), None); } + + #[test] + fn exact_button_center_requires_nonempty_bounds() { + assert_eq!( + exact_button_center(&[(7, 10, 20, 80, 30)], 7).unwrap(), + (50, 35) + ); + assert!(exact_button_center(&[(7, 10, 20, 1, 30)], 7).is_err()); + assert!(exact_button_center(&[(8, 10, 20, 80, 30)], 7).is_err()); + } } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs index 3495892e8c..24dd9d09b9 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/browser_platform.rs @@ -1,6 +1,7 @@ //! Linux identity and endpoint evidence for the first-class browser tools. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::PathBuf; use std::time::Duration; use async_trait::async_trait; @@ -106,28 +107,77 @@ fn parse_proc_net_loopback_listeners(text: &str) -> Vec<(u16, u64)> { .collect() } -fn socket_inodes_for_pid(pid: i64) -> Result, BrowserRefusal> { - let directory = std::fs::read_dir(format!("/proc/{pid}/fd")).map_err(|_| { - refusal( +fn descendant_pids(root: i64, relationships: impl IntoIterator) -> HashSet { + let mut children = HashMap::>::new(); + for (pid, parent) in relationships { + children.entry(parent).or_default().push(pid); + } + let mut family = HashSet::from([root]); + let mut pending = VecDeque::from([root]); + while let Some(parent) = pending.pop_front() { + for child in children.get(&parent).into_iter().flatten() { + if family.insert(*child) { + pending.push_back(*child); + } + } + } + family +} + +fn process_family_pids(root: i64) -> Result, BrowserRefusal> { + if !std::path::Path::new(&format!("/proc/{root}")).exists() { + return Err(refusal( BrowserRefusalCode::BrowserBindingStale, - format!("browser process {pid} is no longer available"), - ) - })?; - Ok(directory + format!("browser process {root} is no longer available"), + )); + } + let relationships = std::fs::read_dir("/proc") + .into_iter() .flatten() - .filter_map(|entry| std::fs::read_link(entry.path()).ok()) - .filter_map(|target| { - let target = target.to_string_lossy(); - target - .strip_prefix("socket:[") - .and_then(|value| value.strip_suffix(']')) - .and_then(|value| value.parse::().ok()) - }) - .collect()) + .flatten() + .filter_map(|entry| { + let pid = entry.file_name().to_string_lossy().parse::().ok()?; + let status = std::fs::read_to_string(entry.path().join("status")).ok()?; + let parent = status + .lines() + .find_map(|line| line.strip_prefix("PPid:"))? + .trim() + .parse::() + .ok()?; + Some((pid, parent)) + }); + Ok(descendant_pids(root, relationships)) +} + +fn socket_inodes_for_process_tree(pid: i64) -> Result, BrowserRefusal> { + let process_family = process_family_pids(pid)?; + let mut inodes = HashSet::new(); + for owner_pid in process_family { + let Ok(directory) = std::fs::read_dir(format!("/proc/{owner_pid}/fd")) else { + continue; + }; + inodes.extend( + directory + .flatten() + .filter_map(|entry| std::fs::read_link(entry.path()).ok()) + .filter_map(|target| { + let target = target.to_string_lossy(); + target + .strip_prefix("socket:[") + .and_then(|value| value.strip_suffix(']')) + .and_then(|value| value.parse::().ok()) + }), + ); + } + Ok(inodes) } fn loopback_ports_for_pid(pid: i64) -> Result, BrowserRefusal> { - let owned = socket_inodes_for_pid(pid)?; + // Chromium may delegate its DevTools listener to a utility child. Core's + // ownership contract explicitly accepts the approved browser PID or one + // of its children, so inspect the bounded descendant tree as well as the + // root process while still attributing the result to the approved root. + let owned = socket_inodes_for_process_tree(pid)?; let mut listeners = Vec::new(); for path in ["/proc/net/tcp", "/proc/net/tcp6"] { if let Ok(text) = std::fs::read_to_string(path) { @@ -144,6 +194,121 @@ fn loopback_ports_for_pid(pid: i64) -> Result, BrowserRefusal> { Ok(listeners) } +fn parse_devtools_active_port(text: &str) -> Option<(u16, &str)> { + let mut lines = text.lines().map(str::trim).filter(|line| !line.is_empty()); + let port = lines.next()?.parse::().ok()?; + let path = lines.next()?; + if lines.next().is_some() { + return None; + } + let instance = path.strip_prefix("/devtools/browser/")?; + (!instance.is_empty() + && instance + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_')) + .then_some((port, path)) +} + +fn default_user_data_dir(product: BrowserProduct) -> Option { + let home = std::env::var_os("HOME").map(PathBuf::from)?; + let relative = match product { + BrowserProduct::GoogleChrome => ".config/google-chrome", + BrowserProduct::MicrosoftEdge => ".config/microsoft-edge", + BrowserProduct::Chromium => ".config/chromium", + _ => return None, + }; + Some(home.join(relative)) +} + +fn user_data_dir_for_pid(pid: i64) -> Result, BrowserRefusal> { + let bytes = std::fs::read(format!("/proc/{pid}/cmdline")).map_err(|_| { + refusal( + BrowserRefusalCode::BrowserBindingStale, + format!("browser process {pid} is no longer available"), + ) + })?; + let args = bytes + .split(|byte| *byte == 0) + .filter(|arg| !arg.is_empty()) + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect::>(); + let mut directories = Vec::new(); + for (index, arg) in args.iter().enumerate() { + if let Some(path) = arg.strip_prefix("--user-data-dir=") { + if !path.is_empty() { + directories.push(PathBuf::from(path)); + } + } else if arg == "--user-data-dir" { + if let Some(path) = args.get(index + 1).filter(|path| !path.is_empty()) { + directories.push(PathBuf::from(path)); + } + } + } + directories.sort(); + directories.dedup(); + match directories.as_slice() { + [] => { + let Some(executable) = std::fs::read_link(format!("/proc/{pid}/exe")) + .ok() + .map(|path| path.to_string_lossy().into_owned()) + else { + return Ok(None); + }; + Ok(default_user_data_dir(browser_product(&executable))) + } + [path] if path.is_absolute() => Ok(Some(path.clone())), + [path] => { + let cwd = std::fs::read_link(format!("/proc/{pid}/cwd")).map_err(|_| { + refusal( + BrowserRefusalCode::BrowserBindingStale, + format!("browser process {pid} working directory is unavailable"), + ) + })?; + Ok(Some(cwd.join(path))) + } + _ => Err(refusal( + BrowserRefusalCode::BrowserBindingAmbiguous, + "browser process has multiple distinct --user-data-dir arguments", + )), + } +} + +fn active_port_endpoint(pid: i64) -> Result, BrowserRefusal> { + let Some(user_data_dir) = user_data_dir_for_pid(pid)? else { + return Ok(None); + }; + let text = match std::fs::read_to_string(user_data_dir.join("DevToolsActivePort")) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(refusal( + BrowserRefusalCode::BrowserRouteUnavailable, + format!("could not read the browser's DevToolsActivePort file: {error}"), + )) + } + }; + let Some((port, path)) = parse_devtools_active_port(&text) else { + return Err(refusal( + BrowserRefusalCode::BrowserEndpointOwnerMismatch, + "the browser's DevToolsActivePort file did not contain one exact browser endpoint", + )); + }; + if !loopback_ports_for_pid(pid)?.contains(&port) { + return Ok(None); + } + Ok(Some(OwnedEndpoint { + ws_url: format!("ws://127.0.0.1:{port}{path}"), + http_port: Some(port), + ownership: EndpointOwnershipProof { + method: EndpointOwnershipMethod::DevtoolsActivePortsFile, + owner_pid: pid, + detail: Some( + "exact /proc argv profile port file plus loopback socket inode owner".to_owned(), + ), + }, + })) +} + fn process_identity(pid: i64) -> Result<(u64, Option), BrowserRefusal> { let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).map_err(|_| { refusal( @@ -395,6 +560,28 @@ impl BrowserPlatform for LinuxBrowserPlatform { })?; if std::env::var_os("WAYLAND_DISPLAY").is_some() { let Some(windows) = crate::wayland::sway_ipc::list_windows() else { + if crate::wayland::is_inject_mode() { + // The private cua-compositor route owns both the native + // toplevel enumeration and its PID correlation. Unlike a + // generic AT-SPI-only Wayland session, that is sufficient + // to attest singleton native-window cardinality for an + // embedded Chromium endpoint. + let owned = tokio::task::spawn_blocking(move || { + crate::wayland::list_windows_dispatch(Some(pid_u32)) + .into_iter() + .filter(|window| window.pid == Some(pid_u32)) + .map(|window| window.xid) + .collect::>() + }) + .await + .map_err(|error| { + refusal( + BrowserRefusalCode::BrowserRouteUnavailable, + format!("could not enumerate cua-compositor browser windows: {error}"), + ) + })?; + return Ok(Some(owned.len() == 1 && owned[0] == window_id)); + } return Ok(None); }; let owned = windows @@ -424,6 +611,9 @@ impl BrowserPlatform for LinuxBrowserPlatform { &self, pid: i64, ) -> Result, BrowserRefusal> { + if let Some(endpoint) = active_port_endpoint(pid)? { + return Ok(Some(endpoint)); + } let ports = tokio::task::spawn_blocking(move || loopback_ports_for_pid(pid)) .await .map_err(|error| { @@ -452,6 +642,9 @@ impl BrowserPlatform for LinuxBrowserPlatform { &self, pid: i64, ) -> Result, BrowserRefusal> { + if let Some(endpoint) = active_port_endpoint(pid)? { + return Ok(Some(endpoint)); + } let ports = tokio::task::spawn_blocking(move || loopback_ports_for_pid(pid)) .await .map_err(|error| { @@ -577,6 +770,11 @@ impl BrowserPlatform for LinuxBrowserPlatform { let deadline = std::time::Instant::now() + Duration::from_secs(6); let endpoint_result = loop { + match active_port_endpoint(request.pid) { + Ok(Some(endpoint)) => break Ok(endpoint), + Ok(None) => {} + Err(error) => break Err(error), + } let ports = match tokio::task::spawn_blocking(move || { loopback_ports_for_pid(request.pid) }) @@ -604,10 +802,14 @@ impl BrowserPlatform for LinuxBrowserPlatform { .filter(|port| !listeners_before.contains(port)) .collect::>(); if let [port] = correlated.as_slice() { + // Chromium's consent-gated server intentionally disables + // `/json/*` discovery and accepts the stable browser route + // without a UUID. The exact setup action plus the newly + // PID-owned listener proves which approval server this is. endpoints.push(( *port, format!("ws://127.0.0.1:{port}/devtools/browser"), - "new PID-owned listener correlated with exact approved setup", + "new PID-owned approval listener correlated with exact setup", )); } else if correlated.len() > 1 { break Err(refusal( @@ -782,6 +984,14 @@ mod tests { ); } + #[test] + fn process_family_contains_only_transitive_descendants() { + assert_eq!( + descendant_pids(10, [(11, 10), (12, 11), (20, 1), (21, 20)]), + HashSet::from([10, 11, 12]) + ); + } + #[test] fn classifier_covers_embedded_and_standalone_chromium() { assert!(is_chromium("CuaTestHarness.Electron")); @@ -829,4 +1039,24 @@ mod tests { ); assert_eq!(loopback_websocket_port("ws://0.0.0.0:9222/devtools"), None); } + + #[test] + fn active_port_parser_requires_one_exact_browser_path() { + assert_eq!( + parse_devtools_active_port("9222\n/devtools/browser/abc-123\n"), + Some((9222, "/devtools/browser/abc-123")) + ); + assert_eq!( + parse_devtools_active_port("9222\n/devtools/browser\n"), + None + ); + assert_eq!( + parse_devtools_active_port("9222\n/devtools/page/abc\n"), + None + ); + assert_eq!( + parse_devtools_active_port("9222\n/devtools/browser/../page\n"), + None + ); + } } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs index e77960d3c8..30d62d903c 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs @@ -14,8 +14,25 @@ pub fn app_state_json_for(window_id: Option, pid: Option) -> Option, pid: Option) -> Option> { let pid = u32::try_from(pid?).ok()?; - let window_id = resolve_window_for_recording(pid, window_id)?.xid; - let result = crate::atspi::walk_tree(pid, window_id, None); + let window_id = if crate::wayland::is_inject_mode() { + // Most injected actions already carry the protocol-verified window id. + // Process-scoped setup calls such as browser_prepare do not, so resolve + // their single target here instead of classifying required AX evidence + // as a capture failure. + match window_id { + Some(window_id) => window_id, + None => resolve_window_for_recording(pid, None)?.xid, + } + } else { + resolve_window_for_recording(pid, window_id)?.xid + }; + let result = if crate::wayland::is_inject_mode() { + // Evidence capture runs inside the daemon call. Keep it below the + // transport deadline so an unresponsive renderer cannot block input. + crate::atspi::walk_tree_for_recording(pid, window_id, std::time::Duration::from_secs(2)) + } else { + crate::atspi::walk_tree(pid, window_id, None) + }; if result.nodes.is_empty() || result.tree_markdown.trim().is_empty() { return None; } @@ -33,6 +50,27 @@ fn app_state_json_for_blocking(window_id: Option, pid: Option) -> Opti serde_json::to_vec_pretty(&payload).ok() } +#[cfg(target_os = "linux")] +pub fn screenshot_for_recording(window_id: Option, pid: Option) -> Option> { + if crate::wayland::is_inject_mode() { + // A full-output frame is the strongest evidence for the nested + // compositor's background/focus guarantees and needs no slow AT-SPI + // geometry re-resolution. Per-window screenshots elsewhere retain the + // normal crop behavior. + return crate::wayland::screenshot_display_dispatch().ok(); + } + if let Some(window_id) = window_id { + crate::wayland::screenshot_dispatch(window_id).ok() + } else if let Some(pid) = pid.and_then(|pid| u32::try_from(pid).ok()) { + let windows = crate::wayland::list_windows_dispatch(Some(pid)); + windows + .first() + .and_then(|window| crate::wayland::screenshot_dispatch(window.xid).ok()) + } else { + crate::capture::screenshot_display_bytes().ok() + } +} + #[cfg(target_os = "linux")] pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> { if tokio::runtime::Handle::try_current().is_ok() { @@ -85,6 +123,11 @@ pub fn app_state_json_for(_window_id: Option, _pid: Option) -> Option< None } +#[cfg(not(target_os = "linux"))] +pub fn screenshot_for_recording(_window_id: Option, _pid: Option) -> Option> { + None +} + #[cfg(not(target_os = "linux"))] pub fn element_window_local_xy( _window_id: u64, diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 49abf9805c..7f12769b5b 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -2031,7 +2031,7 @@ impl Tool for ClickTool { } } if crate::wayland::is_inject_mode() { - crate::wayland::inject_click(xid, x, y, count as u32, button)?; + crate::wayland::inject_click(pid, xid, x, y, count as u32, button)?; return Ok("wayland_cua_compositor"); } if !delivery.is_foreground() { @@ -2181,7 +2181,7 @@ async fn focus_nested_inject_target( } if let Some((x, y)) = pixel { return match tokio::task::spawn_blocking(move || { - crate::wayland::inject_click(window_id, x, y, 1, 1) + crate::wayland::inject_click(pid, window_id, x, y, 1, 1) }) .await { @@ -2354,9 +2354,10 @@ impl Tool for TypeTextTool { return error; } let text_w = text.clone(); - let result = - tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)) - .await; + let result = tokio::task::spawn_blocking(move || { + crate::wayland::inject_type_text(pid, xid, &text_w) + }) + .await; return match result { Ok(Ok(())) => ToolResult::text(format!( "Typed {text_len} character(s) (focus-free via cua-compositor)." @@ -2923,12 +2924,14 @@ impl Tool for PressKeyTool { } let result = if mods.is_empty() { let key_w = key.clone(); - tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w)) - .await + tokio::task::spawn_blocking(move || { + crate::wayland::inject_press_key(pid, xid, &key_w) + }) + .await } else { let mut chord = mods.clone(); chord.push(key.clone()); - tokio::task::spawn_blocking(move || crate::wayland::inject_hotkey(xid, &chord)) + tokio::task::spawn_blocking(move || crate::wayland::inject_hotkey(pid, xid, &chord)) .await }; return match result { @@ -3229,9 +3232,10 @@ impl Tool for HotkeyTool { } let mut chord = mods.clone(); chord.push(key.clone()); - let result = - tokio::task::spawn_blocking(move || crate::wayland::inject_hotkey(xid, &chord)) - .await; + let result = tokio::task::spawn_blocking(move || { + crate::wayland::inject_hotkey(pid, xid, &chord) + }) + .await; return match result { Ok(Ok(())) => ToolResult::text(format!( "Pressed hotkey '{key_display}' (focus-free via cua-compositor)." @@ -3633,7 +3637,7 @@ impl Tool for ScrollTool { }; let direction_for_inject = direction.clone(); let result = tokio::task::spawn_blocking(move || { - crate::wayland::inject_scroll(xid, x, y, &direction_for_inject, amount as u32) + crate::wayland::inject_scroll(pid, xid, x, y, &direction_for_inject, amount as u32) }) .await; return match result { @@ -3941,7 +3945,7 @@ impl Tool for DoubleClickTool { let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { if crate::wayland::is_inject_mode() { - return crate::wayland::inject_click(xid, lx, ly, 2, 1); + return crate::wayland::inject_click(pid, xid, lx, ly, 2, 1); } if crate::wayland::wayland_input_enabled() { let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); @@ -4029,7 +4033,7 @@ impl Tool for DoubleClickTool { let cursor_id_for_task = cursor_id.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { if crate::wayland::is_inject_mode() { - return crate::wayland::inject_click(xid, x, y, 2, 1); + return crate::wayland::inject_click(pid, xid, x, y, 2, 1); } if crate::wayland::wayland_input_enabled() { let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); @@ -4173,7 +4177,7 @@ impl Tool for RightClickTool { let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { if crate::wayland::is_inject_mode() { - return crate::wayland::inject_click(xid, lx, ly, 1, 3); + return crate::wayland::inject_click(pid, xid, lx, ly, 1, 3); } if crate::wayland::wayland_input_enabled() { let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); @@ -4261,7 +4265,7 @@ impl Tool for RightClickTool { let cursor_id_for_task = cursor_id.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { if crate::wayland::is_inject_mode() { - return crate::wayland::inject_click(xid, x, y, 1, 3); + return crate::wayland::inject_click(pid, xid, x, y, 1, 3); } if crate::wayland::wayland_input_enabled() { let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); @@ -4521,6 +4525,7 @@ impl Tool for DragTool { let drag_result = if crate::wayland::is_inject_mode() { tokio::task::spawn_blocking(move || { crate::wayland::inject_drag( + pid, xid, (from_x, from_y), (to_x, to_y), diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/page.rs index 401aa9dbee..27b71fac47 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/page.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/page.rs @@ -24,9 +24,9 @@ impl Default for LinuxPageBackend { #[async_trait] impl PageBackend for LinuxPageBackend { - async fn get_text(&self, pid: i32, window_id: u32) -> anyhow::Result { + async fn get_text(&self, pid: i32, window_id: u64) -> anyhow::Result { let pid_u = pid as u32; - let xid = window_id as u64; + let xid = window_id; let result = tokio::task::spawn_blocking(move || crate::atspi::walk_tree(pid_u, xid, None)) .await .map_err(|e| anyhow::anyhow!("AT-SPI walk task failed: {e}"))?; @@ -36,12 +36,12 @@ impl PageBackend for LinuxPageBackend { async fn query_dom( &self, pid: i32, - window_id: u32, + window_id: u64, css_selector: &str, _attributes: &[String], ) -> anyhow::Result { let pid_u = pid as u32; - let xid = window_id as u64; + let xid = window_id; let selector = css_selector.to_owned(); let result = tokio::task::spawn_blocking(move || crate::atspi::walk_tree(pid_u, xid, None)) .await @@ -105,7 +105,7 @@ impl PageBackend for LinuxPageBackend { async fn execute_javascript( &self, _pid: i32, - _window_id: u32, + _window_id: u64, javascript: &str, ) -> anyhow::Result { let port: u16 = match std::env::var("CUA_DRIVER_CDP_PORT") @@ -127,7 +127,7 @@ impl PageBackend for LinuxPageBackend { async fn execute_javascript_targeted( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, cdp_port: Option, target_url_contains: Option<&str>, diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs index cb040f39d1..6982b14fbc 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs @@ -2544,48 +2544,87 @@ fn no_app_id(window_id: u64) -> anyhow::Error { /// is the same credential the compositor observes on the owning wl_client. /// Fall back to app_id for clients whose accessibility metadata has no PID. pub fn inject_target_for_window(window_id: u64) -> anyhow::Result { - if let Some(pid) = crate::atspi::list_windows(None) - .into_iter() + inject_target_for_window_with_pid(window_id, None) +} + +fn inject_target_for_window_with_pid( + window_id: u64, + target_pid: Option, +) -> anyhow::Result { + if let Some(pid) = target_pid { + anyhow::ensure!(pid > 0, "cua-compositor target pid must be positive"); + // Electron/Chromium may create the xdg_toplevel from a renderer child + // rather than the public tool target. The compositor verifies the + // wl_client owner is this process or one of its descendants. + return Ok(format!("root:{pid}")); + } + let atspi = crate::atspi::list_windows(None); + let direct_pid = atspi + .iter() .find(|window| window.xid == window_id) - .and_then(|window| window.pid) - { + .and_then(|window| window.pid); + let correlated_pid = identity_for(window_id) + .as_ref() + .and_then(|identity| unique_atspi_pid_for_identity(identity, &atspi)); + if let Some(pid) = direct_pid.or(correlated_pid) { return Ok(format!("pid:{pid}")); } app_id_for_window(window_id).ok_or_else(|| no_app_id(window_id)) } +/// Correlate a connection-local native toplevel with its AT-SPI process. Exact +/// titles are the same bridge used by window enumeration; requiring one unique +/// PID prevents a shared toolkit app_id from silently selecting another app. +fn unique_atspi_pid_for_identity( + identity: &ToplevelIdentity, + windows: &[WindowInfo], +) -> Option { + if identity.title.is_empty() { + return None; + } + let mut pids = windows + .iter() + .filter(|window| window.title == identity.title) + .filter_map(|window| window.pid) + .collect::>(); + pids.sort_unstable(); + pids.dedup(); + (pids.len() == 1).then(|| pids[0]) +} + /// Focus-free type into the window's surface (no focus change). Rejects any /// character the compositor cannot emit before touching the socket. -pub fn inject_type_text(window_id: u64, text: &str) -> anyhow::Result<()> { +pub fn inject_type_text(target_pid: u32, window_id: u64, text: &str) -> anyhow::Result<()> { validate_injectable_text(text)?; - let app = inject_target_for_window(window_id)?; + let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; inject_send(&[format!("t {app} {}", to_hex(text))]) } /// Focus-free named-key press into the window's surface. Rejects any key /// outside the compositor's whitelist before touching the socket. -pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> { +pub fn inject_press_key(target_pid: u32, window_id: u64, key: &str) -> anyhow::Result<()> { validate_injectable_key(key)?; - let app = inject_target_for_window(window_id)?; + let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; inject_send(&[format!("k {app} {}", key.trim())]) } /// Focus-free modifier chord into the target surface. -pub fn inject_hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> { +pub fn inject_hotkey(target_pid: u32, window_id: u64, keys: &[String]) -> anyhow::Result<()> { let (modifiers, key) = validate_injectable_hotkey(keys)?; - let app = inject_target_for_window(window_id)?; + let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; inject_send(&[format!("h {app} {modifiers} {key}")]) } /// Focus-free wheel/axis input at one target-local point. pub fn inject_scroll( + target_pid: u32, window_id: u64, x: f64, y: f64, direction: &str, amount: u32, ) -> anyhow::Result<()> { - let app = inject_target_for_window(window_id)?; + let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; let (axis, value) = match direction.to_ascii_lowercase().as_str() { "up" => (0, -15.0), "down" | "page" => (0, 15.0), @@ -2600,8 +2639,15 @@ pub fn inject_scroll( /// Focus-free click into the window's surface via the nested cua-compositor. /// Coordinates are window-local, matching the rest of the inject protocol. -pub fn inject_click(window_id: u64, x: f64, y: f64, count: u32, button: u8) -> anyhow::Result<()> { - let app = inject_target_for_window(window_id)?; +pub fn inject_click( + target_pid: u32, + window_id: u64, + x: f64, + y: f64, + count: u32, + button: u8, +) -> anyhow::Result<()> { + let app = inject_target_for_window_with_pid(window_id, Some(target_pid))?; let btn = evdev_button(button as u32); let n = count.max(1); let mut lines = Vec::with_capacity((n as usize) * 4); @@ -2707,13 +2753,14 @@ pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> { /// Focus-free single drag using the same per-surface path as parallel drags. pub fn inject_drag( + target_pid: u32, window_id: u64, from: (f64, f64), to: (f64, f64), steps: usize, x_button: u32, ) -> anyhow::Result<()> { - let app_id = inject_target_for_window(window_id)?; + let app_id = inject_target_for_window_with_pid(window_id, Some(target_pid))?; inject_parallel_drags(&[InjectDrag { app_id, idx: 0, @@ -3143,6 +3190,44 @@ mod tests { ); } + #[test] + fn inject_target_correlates_native_identity_to_unique_atspi_pid() { + let identity = ToplevelIdentity { + title: "Unique sentinel".into(), + app_id: "electron".into(), + }; + let windows = vec![ + window(10, Some(100), "Background fixture"), + window(20, Some(200), "Unique sentinel"), + ]; + assert_eq!( + unique_atspi_pid_for_identity(&identity, &windows), + Some(200) + ); + } + + #[test] + fn inject_target_prefers_explicit_positive_pid() { + assert_eq!( + inject_target_for_window_with_pid(99, Some(123)).unwrap(), + "root:123" + ); + assert!(inject_target_for_window_with_pid(99, Some(0)).is_err()); + } + + #[test] + fn inject_target_refuses_ambiguous_title_pid_correlation() { + let identity = ToplevelIdentity { + title: "Shared title".into(), + app_id: "electron".into(), + }; + let windows = vec![ + window(10, Some(100), "Shared title"), + window(20, Some(200), "Shared title"), + ]; + assert_eq!(unique_atspi_pid_for_identity(&identity, &windows), None); + } + #[test] fn sway_window_capture_is_cropped_to_compositor_geometry() { let source = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel( diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs index e70b78fa14..db400972f7 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/page.rs @@ -41,7 +41,7 @@ impl MacOsPageBackend { #[async_trait] impl PageBackend for MacOsPageBackend { - async fn get_text(&self, pid: i32, window_id: u32) -> anyhow::Result { + async fn get_text(&self, pid: i32, window_id: u64) -> anyhow::Result { let bundle_id = Self::bundle_id_for(pid).await; let use_ax_fallback = !BrowserJs::supports(&bundle_id) @@ -63,7 +63,7 @@ impl PageBackend for MacOsPageBackend { async fn query_dom( &self, pid: i32, - window_id: u32, + window_id: u64, css_selector: &str, attributes: &[String], ) -> anyhow::Result { @@ -92,7 +92,7 @@ impl PageBackend for MacOsPageBackend { async fn execute_javascript( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, ) -> anyhow::Result { let bundle_id = Self::bundle_id_for(pid).await; @@ -102,7 +102,7 @@ impl PageBackend for MacOsPageBackend { async fn execute_javascript_targeted( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -120,7 +120,7 @@ impl PageBackend for MacOsPageBackend { async fn click_element( &self, pid: i32, - window_id: u32, + window_id: u64, selector: &str, ) -> anyhow::Result { let selector_js = json_string(selector); @@ -201,7 +201,7 @@ impl PageBackend for MacOsPageBackend { async fn insert_text( &self, pid: i32, - _window_id: u32, + _window_id: u64, text: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -220,7 +220,7 @@ impl PageBackend for MacOsPageBackend { async fn type_keystrokes( &self, pid: i32, - _window_id: u32, + _window_id: u64, text: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -263,8 +263,10 @@ async fn resolve_cdp_port(pid: i32, cdp_port: Option, action: &str) -> anyh } /// Route JavaScript execution to the appropriate backend. -async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u32) -> anyhow::Result { +async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u64) -> anyhow::Result { if BrowserJs::supports(bundle_id) { + let window_id = u32::try_from(window_id) + .map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?; return BrowserJs::execute(js, bundle_id, window_id).await; } let is_electron = tokio::task::spawn_blocking(move || ElectronJs::is_electron(pid)).await?; @@ -282,7 +284,9 @@ async fn execute_js(js: &str, bundle_id: &str, pid: i32, window_id: u32) -> anyh } /// Extract page text via the AX tree. -async fn ax_text_fallback(pid: i32, window_id: u32) -> anyhow::Result { +async fn ax_text_fallback(pid: i32, window_id: u64) -> anyhow::Result { + let window_id = u32::try_from(window_id) + .map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?; let result = tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None)) .await @@ -293,9 +297,11 @@ async fn ax_text_fallback(pid: i32, window_id: u32) -> anyhow::Result { /// Query AX tree by CSS selector. async fn ax_query_fallback( pid: i32, - window_id: u32, + window_id: u64, selector: &str, ) -> anyhow::Result> { + let window_id = u32::try_from(window_id) + .map_err(|_| anyhow::anyhow!("macOS window_id {window_id} is out of u32 range"))?; let sel = selector.to_owned(); let result = tokio::task::spawn_blocking(move || crate::ax::tree::walk_tree(pid, Some(window_id), None)) diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs index 7d184c7d77..bbb12e2667 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs @@ -60,8 +60,8 @@ impl Default for WindowsPageBackend { #[async_trait] impl PageBackend for WindowsPageBackend { - async fn get_text(&self, _pid: i32, window_id: u32) -> anyhow::Result { - let hwnd = window_id as u64; + async fn get_text(&self, _pid: i32, window_id: u64) -> anyhow::Result { + let hwnd = window_id; tokio::task::spawn_blocking(move || unsafe { get_text_blocking(hwnd) }) .await .map_err(|e| anyhow::anyhow!("join error: {e}"))? @@ -70,11 +70,11 @@ impl PageBackend for WindowsPageBackend { async fn query_dom( &self, _pid: i32, - window_id: u32, + window_id: u64, css_selector: &str, attributes: &[String], ) -> anyhow::Result { - let hwnd = window_id as u64; + let hwnd = window_id; let selector = css_selector.to_owned(); let attrs: Vec = attributes.to_vec(); tokio::task::spawn_blocking(move || unsafe { query_dom_blocking(hwnd, &selector, &attrs) }) @@ -85,7 +85,7 @@ impl PageBackend for WindowsPageBackend { async fn execute_javascript( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, ) -> anyhow::Result { // 1) Bookmark-based UIA exec — zero config, no launch flag needed. @@ -94,7 +94,15 @@ impl PageBackend for WindowsPageBackend { // sits there). Any failure (favorites bar hidden + Ctrl+Shift+B // fails to summon, dialog drift, title-poll timeout) is logged // and falls through to the CDP path. - match super::page_bookmark::try_bookmark_exec(pid, window_id, javascript).await { + let bookmark_result = match u32::try_from(window_id) { + Ok(window_id) => { + super::page_bookmark::try_bookmark_exec(pid, window_id, javascript).await + } + Err(_) => Err(anyhow::anyhow!( + "window_id {window_id} is outside the legacy bookmark transport's u32 range" + )), + }; + match bookmark_result { Ok(v) => { return Ok(format!("uia.bookmark_exec: {v}")); } @@ -131,7 +139,7 @@ impl PageBackend for WindowsPageBackend { async fn execute_javascript_targeted( &self, pid: i32, - window_id: u32, + window_id: u64, javascript: &str, cdp_port: Option, target_url_contains: Option<&str>, @@ -159,7 +167,7 @@ impl PageBackend for WindowsPageBackend { async fn click_element( &self, pid: i32, - window_id: u32, + window_id: u64, selector: &str, ) -> anyhow::Result { // Two-step: diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js index fa06203603..edeb4c4e49 100644 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js @@ -9,6 +9,7 @@ const http = require('http'); const path = require('path'); const sentinelMode = process.env.CUA_E2E_SENTINEL === '1'; const nativeWayland = process.platform === 'linux' && Boolean(process.env.WAYLAND_DISPLAY); +const customCuaCompositor = process.env.CUA_E2E_WAYLAND_SESSION === 'cua-compositor'; const fixtureJournalUrl = process.env.CUA_E2E_FIXTURE_JOURNAL_URL || ''; const sentinelJournalPath = process.env.CUA_E2E_SENTINEL_JOURNAL || ''; if (process.env.CUA_E2E_USER_DATA_DIR) { @@ -76,8 +77,11 @@ function createWindow() { title: fixedTitle, // Map the normal harness immediately. Xvfb/Openbox can enumerate a // deferred BrowserWindow while never painting it into the root desktop. - // The sentinel stays hidden until it has maximized and claimed focus. - show: !sentinelMode, + // The sentinel normally stays hidden until it has maximized and claimed + // focus. cua-compositor has no window-policy transition to perform, so map + // it at construction time; a synchronous show() after DOMContentLoaded can + // otherwise stall Chromium before the renderer paints or schedules timers. + show: !sentinelMode || customCuaCompositor, // A floating-level macOS window is omitted by cua-driver's deliberate // layer-0 top-level window contract. Foreground + maximized is sufficient // for occlusion there and lets an unexpected target raise remain visible. @@ -87,6 +91,9 @@ function createWindow() { nodeIntegration: false, contextIsolation: true, sandbox: !sentinelMode, + // The sentinel's heartbeat is an E2E oracle. It must keep ticking while + // the focus-loss canary deliberately places the window in the background. + backgroundThrottling: !sentinelMode, preload: path.join(__dirname, 'preload.js'), }, }); @@ -125,13 +132,22 @@ function createWindow() { if (process.platform !== 'darwin' && !nativeWayland) { mainWindow.setAlwaysOnTop(true); } - if (process.platform === 'linux' && process.env.WAYLAND_DISPLAY) { + if (nativeWayland && !customCuaCompositor) { mainWindow.setFullScreen(true); - } else { + } else if (!customCuaCompositor) { mainWindow.maximize(); } - mainWindow.show(); - mainWindow.focus(); + // The minimal nested cua-compositor intentionally has no fullscreen + // policy implementation. Requesting fullscreen leaves Chromium + // waiting on a configure transition and stops the heartbeat oracle. + // Its 1280x900 sentinel already covers the smaller fixture at origin. + // cua-compositor mapped and focused this toplevel at construction + // time. Avoid a second synchronous show/configure/focus transition + // after the renderer has emitted its ready event. + if (!customCuaCompositor) { + mainWindow.show(); + mainWindow.focus(); + } } else { // Xvfb/Openbox can keep a showInactive window inspectable through // AT-SPI while never mapping it onto the captured root desktop. diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml index 3f47176450..73b9780a86 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml +++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml @@ -81,6 +81,54 @@ + + + + + + + + + + + + + + + + + @@ -165,51 +213,6 @@ - - - - - - - - - - - - - - - - - diff --git a/nix/cua-driver/compositor/cua_compositor_patch.py b/nix/cua-driver/compositor/cua_compositor_patch.py index 7283f44c46..cf8e6733e2 100644 --- a/nix/cua-driver/compositor/cua_compositor_patch.py +++ b/nix/cua-driver/compositor/cua_compositor_patch.py @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,7 @@ static struct cua_devstate cua_kbd_state[CUA_MAXDEV]; static int g_keymap_fd = -1; static size_t g_keymap_size = 0; +static struct wlr_keyboard g_keyboard; static xkb_mod_mask_t g_shift_mask = 1; static xkb_mod_mask_t g_ctrl_mask = 0; static xkb_mod_mask_t g_alt_mask = 0; @@ -63,13 +65,18 @@ struct cua_keyent { uint32_t keycode; int shift; int valid; }; static struct cua_keyent g_chartab[128]; static struct wlr_foreign_toplevel_manager_v1 *g_ftl_mgr = NULL; +struct tinywl_toplevel; static void cua_ftl_request_activate(struct wl_listener *listener, void *data); +static void cua_maybe_focus_new_toplevel(struct tinywl_toplevel *toplevel); +static pid_t cua_toplevel_pid(struct tinywl_toplevel *t); +static bool cua_pid_in_family(pid_t pid, pid_t root_pid); """ # A foreign-toplevel handle pointer on each toplevel (for list_windows). STRUCT_FIELD = ( "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n" + "\tbool cua_initial_activation_sent;\n" "\tstruct wlr_foreign_toplevel_handle_v1 *ftl;\n" "\tstruct wl_listener ftl_request_activate;\n" ) @@ -78,10 +85,46 @@ /* v1 control-protocol banner: the client sends this line, the compositor echoes * it to confirm both speak v1. Any other first line is refused. */ #define CUA_PROTO_HELLO "cua-inject v1" +static void cua_focus_toplevel(struct tinywl_toplevel *toplevel) { + focus_toplevel(toplevel); + /* tinywl only notifies seat keyboard focus when a physical wlr_keyboard is + * attached. Headless CI has none, so explicit activation establishes the + * logical focus without coupling it to the initial map/configure handshake. */ + struct wlr_surface *surface = toplevel->xdg_toplevel->base->surface; + if (toplevel->server->seat->keyboard_state.focused_surface != surface) { + struct wlr_keyboard_modifiers modifiers = {0}; + wlr_seat_keyboard_notify_enter( + toplevel->server->seat, surface, NULL, 0, &modifiers); + } +} +/* New child toplevels may request focus as part of their normal map sequence. + * Preserve that behavior only when the current keyboard focus belongs to the + * same process family. A background application opening a child must not steal + * focus from the foreground sentinel (or from any other application). */ +static void cua_maybe_focus_new_toplevel(struct tinywl_toplevel *toplevel) { + struct wlr_surface *focused = toplevel->server->seat->keyboard_state.focused_surface; + struct wlr_surface *focused_root = focused ? wlr_surface_get_root_surface(focused) : NULL; + if (!focused_root) { + cua_focus_toplevel(toplevel); + return; + } + struct tinywl_toplevel *current; + wl_list_for_each(current, &toplevel->server->toplevels, link) { + struct wlr_surface *surface = current->xdg_toplevel ? current->xdg_toplevel->base->surface : NULL; + if (surface != focused_root) continue; + pid_t requested_pid = cua_toplevel_pid(toplevel); + pid_t focused_pid = cua_toplevel_pid(current); + if (requested_pid == focused_pid || + cua_pid_in_family(requested_pid, focused_pid) || + cua_pid_in_family(focused_pid, requested_pid)) + cua_focus_toplevel(toplevel); + return; + } +} static void cua_ftl_request_activate(struct wl_listener *listener, void *data) { (void)data; struct tinywl_toplevel *t = wl_container_of(listener, t, ftl_request_activate); - focus_toplevel(t); + cua_focus_toplevel(t); } static uint32_t cua_now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); @@ -105,7 +148,28 @@ if (wl_resource_get_version(res) >= WL_POINTER_FRAME_SINCE_VERSION) wl_pointer_send_frame(res); } -static pid_t cua_toplevel_pid(struct tinywl_toplevel *t); +static bool cua_pid_in_family(pid_t pid, pid_t root_pid) { + if (pid <= 0 || root_pid <= 0) return false; + for (int depth = 0; depth < 64 && pid > 1; depth++) { + if (pid == root_pid) return true; + char path[64], stat[4096]; + snprintf(path, sizeof path, "/proc/%d/stat", (int)pid); + FILE *file = fopen(path, "r"); + if (!file) return false; + size_t n = fread(stat, 1, sizeof stat - 1, file); + fclose(file); + if (!n) return false; + stat[n] = '\0'; + /* comm is parenthesized and may contain spaces or ')', so parse the + * state + parent pid after its final closing parenthesis. */ + char *close = strrchr(stat, ')'), state = '\0'; + long parent = 0; + if (!close || sscanf(close + 2, "%c %ld", &state, &parent) != 2 || + parent <= 0 || parent == pid) return false; + pid = (pid_t)parent; + } + return pid == root_pid; +} /* Resolve a target window by its xdg app_id, refusing missing and ambiguous * matches so a command never silently drives the wrong window. In v1 duplicate * app_ids are simply not addressable. On failure returns NULL and points *err @@ -113,6 +177,17 @@ static struct tinywl_toplevel *cua_resolve_target(struct tinywl_server *server, const char *app_id, const char **err) { struct tinywl_toplevel *t, *found = NULL; int matches = 0; + if (!strncmp(app_id, "root:", 5)) { + char *end = NULL; + long pid = strtol(app_id + 5, &end, 10); + if (pid <= 0 || !end || *end) { *err = "bad-root-pid"; return NULL; } + wl_list_for_each(t, &server->toplevels, link) { + if (cua_pid_in_family(cua_toplevel_pid(t), (pid_t)pid)) { found = t; matches++; } + } + if (matches == 0) { *err = "unknown-root-pid"; return NULL; } + if (matches > 1) { *err = "ambiguous-root-pid"; return NULL; } + return found; + } if (!strncmp(app_id, "pid:", 4)) { char *end = NULL; long pid = strtol(app_id + 4, &end, 10); @@ -152,15 +227,7 @@ } if (matches == 0) return "unknown-pid"; if (matches > 1) return "ambiguous-pid"; - focus_toplevel(found); - /* tinywl only notifies seat keyboard focus when a physical wlr_keyboard is - * attached. Headless CI has none, so establish the logical focus explicitly - * for observer truth and client activation semantics. */ - struct wlr_surface *surface = found->xdg_toplevel->base->surface; - if (server->seat->keyboard_state.focused_surface != surface) { - struct wlr_keyboard_modifiers modifiers = {0}; - wlr_seat_keyboard_notify_enter(server->seat, surface, NULL, 0, &modifiers); - } + cua_focus_toplevel(found); return NULL; } /* Independent observer query used only by the Rust E2E testkit. The target is @@ -186,7 +253,10 @@ struct tinywl_toplevel *t, *target = NULL; int matches = 0; wl_list_for_each(t, &server->toplevels, link) { - if (cua_toplevel_pid(t) == target_pid) { target = t; matches++; } + /* Electron exposes accessibility under the browser root while its + * xdg_toplevel can be owned by a descendant GPU/renderer client. Match + * the same bounded process family accepted by root: injection. */ + if (cua_pid_in_family(cua_toplevel_pid(t), target_pid)) { target = t; matches++; } } if (!matches) return "target-not-found"; if (matches > 1) return "ambiguous-pid"; @@ -242,6 +312,22 @@ sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); if (!sc || wl_list_empty(&sc->pointers)) return false; } + /* Chromium consumes pointer input through wlroots' seat pointer state. Raw + * wl_pointer resource sends are sufficient for GTK, but Chromium can ACK + * them without dispatching DOM mouse events because the compositor-side + * focus/grab state was never updated. Device 0 is the normal single-pointer + * route, so use the protocol-complete seat notifications there. Higher + * logical device indices retain direct delivery for independent cursors. */ + if (idx == 0) { + wlr_seat_pointer_notify_enter(server->seat, surface, local_x, local_y); + wlr_seat_pointer_notify_motion(server->seat, cua_now_ms(), local_x, local_y); + /* Real cursors emit a separate frame event after the motion callback. + * Synthetic commands have no cursor-frame signal, so terminate the + * protocol batch here; Chromium buffers motion/button events until it. */ + wlr_seat_pointer_notify_frame(server->seat); + cua_ptr[idx].entered = surface; + return true; + } wl_fixed_t sx = wl_fixed_from_double(local_x), sy = wl_fixed_from_double(local_y); struct wl_resource *res; if (cua_ptr[idx].entered != surface) { @@ -263,21 +349,14 @@ if (!t || !surface) return NULL; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); if (!sc || wl_list_empty(&sc->pointers)) return NULL; - struct wl_resource *res; - if (cua_ptr[0].entered != surface) { - if (cua_ptr[0].entered) cua_ptr_leave(server->seat, cua_ptr[0].entered); - uint32_t serial = wlr_seat_client_next_serial(sc); - wl_resource_for_each(res, &sc->pointers) { - wl_pointer_send_enter(res, serial, surface->resource, wl_fixed_from_double(sx), wl_fixed_from_double(sy)); - cua_pframe(res); - } - cua_ptr[0].entered = surface; - } - uint32_t tm = cua_now_ms(); - wl_resource_for_each(res, &sc->pointers) { - wl_pointer_send_motion(res, tm, wl_fixed_from_double(sx), wl_fixed_from_double(sy)); - cua_pframe(res); - } + /* Desktop scope also uses logical device 0. Keep wlroots' seat pointer + * focus in sync before cua_button() sends its protocol-complete seat + * notification; raw resource enters here would leave the seat targeting a stale + * surface, so the following button notification never reaches this point. */ + wlr_seat_pointer_notify_enter(server->seat, surface, sx, sy); + wlr_seat_pointer_notify_motion(server->seat, cua_now_ms(), sx, sy); + wlr_seat_pointer_notify_frame(server->seat); + cua_ptr[0].entered = surface; return t; } static bool cua_button(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t button, bool pressed) { @@ -287,6 +366,14 @@ struct wlr_surface *surface = cua_ptr[idx].entered ? cua_ptr[idx].entered : t->xdg_toplevel->base->surface; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); if (!sc || wl_list_empty(&sc->pointers)) return false; + if (idx == 0) { + wlr_seat_pointer_notify_button(server->seat, cua_now_ms(), button, + pressed ? WLR_BUTTON_PRESSED : WLR_BUTTON_RELEASED); + /* See cua_motion: there is no hardware cursor-frame callback for the + * virtual device, so each injected command must close its own batch. */ + wlr_seat_pointer_notify_frame(server->seat); + return true; + } uint32_t tm = cua_now_ms(), bs = wlr_seat_client_next_serial(sc); struct wl_resource *res; wl_resource_for_each(res, &sc->pointers) { @@ -315,7 +402,7 @@ } return true; } -static void cua_init_keymap(void) { +static void cua_init_keymap(struct tinywl_server *server) { struct xkb_context *ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS); struct xkb_rule_names names = {0}; struct xkb_keymap *km = xkb_keymap_new_from_names(ctx, &names, XKB_KEYMAP_COMPILE_NO_FLAGS); @@ -354,6 +441,13 @@ /* Control characters used by the protocol map to dedicated keys. */ g_chartab['\n'] = (struct cua_keyent){ KEY_ENTER, 0, 1 }; g_chartab['\t'] = (struct cua_keyent){ KEY_TAB, 0, 1 }; + /* A keyboard-capable seat must attach a real wlr_keyboard before clients + * bind. wlroots then sends keymap + repeat-info before any enter event; + * Chromium can stall if it receives an enter from a device-less seat. */ + wlr_keyboard_init(&g_keyboard, NULL, "cua-virtual-keyboard"); + wlr_keyboard_set_keymap(&g_keyboard, km); + wlr_keyboard_set_repeat_info(&g_keyboard, 25, 600); + wlr_seat_set_keyboard(server->seat, &g_keyboard); xkb_keymap_unref(km); xkb_context_unref(ctx); wlr_log(WLR_INFO, "[cua] xkb keymap + chartab ready (%zu bytes)", g_keymap_size); } @@ -363,6 +457,13 @@ struct wlr_surface *surface = t->xdg_toplevel->base->surface; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); if (!sc || wl_list_empty(&sc->keyboards)) return NULL; + struct wlr_surface *focused = server->seat->keyboard_state.focused_surface; + if (focused && wlr_surface_get_root_surface(focused) == surface) { + /* Foreground delivery already has a protocol-complete seat enter. The + * key path below uses wlr_seat_keyboard_notify_key for this target. */ + cua_kbd_state[0].entered = surface; + return sc; + } if (cua_kbd_state[0].entered != surface) { struct wl_resource *res; struct wl_array keys; wl_array_init(&keys); wl_resource_for_each(res, &sc->keyboards) { @@ -387,14 +488,31 @@ wl_keyboard_send_key(res, wlr_seat_client_next_serial(sc), tm, keycode, pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED); } +/* Preserve compositor-native keyboard delivery when the addressed surface is + * already the logical seat focus. Chromium relies on wlroots' focused-seat + * state for foreground keyboard events; sending a raw wl_keyboard.key to its + * resource can be acknowledged while never reaching the renderer. Background + * targets retain the direct resource path that makes focus-free input possible. */ +static void cua_kbd_key_target(struct tinywl_server *server, struct tinywl_toplevel *t, + struct wlr_seat_client *sc, uint32_t keycode, bool pressed) { + struct wlr_surface *target = wlr_surface_get_root_surface(t->xdg_toplevel->base->surface); + struct wlr_surface *focused = server->seat->keyboard_state.focused_surface; + struct wlr_surface *focused_root = focused ? wlr_surface_get_root_surface(focused) : NULL; + if (target == focused_root) { + wlr_seat_keyboard_notify_key(server->seat, cua_now_ms(), keycode, + pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED); + } else { + cua_kbd_key(sc, keycode, pressed); + } +} static bool cua_type_cp(struct tinywl_server *server, struct tinywl_toplevel *t, uint32_t cp) { if (cp >= 128 || !g_chartab[cp].valid) return false; struct wlr_seat_client *sc = cua_kbd_enter(server, t); if (!sc) return false; struct cua_keyent e = g_chartab[cp]; if (e.shift) cua_kbd_mods(sc, g_shift_mask); - cua_kbd_key(sc, e.keycode, true); - cua_kbd_key(sc, e.keycode, false); + cua_kbd_key_target(server, t, sc, e.keycode, true); + cua_kbd_key_target(server, t, sc, e.keycode, false); if (e.shift) cua_kbd_mods(sc, 0); return true; } @@ -435,8 +553,8 @@ if (!kc) return 0; struct wlr_seat_client *sc = cua_kbd_enter(server, t); if (!sc) return -1; - cua_kbd_key(sc, kc, true); - cua_kbd_key(sc, kc, false); + cua_kbd_key_target(server, t, sc, kc, true); + cua_kbd_key_target(server, t, sc, kc, false); return 1; } static int cua_hotkey(struct tinywl_server *server, struct tinywl_toplevel *t, const char *mods, const char *key) { @@ -460,8 +578,8 @@ struct wlr_seat_client *sc = cua_kbd_enter(server, t); if (!sc) return -1; cua_kbd_mods(sc, mask); - cua_kbd_key(sc, kc, true); - cua_kbd_key(sc, kc, false); + cua_kbd_key_target(server, t, sc, kc, true); + cua_kbd_key_target(server, t, sc, kc, false); cua_kbd_mods(sc, 0); return 1; } @@ -606,6 +724,7 @@ return 0; } static void cua_setup_control_socket(struct tinywl_server *server) { + cua_init_keymap(server); const char *path = getenv("CUA_INJECT_SOCKET"); if (!path) { wlr_log(WLR_INFO, "[cua] no CUA_INJECT_SOCKET; injection disabled"); return; } int fd = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); @@ -616,7 +735,6 @@ if (bind(fd, (struct sockaddr *)&addr, sizeof addr) < 0 || listen(fd, 4) < 0) { wlr_log(WLR_ERROR, "[cua] bind/listen %s: %s", path, strerror(errno)); close(fd); return; } - cua_init_keymap(); wl_event_loop_add_fd(wl_display_get_event_loop(server->wl_display), fd, WL_EVENT_READABLE, cua_listen_cb, server); wlr_log(WLR_INFO, "[cua] injection control socket on %s", path); @@ -632,6 +750,44 @@ def repl(s, old, new, label, count=1): # 1) includes + globals + a foreign-toplevel handle field on the toplevel. src = repl(src, "struct tinywl_server {", INCLUDES + "struct tinywl_server {", "includes") +# tinywl's desktop-oriented focus helper toggles xdg_toplevel activation on +# every focus transition. Chromium needs one coherent transition (deactivate +# the old surface, activate the new one) to finish renderer startup, but can +# stop scheduling frames after later toggles in this minimal headless +# compositor. Send that startup transition once per toplevel; seat focus plus +# scene stacking are authoritative after that. +src = repl(src, + "\tif (prev_surface) {\n" + "\t\t/*\n" + "\t\t * Deactivate the previously focused surface. This lets the client know\n" + "\t\t * it no longer has focus and the client will repaint accordingly, e.g.\n" + "\t\t * stop displaying a caret.\n" + "\t\t */\n" + "\t\tstruct wlr_xdg_toplevel *prev_toplevel =\n" + "\t\t\twlr_xdg_toplevel_try_from_wlr_surface(prev_surface);\n" + "\t\tif (prev_toplevel != NULL) {\n" + "\t\t\twlr_xdg_toplevel_set_activated(prev_toplevel, false);\n" + "\t\t}\n" + "\t}\n", + "\tif (!toplevel->cua_initial_activation_sent && prev_surface) {\n" + "\t\tstruct wlr_xdg_toplevel *prev_toplevel =\n" + "\t\t\twlr_xdg_toplevel_try_from_wlr_surface(prev_surface);\n" + "\t\tif (prev_toplevel != NULL) {\n" + "\t\t\twlr_xdg_toplevel_set_activated(prev_toplevel, false);\n" + "\t\t}\n" + "\t}\n", + "headless-startup-deactivation") +src = repl(src, + "\t/* Activate the new surface */\n" + "\twlr_xdg_toplevel_set_activated(toplevel->xdg_toplevel, true);\n", + "\t/* Chromium needs one activated configure to complete renderer startup.\n" + "\t * Later focus changes use the seat and scene only, avoiding activation\n" + "\t * toggles that can stall it in this private headless compositor. */\n" + "\tif (!toplevel->cua_initial_activation_sent) {\n" + "\t\twlr_xdg_toplevel_set_activated(toplevel->xdg_toplevel, true);\n" + "\t\ttoplevel->cua_initial_activation_sent = true;\n" + "\t}\n", + "headless-initial-activation") src = repl(src, "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n", STRUCT_FIELD, "ftl-field") @@ -651,7 +807,7 @@ def repl(s, old, new, label, count=1): "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n" "\t\ttoplevel->ftl_request_activate.notify = cua_ftl_request_activate;\n" "\t\twl_signal_add(&toplevel->ftl->events.request_activate, &toplevel->ftl_request_activate);\n" - "\t}\n\n\tfocus_toplevel(toplevel);", + "\t}\n\n\tcua_maybe_focus_new_toplevel(toplevel);", "ftl-on-map") # 4) On unmap: drop the foreign-toplevel handle. @@ -705,6 +861,15 @@ def repl(s, old, new, label, count=1): "\twl_display_run(server.wl_display);", "\tcua_setup_control_socket(&server);\n\twl_display_run(server.wl_display);", "control-socket-setup") +src = repl(src, + "\twlr_backend_destroy(server.backend);\n" + "\twl_display_destroy(server.wl_display);\n" + "\treturn 0;", + "\twlr_backend_destroy(server.backend);\n" + "\twlr_keyboard_finish(&g_keyboard);\n" + "\twl_display_destroy(server.wl_display);\n" + "\treturn 0;", + "virtual-keyboard-cleanup") io.open(out, "w", encoding="utf-8").write(src) sys.stderr.write("cua-compositor.c written (%d bytes)\n" % len(src)) diff --git a/nix/cua-driver/tests/policy-rego.nix b/nix/cua-driver/tests/policy-rego.nix index cd2331247d..7cf4288a83 100644 --- a/nix/cua-driver/tests/policy-rego.nix +++ b/nix/cua-driver/tests/policy-rego.nix @@ -61,10 +61,10 @@ let jq -e '.id == 2 and .error == null and .result != null' <<<"$response" >/dev/null request '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"shell_execute","arguments":{}}}' - jq -e '.id == 3 and (.error.message | startswith("Permission denied:"))' <<<"$response" >/dev/null + jq -e '.id == 3 and .error == null and .result.isError == true and (.result.content[0].text | startswith("Permission denied:"))' <<<"$response" >/dev/null request '{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"click","arguments":{"x":2000,"y":100}}}' - jq -e '.id == 4 and (.error.message | startswith("Permission denied:"))' <<<"$response" >/dev/null + jq -e '.id == 4 and .error == null and .result.isError == true and (.result.content[0].text | startswith("Permission denied:"))' <<<"$response" >/dev/null ''; in pkgs.testers.runNixOSTest { diff --git a/nix/cua-driver/tests/policy-yaml.nix b/nix/cua-driver/tests/policy-yaml.nix index 9c45543764..65e9fe1120 100644 --- a/nix/cua-driver/tests/policy-yaml.nix +++ b/nix/cua-driver/tests/policy-yaml.nix @@ -55,10 +55,10 @@ let jq -e '.id == 2 and .error == null and .result != null' <<<"$response" >/dev/null request '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"shell_execute","arguments":{}}}' - jq -e '.id == 3 and (.error.message | startswith("Permission denied:"))' <<<"$response" >/dev/null + jq -e '.id == 3 and .error == null and .result.isError == true and (.result.content[0].text | startswith("Permission denied:"))' <<<"$response" >/dev/null request '{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"click","arguments":{"x":1921,"y":100}}}' - jq -e '.id == 4 and (.error.message | startswith("Permission denied:"))' <<<"$response" >/dev/null + jq -e '.id == 4 and .error == null and .result.isError == true and (.result.content[0].text | startswith("Permission denied:"))' <<<"$response" >/dev/null ''; in pkgs.testers.runNixOSTest {