diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 751f8f6e3d..5981f4d9df 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -265,6 +265,18 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -680,6 +692,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "evdev" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6055a93a963297befb0f4f6e18f314aec9767a4bbe88b151126df2433610a7" +dependencies = [ + "bitvec", + "cfg-if", + "libc", + "nix", + "thiserror", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -824,6 +849,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-core" version = "0.3.32" @@ -1222,6 +1253,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -1279,6 +1319,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1544,6 +1597,7 @@ dependencies = [ "base64", "cua-driver-core", "cursor-overlay", + "evdev", "image", "libc", "pip-preview", @@ -1553,6 +1607,7 @@ dependencies = [ "tiny-skia", "tokio", "tracing", + "x11", "x11rb", ] @@ -1733,6 +1788,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.6" @@ -2177,6 +2238,12 @@ dependencies = [ "syn", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.45" @@ -2536,7 +2603,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ - "memoffset", + "memoffset 0.9.1", "tempfile", "windows-sys 0.61.2", ] @@ -3312,6 +3379,25 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x11rb" version = "0.13.2" diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs index ae2f4cf3e9..7d395b8e88 100644 --- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs +++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs @@ -283,8 +283,12 @@ pub enum OverlayMsg { pub enum OverlayCommand { /// Animate the cursor to a new screen position. MoveTo { x: f64, y: f64, end_heading_radians: f64 }, + /// Snap the cursor immediately to a screen position, optionally updating heading. + SnapTo { x: f64, y: f64, heading_radians: Option }, /// Start the click-press visual. ClickPulse { x: f64, y: f64 }, + /// Toggle the held-button visual state. + SetPressed(bool), /// Show or hide the overlay. SetEnabled(bool), /// Update the motion/timing config live. diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs index 3d3579a054..34a07183a2 100644 --- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs +++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs @@ -9,7 +9,7 @@ //! //! - [`RenderStateCore`] — the platform-agnostic animation fields //! (`cfg`, `palette`, `motion`, `pos`, `heading`, `path`, `dist`, `spring`, -//! `spring_tgt`, `click_t`, `shape`, `visible`, `idle_secs`, `idle_alpha`, +//! `spring_tgt`, `click_t`, `pressed`, `shape`, `visible`, `idle_secs`, `idle_alpha`, //! `pinned_wid`, `gradient_colors`, `bloom_override`). //! - [`RenderStateCore::tick_motion`] — speed-profile + spring physics + //! click-pulse + idle-fade using runtime [`MotionConfig`] (Windows + Linux). @@ -68,6 +68,8 @@ pub struct RenderStateCore { pub spring_tgt: Option<(f64, f64, f64)>, /// Click-pulse phase 0..1; `None` = no pulse in flight. pub click_t: Option, + /// Whether a button is currently being held for this cursor. + pub pressed: bool, /// Custom cursor shape; `None` = built-in gradient arrow. pub shape: Option, /// User-controlled visibility. @@ -108,6 +110,7 @@ impl RenderStateCore { spring: None, spring_tgt: None, click_t: None, + pressed: false, visible: true, idle_secs: 0.0, idle_alpha: 1.0, @@ -417,6 +420,23 @@ impl RenderStateCore { self.idle_alpha = 1.0; true } + OverlayCommand::SnapTo { + x, + y, + heading_radians, + } => { + self.pos = (x, y); + if let Some(heading) = heading_radians { + self.heading = heading; + } + self.path = None; + self.dist = 0.0; + self.spring = None; + self.spring_tgt = None; + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } OverlayCommand::ClickPulse { x, y } => { if click_pulse_sentinel_only { // macOS: only snap position on first placement (sentinel state). @@ -438,6 +458,12 @@ impl RenderStateCore { self.idle_alpha = 1.0; true } + OverlayCommand::SetPressed(v) => { + self.pressed = v; + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } OverlayCommand::SetEnabled(v) => { self.visible = v; true @@ -534,7 +560,7 @@ pub fn paint_cursor( let alpha_scale = core.idle_alpha as f32; // --- Bloom (radial gradient behind the arrow) --- - let bloom_r: f32 = 22.0; + let bloom_r: f32 = if core.pressed { 34.0 } else { 22.0 }; // Use runtime bloom_override if set, otherwise fall back to palette. let (br, bg, bb) = if let Some([r, g, b, _]) = core.bloom_override { (r, g, b) @@ -580,6 +606,46 @@ pub fn paint_cursor( pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); } + if core.pressed { + let [pr, pg, pb, _] = core.palette.cursor_mid; + let ring_color = + tiny_skia::Color::from_rgba8(pr, pg, pb, (210.0 * alpha_scale) as u8); + let mut ring_paint = tiny_skia::Paint::default(); + ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); + ring_paint.anti_alias = true; + let stroke = tiny_skia::Stroke { + width: 3.0, + ..Default::default() + }; + let core_fill = + tiny_skia::Color::from_rgba8(pr, pg, pb, (110.0 * alpha_scale) as u8); + let mut fill_paint = tiny_skia::Paint::default(); + fill_paint.shader = tiny_skia::Shader::SolidColor(core_fill); + fill_paint.anti_alias = true; + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_circle(px as f32, py as f32, 6.5); + if let Some(path) = pb.finish() { + pm.fill_path( + &path, + &fill_paint, + tiny_skia::FillRule::Winding, + tiny_skia::Transform::identity(), + None, + ); + } + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_circle(px as f32, py as f32, 13.0); + if let Some(path) = pb.finish() { + pm.stroke_path( + &path, + &ring_paint, + &stroke, + tiny_skia::Transform::identity(), + None, + ); + } + } + // --- Focus rect highlight (macOS only — others pass None) --- // Cyan glow border + faint fill, matching Swift AgentCursor.showFocusRect. if let Some(fr) = focus_rect { diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml index 1d7cdfa48f..500e96b058 100644 --- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml +++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml @@ -20,6 +20,7 @@ tiny-skia = { version = "0.11", default-features = false, features = ["std"] } [target.'cfg(target_os = "linux")'.dependencies] # X11 background input + window enumeration x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape", "xtest"] } +x11 = { version = "2.21", features = ["xlib", "xinput", "xtest"] } base64 = { workspace = true } image = { workspace = true } # kill(2) for the kill_app tool — SIGKILL via libc::kill. @@ -29,3 +30,4 @@ libc = "0.2" # typelibs at runtime. `tokio` matches the driver's async runtime; `zbus` # re-exports the bus types we need (fdo::DBusProxy for pid resolution). atspi = { version = "0.30", features = ["tokio", "zbus"] } +evdev = "0.12" diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs index e9203d42d0..5de3399aed 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs @@ -11,9 +11,17 @@ //! them, because XTest delivers to the *focused* window and would break the //! no-focus-steal contract. -use anyhow::Result; +use anyhow::{anyhow, bail, Result}; +use std::collections::HashMap; +use std::ffi::{CStr, CString}; +use std::fs; +use std::ptr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::sleep; use std::time::Duration; +use evdev::uinput::VirtualDevice; +use evdev::{AttributeSet, EventType, InputEvent, Key, RelativeAxisType}; use x11rb::connection::Connection; use x11rb::protocol::xproto::*; use x11rb::rust_connection::RustConnection; @@ -21,6 +29,996 @@ use x11rb::rust_connection::RustConnection; const CLICK_DELAY_MS: u64 = 35; const KEY_DELAY_MS: u64 = 10; +#[derive(Clone, Copy, Debug)] +pub struct VirtualPointerDrag { + pub target_window: u64, + pub button: u8, + pub from_x: i32, + pub from_y: i32, + pub to_x: i32, + pub to_y: i32, + pub duration_ms: u64, + pub steps: usize, +} + +#[derive(Clone, Copy, Debug)] +struct MasterPointerIds { + pointer_id: i32, + keyboard_id: i32, + slave_pointer_id: i32, +} + +static MPX_POINTERS: OnceLock>> = OnceLock::new(); +static UINPUT_POINTERS: OnceLock>>>> = OnceLock::new(); +static XLIB_THREADS_READY: OnceLock> = OnceLock::new(); +static MPX_NAME_COUNTER: AtomicU64 = AtomicU64::new(1); + +fn mpx_pointers() -> &'static Mutex> { + MPX_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn uinput_pointers() -> &'static Mutex>>> { + UINPUT_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn master_pointer_name(cursor_id: &str) -> String { + let nonce = MPX_NAME_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("CUA {cursor_id} mp-{}-{nonce}", std::process::id()) +} + +fn slave_pointer_name(master_name: &str) -> String { + format!("{master_name} uinput pointer") +} + +fn master_pointer_device_name(master_name: &str) -> String { + format!("{master_name} pointer") +} + +fn master_keyboard_device_name(master_name: &str) -> String { + format!("{master_name} keyboard") +} + +fn open_display() -> Result<*mut x11::xlib::Display> { + match XLIB_THREADS_READY.get_or_init(|| { + let rc = unsafe { x11::xlib::XInitThreads() }; + if rc == 0 { + Err("XInitThreads failed".to_owned()) + } else { + Ok(()) + } + }) { + Ok(()) => {} + Err(err) => bail!("{err}"), + } + let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) }; + if display.is_null() { + bail!("XOpenDisplay returned null"); + } + Ok(display) +} + +fn xi2_query_devices( + display: *mut x11::xlib::Display, +) -> Result> { + let mut count = 0; + let ptr = unsafe { x11::xinput2::XIQueryDevice(display, x11::xinput2::XIAllDevices, &mut count) }; + if ptr.is_null() { + bail!("XIQueryDevice returned null"); + } + let mut out = Vec::new(); + for i in 0..count { + let info = unsafe { *ptr.add(i as usize) }; + let name = if info.name.is_null() { + String::new() + } else { + unsafe { CStr::from_ptr(info.name) }.to_string_lossy().into_owned() + }; + out.push((info.deviceid, info._use, name)); + } + unsafe { x11::xinput2::XIFreeDeviceInfo(ptr) }; + Ok(out) +} + +fn x_server_vendor(display: *mut x11::xlib::Display) -> String { + let ptr = unsafe { x11::xlib::XServerVendor(display) }; + if ptr.is_null() { + return String::new(); + } + unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned() +} + +fn supports_parallel_pointer_injection(display: *mut x11::xlib::Display) -> Result<()> { + let vendor = x_server_vendor(display); + if vendor.to_ascii_lowercase().contains("tigervnc") { + bail!( + "parallel_mouse_drag is not supported on this X server ('{vendor}'). \ + Xtigervnc exposes only its built-in VNC/XTEST devices, so Linux uinput/libinput \ + pointers cannot become real X input devices here." + ); + } + if is_xtigervnc_process_running() { + let display_name = std::env::var("DISPLAY").unwrap_or_else(|_| "".to_owned()); + bail!( + "parallel_mouse_drag is not supported on display {display_name} because the active X server is Xtigervnc. \ + Xtigervnc exposes only its built-in VNC/XTEST devices, so Linux uinput/libinput pointers \ + cannot become real X input devices in this environment." + ); + } + Ok(()) +} + +fn is_xtigervnc_process_running() -> bool { + let display_name = std::env::var("DISPLAY").ok(); + let Ok(proc_entries) = fs::read_dir("/proc") else { + return false; + }; + for entry in proc_entries.flatten() { + let file_name = entry.file_name(); + let Some(pid) = file_name.to_str() else { + continue; + }; + if !pid.bytes().all(|b| b.is_ascii_digit()) { + continue; + } + let Ok(cmdline) = fs::read(entry.path().join("cmdline")) else { + continue; + }; + if cmdline.is_empty() { + continue; + } + let cmd = String::from_utf8_lossy(&cmdline).replace('\0', " "); + if !cmd.contains("Xtigervnc") { + continue; + } + if let Some(display_name) = &display_name { + if cmd.contains(display_name) { + return true; + } + } else { + return true; + } + } + false +} + +pub fn check_parallel_pointer_support() -> Result<()> { + let display = open_display()?; + let result = supports_parallel_pointer_injection(display); + unsafe { x11::xlib::XCloseDisplay(display) }; + result +} + +fn ensure_master_pointer(cursor_id: &str) -> Result { + if let Some(ids) = mpx_pointers().lock().unwrap().get(cursor_id).copied() { + return Ok(ids); + } + + let display = open_display()?; + let mut major = 2; + let mut minor = 3; + let rc = unsafe { x11::xinput2::XIQueryVersion(display, &mut major, &mut minor) }; + if rc != 0 { + unsafe { x11::xlib::XCloseDisplay(display) }; + bail!("XIQueryVersion failed with status {rc}"); + } + + let base = master_pointer_name(cursor_id); + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + let name = CString::new(base.clone())?; + unsafe { + let add = change.add(); + (*add)._type = x11::xinput2::XIAddMaster; + (*add).name = name.as_ptr() as *mut _; + // Core events stay on so core-only apps (xterm, Tk, …) receive the + // drags too. Note this is not what makes the WM focus the dragged + // window — XI2-aware WMs grab buttons for XIAllMasterDevices — see + // the active-window save/restore in send_parallel_virtual_pointer_drags. + (*add).send_core = 1; + (*add).enable = 1; + } + let rc = unsafe { x11::xinput2::XIChangeHierarchy(display, &mut change, 1) }; + unsafe { + x11::xlib::XSync(display, 0); + } + if rc != 0 { + unsafe { x11::xlib::XCloseDisplay(display) }; + bail!("XIChangeHierarchy(XIAddMaster) failed with status {rc}"); + } + + let devices = xi2_query_devices(display)?; + let mut pointer_id = None; + let mut keyboard_id = None; + let pointer_name = master_pointer_device_name(&base); + let keyboard_name = master_keyboard_device_name(&base); + for (device_id, use_, device_name) in devices { + if use_ == x11::xinput2::XIMasterPointer && device_name == pointer_name { + pointer_id = Some(device_id); + } else if use_ == x11::xinput2::XIMasterKeyboard && device_name == keyboard_name { + keyboard_id = Some(device_id); + } + } + + let pointer_id = pointer_id.ok_or_else(|| anyhow!("failed to locate created master pointer for '{cursor_id}'"))?; + let keyboard_id = keyboard_id.ok_or_else(|| anyhow!("failed to locate created master keyboard for '{cursor_id}'"))?; + + let device_name = slave_pointer_name(&base); + let uinput_device = create_uinput_pointer(&device_name)?; + let slave_pointer_id = wait_for_slave_pointer_id(display, &device_name)?; + attach_slave_to_master(display, slave_pointer_id, pointer_id)?; + set_flat_pointer_accel(display, slave_pointer_id); + unsafe { x11::xlib::XCloseDisplay(display) }; + + let ids = MasterPointerIds { pointer_id, keyboard_id, slave_pointer_id }; + mpx_pointers().lock().unwrap().insert(cursor_id.to_owned(), ids); + uinput_pointers() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), Arc::new(Mutex::new(uinput_device))); + Ok(ids) +} + +pub fn forget_master_pointer(cursor_id: &str) { + uinput_pointers().lock().unwrap().remove(cursor_id); + let Some(ids) = mpx_pointers().lock().unwrap().remove(cursor_id) else { + return; + }; + + let Ok(display) = open_display() else { + return; + }; + + let Ok(devices) = xi2_query_devices(display) else { + unsafe { x11::xlib::XCloseDisplay(display) }; + return; + }; + + let mut virtual_core_pointer = None; + let mut virtual_core_keyboard = None; + for (device_id, use_, device_name) in devices { + if device_name == "Virtual core pointer" && use_ == x11::xinput2::XIMasterPointer { + virtual_core_pointer = Some(device_id); + } else if device_name == "Virtual core keyboard" && use_ == x11::xinput2::XIMasterKeyboard { + virtual_core_keyboard = Some(device_id); + } + } + + let (Some(return_pointer), Some(return_keyboard)) = (virtual_core_pointer, virtual_core_keyboard) else { + unsafe { x11::xlib::XCloseDisplay(display) }; + return; + }; + + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + unsafe { + let remove = change.remove(); + (*remove)._type = x11::xinput2::XIRemoveMaster; + (*remove).deviceid = ids.pointer_id; + (*remove).return_mode = x11::xinput2::XIAttachToMaster; + (*remove).return_pointer = return_pointer; + (*remove).return_keyboard = return_keyboard; + let _ = x11::xinput2::XIChangeHierarchy(display, &mut change, 1); + x11::xlib::XSync(display, 0); + x11::xlib::XCloseDisplay(display); + } +} + +fn create_uinput_pointer(name: &str) -> Result { + let mut keys = AttributeSet::::new(); + keys.insert(Key::BTN_LEFT); + keys.insert(Key::BTN_RIGHT); + keys.insert(Key::BTN_MIDDLE); + + let mut rel_axes = AttributeSet::::new(); + rel_axes.insert(RelativeAxisType::REL_X); + rel_axes.insert(RelativeAxisType::REL_Y); + rel_axes.insert(RelativeAxisType::REL_WHEEL); + + Ok( + evdev::uinput::VirtualDeviceBuilder::new()? + .name(name) + .with_keys(&keys)? + .with_relative_axes(&rel_axes)? + .build()?, + ) +} + +fn wait_for_slave_pointer_id(display: *mut x11::xlib::Display, device_name: &str) -> Result { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + for (device_id, use_, seen_name) in xi2_query_devices(display)? { + if use_ == x11::xinput2::XISlavePointer && seen_name == device_name { + return Ok(device_id); + } + } + if std::time::Instant::now() >= deadline { + bail!("timed out waiting for X input slave pointer '{device_name}'"); + } + sleep(Duration::from_millis(50)); + } +} + +fn attach_slave_to_master(display: *mut x11::xlib::Display, slave_pointer_id: i32, master_pointer_id: i32) -> Result<()> { + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + unsafe { + let attach = change.attach(); + (*attach)._type = x11::xinput2::XIAttachSlave; + (*attach).deviceid = slave_pointer_id; + (*attach).new_master = master_pointer_id; + } + let rc = unsafe { x11::xinput2::XIChangeHierarchy(display, &mut change, 1) }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIChangeHierarchy(XIAttachSlave) failed with status {rc}"); + } + Ok(()) +} + +fn set_flat_pointer_accel(display: *mut x11::xlib::Display, slave_pointer_id: i32) { + // Pin libinput's accel profile to flat so relative deltas map 1:1 onto + // cursor movement — the default adaptive profile rescales small deltas + // and makes drag endpoints drift off-target by a few pixels. + // Best-effort: the property only exists under xf86-input-libinput. + unsafe { + let prop = x11::xlib::XInternAtom( + display, + c"libinput Accel Profile Enabled".as_ptr(), + x11::xlib::True, + ); + if prop == 0 { + return; + } + let mut type_ret: x11::xlib::Atom = 0; + let mut format_ret: std::os::raw::c_int = 0; + let mut num_items: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut data: *mut std::os::raw::c_uchar = std::ptr::null_mut(); + let rc = x11::xinput2::XIGetProperty( + display, + slave_pointer_id, + prop, + 0, + 16, + x11::xlib::False, + x11::xlib::AnyPropertyType as x11::xlib::Atom, + &mut type_ret, + &mut format_ret, + &mut num_items, + &mut bytes_after, + &mut data, + ); + if rc != x11::xlib::Success as i32 || data.is_null() { + return; + } + // Profile order is (adaptive, flat[, custom]); enable flat only. + if format_ret == 8 && (2..=8).contains(&num_items) { + let mut values = vec![0u8; num_items as usize]; + values[1] = 1; + x11::xinput2::XIChangeProperty( + display, + slave_pointer_id, + prop, + type_ret, + 8, + x11::xlib::PropModeReplace, + values.as_mut_ptr(), + num_items as std::os::raw::c_int, + ); + x11::xlib::XSync(display, 0); + } + x11::xlib::XFree(data as *mut _); + } +} + +fn warp_master_pointer(display: *mut x11::xlib::Display, ids: MasterPointerIds, x: i32, y: i32) -> Result<()> { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let rc = unsafe { + x11::xinput2::XIWarpPointer( + display, + ids.pointer_id, + 0, + root, + 0.0, + 0.0, + 0, + 0, + x as f64, + y as f64, + ) + }; + // XSync (not XFlush): the button press that follows is emitted through + // uinput on a separate kernel pipeline, and races ahead of a merely + // queued warp request. Once XSync returns the server has executed the + // warp, so the press lands at the warped position. + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIWarpPointer failed with status {rc}"); + } + Ok(()) +} + +/// XIAnyModifier (1u32 << 31). The x11 crate doesn't export it. +const XI_ANY_MODIFIER: std::os::raw::c_int = 0x8000_0000u32 as std::os::raw::c_int; + +fn xi_mask_len() -> usize { + (x11::xinput2::XI_LASTEVENT as usize >> 3) + 1 +} + +/// Look up the XInputExtension major opcode so we can recognise its +/// GenericEvent cookies on the display connection. +fn xinput_opcode(display: *mut x11::xlib::Display) -> Option { + let name = match CString::new("XInputExtension") { + Ok(n) => n, + Err(_) => return None, + }; + let mut opcode = 0; + let mut event = 0; + let mut error = 0; + let present = unsafe { + x11::xlib::XQueryExtension(display, name.as_ptr(), &mut opcode, &mut event, &mut error) + }; + if present != 0 { + Some(opcode) + } else { + None + } +} + +/// Install a device-specific XI2 synchronous passive button grab on `window` +/// for `device_id`. This shields the drag: the grab is newer than (and thus +/// checked before) the window manager's click-to-focus grab on the same +/// window, and being device-specific it does not conflict with the WM's +/// core/all-master grabs. The matching press freezes the device and is +/// delivered to us; replaying it (XIReplayDevice) re-checks grabs only +/// *below* this window and then delivers the event normally to the app, so +/// the WM never sees the press and never steals focus. +fn install_shield_grab( + display: *mut x11::xlib::Display, + device_id: i32, + window: x11::xlib::Window, + button: u8, +) -> Result<()> { + let mut mask_bits = vec![0u8; xi_mask_len()]; + x11::xinput2::XISetMask(&mut mask_bits, x11::xinput2::XI_ButtonPress); + let mut evmask = x11::xinput2::XIEventMask { + deviceid: device_id, + mask_len: mask_bits.len() as std::os::raw::c_int, + mask: mask_bits.as_mut_ptr(), + }; + let mut mods = x11::xinput2::XIGrabModifiers { + modifiers: XI_ANY_MODIFIER, + status: 0, + }; + let rc = unsafe { + x11::xinput2::XIGrabButton( + display, + device_id, + button as std::os::raw::c_int, + window, + 0, // cursor: None + x11::xinput2::XIGrabModeSync, // freeze the pointer on press + x11::xinput2::XIGrabModeAsync, // leave the paired keyboard alone + x11::xlib::False, // owner_events: deliver to us + &mut evmask, + 1, + &mut mods, + ) + }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIGrabButton(shield) failed with status {rc}"); + } + Ok(()) +} + +fn remove_shield_grab(display: *mut x11::xlib::Display, device_id: i32, window: x11::xlib::Window, button: u8) { + let mut mods = x11::xinput2::XIGrabModifiers { + modifiers: XI_ANY_MODIFIER, + status: 0, + }; + unsafe { + let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xinput2::XIUngrabButton(display, device_id, button as std::os::raw::c_int, window, 1, &mut mods); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev); + } +} + +/// Drain the frozen shield presses for `pending_devices` and replay each so +/// it continues to the application. Returns the set of device ids we failed +/// to see within the timeout (their drags still proceed; the focus-restore +/// safety net covers any leak). +fn replay_shielded_presses( + display: *mut x11::xlib::Display, + xi_opcode: std::os::raw::c_int, + pending_devices: &mut std::collections::HashSet, + timeout: Duration, +) { + let deadline = std::time::Instant::now() + timeout; + while !pending_devices.is_empty() && std::time::Instant::now() < deadline { + // Only block on XNextEvent when something is queued, so a missing + // press can't hang us past the deadline. + if unsafe { x11::xlib::XPending(display) } == 0 { + sleep(Duration::from_millis(2)); + continue; + } + let mut ev: x11::xlib::XEvent = unsafe { std::mem::zeroed() }; + unsafe { x11::xlib::XNextEvent(display, &mut ev) }; + if unsafe { ev.type_ } != x11::xlib::GenericEvent { + continue; + } + let mut cookie = unsafe { ev.generic_event_cookie }; + if cookie.extension != xi_opcode || cookie.evtype != x11::xinput2::XI_ButtonPress { + continue; + } + if unsafe { x11::xlib::XGetEventData(display, &mut cookie) } == 0 { + continue; + } + let de = cookie.data as *const x11::xinput2::XIDeviceEvent; + if !de.is_null() { + let device_id = unsafe { (*de).deviceid }; + let time = unsafe { (*de).time }; + if pending_devices.remove(&device_id) { + unsafe { + x11::xinput2::XIAllowEvents(display, device_id, x11::xinput2::XIReplayDevice, time); + x11::xlib::XSync(display, 0); + } + } + } + unsafe { x11::xlib::XFreeEventData(display, &mut cookie) }; + } +} + +fn ewmh_active_window(display: *mut x11::xlib::Display) -> Option { + unsafe { + let atom = x11::xlib::XInternAtom( + display, + c"_NET_ACTIVE_WINDOW".as_ptr(), + x11::xlib::True, + ); + if atom == 0 { + return None; + } + let root = x11::xlib::XDefaultRootWindow(display); + let mut type_ret: x11::xlib::Atom = 0; + let mut format_ret: std::os::raw::c_int = 0; + let mut nitems: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut data: *mut std::os::raw::c_uchar = std::ptr::null_mut(); + let rc = x11::xlib::XGetWindowProperty( + display, + root, + atom, + 0, + 1, + x11::xlib::False, + x11::xlib::XA_WINDOW, + &mut type_ret, + &mut format_ret, + &mut nitems, + &mut bytes_after, + &mut data, + ); + if rc != x11::xlib::Success as i32 || data.is_null() { + return None; + } + let window = if nitems >= 1 && format_ret == 32 { + Some(*(data as *const std::os::raw::c_ulong) as x11::xlib::Window) + } else { + None + }; + x11::xlib::XFree(data as *mut _); + window.filter(|w| *w != 0) + } +} + +/// Current X server time via the standard PropertyNotify round-trip. +/// EWMH activation requests stamped CurrentTime(0) lose to the WM's +/// focus-stealing prevention whenever any newer input exists. +fn x_server_time(display: *mut x11::xlib::Display) -> x11::xlib::Time { + unsafe { + let root = x11::xlib::XDefaultRootWindow(display); + let win = x11::xlib::XCreateSimpleWindow(display, root, -1, -1, 1, 1, 0, 0, 0); + x11::xlib::XSelectInput(display, win, x11::xlib::PropertyChangeMask); + let atom = x11::xlib::XInternAtom(display, c"CUA_TIME_PROBE".as_ptr(), x11::xlib::False); + x11::xlib::XChangeProperty( + display, + win, + atom, + x11::xlib::XA_STRING, + 8, + x11::xlib::PropModeReplace, + [0u8].as_ptr(), + 0, + ); + x11::xlib::XSync(display, 0); + let mut time: x11::xlib::Time = x11::xlib::CurrentTime; + let mut ev: x11::xlib::XEvent = std::mem::zeroed(); + while x11::xlib::XCheckWindowEvent( + display, + win, + x11::xlib::PropertyChangeMask, + &mut ev, + ) != 0 + { + if ev.get_type() == x11::xlib::PropertyNotify { + time = ev.property.time; + } + } + x11::xlib::XDestroyWindow(display, win); + x11::xlib::XFlush(display); + time + } +} + +fn ewmh_activate_window( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, + current_active: x11::xlib::Window, +) { + unsafe { + let atom = x11::xlib::XInternAtom( + display, + c"_NET_ACTIVE_WINDOW".as_ptr(), + x11::xlib::True, + ); + if atom == 0 { + return; + } + let root = x11::xlib::XDefaultRootWindow(display); + let mut ev: x11::xlib::XClientMessageEvent = std::mem::zeroed(); + ev.type_ = x11::xlib::ClientMessage; + ev.window = window; + ev.message_type = atom; + ev.format = 32; + ev.data.set_long(0, 2); // source indication: pager/tool + ev.data.set_long(1, x_server_time(display) as std::os::raw::c_long); + ev.data.set_long(2, current_active as std::os::raw::c_long); + x11::xlib::XSendEvent( + display, + root, + x11::xlib::False, + x11::xlib::SubstructureRedirectMask | x11::xlib::SubstructureNotifyMask, + &mut ev as *mut _ as *mut x11::xlib::XEvent, + ); + x11::xlib::XSync(display, 0); + } +} + +fn button_code(button: u8) -> Result { + match button { + 1 => Ok(Key::BTN_LEFT), + 2 => Ok(Key::BTN_MIDDLE), + 3 => Ok(Key::BTN_RIGHT), + _ => bail!("unsupported button {button} for uinput pointer"), + } +} + +fn emit_button(device: &mut VirtualDevice, button: u8, press: bool) -> Result<()> { + let code = button_code(button)?; + device.emit(&[InputEvent::new(EventType::KEY, code.0, if press { 1 } else { 0 })])?; + Ok(()) +} + +fn emit_relative_motion(device: &mut VirtualDevice, dx: i32, dy: i32) -> Result<()> { + let mut events = Vec::with_capacity(2); + if dx != 0 { + events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_X.0, dx)); + } + if dy != 0 { + events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_Y.0, dy)); + } + if events.is_empty() { + return Ok(()); + } + device.emit(&events)?; + Ok(()) +} + +pub fn send_parallel_virtual_pointer_drags( + drags: &[(String, VirtualPointerDrag)], +) -> Result<()> { + let display = open_display()?; + supports_parallel_pointer_injection(display)?; + let xi_opcode = xinput_opcode(display); + + struct ActiveDrag { + cursor_id: String, + ids: MasterPointerIds, + device: Arc>, + drag: VirtualPointerDrag, + steps: usize, + step_delay: Duration, + current_step: usize, + next_at: std::time::Instant, + last_x: i32, + last_y: i32, + } + + let start_at = std::time::Instant::now() + Duration::from_millis(120); + let mut active = Vec::with_capacity(drags.len()); + + // Click-to-focus WMs grab buttons for XIAllMasterDevices, so the drag's + // press activates the target window exactly like a user click would. + // Remember the focus state and hand it back afterwards so parallel + // drags don't steal it. + let saved_focus = save_focus_state(display); + + let result = (|| -> Result<()> { + for (cursor_id, drag) in drags { + let ids = ensure_master_pointer(cursor_id)?; + let device = uinput_pointers() + .lock() + .unwrap() + .get(cursor_id) + .cloned() + .ok_or_else(|| anyhow!("missing uinput pointer for '{cursor_id}'"))?; + active.push(ActiveDrag { + cursor_id: cursor_id.clone(), + ids, + device, + drag: *drag, + steps: drag.steps.max(1), + step_delay: if drag.steps.max(1) > 1 { + Duration::from_millis(drag.duration_ms / drag.steps.max(1) as u64) + } else { + Duration::from_millis(drag.duration_ms) + }, + current_step: 0, + next_at: start_at, + last_x: drag.from_x, + last_y: drag.from_y, + }); + } + + let now = std::time::Instant::now(); + if start_at > now { + std::thread::sleep(start_at - now); + } + + // Shield each drag from the WM's click-to-focus grab, then press. + // Per item: install a device-specific sync grab on the target window, + // warp, press, and immediately replay the frozen press so it reaches + // the app while the WM stays blind to it. We replay each press before + // emitting the next so only ONE device is ever frozen at a time — the + // X server drops replayed presses when several devices are frozen on + // the same window and replayed together. The few-ms stagger this adds + // to the presses is invisible; the concurrency that matters is motion. + // If a shield fails to install we still press (the drag works, only + // focus protection is lost — the restore safety net covers it). + let mut shielded = std::collections::HashSet::new(); + for item in &active { + let did_shield = if xi_opcode.is_some() { + match install_shield_grab( + display, + item.ids.pointer_id, + item.drag.target_window as x11::xlib::Window, + item.drag.button, + ) { + Ok(()) => { + shielded.insert(item.ids.pointer_id); + true + } + Err(e) => { + tracing::warn!("shield grab failed for '{}': {e}", item.cursor_id); + false + } + } + } else { + false + }; + warp_master_pointer(display, item.ids, item.drag.from_x, item.drag.from_y)?; + { + let mut device = item.device.lock().unwrap(); + emit_button(&mut device, item.drag.button, true)?; + } + if let (true, Some(opcode)) = (did_shield, xi_opcode) { + let mut pending = std::collections::HashSet::from([item.ids.pointer_id]); + replay_shielded_presses(display, opcode, &mut pending, Duration::from_millis(1000)); + if !pending.is_empty() { + tracing::warn!("shield replay: press for '{}' not seen before timeout", item.cursor_id); + } + } + } + + while active.iter().any(|item| item.current_step < item.steps) { + let now = std::time::Instant::now(); + let mut advanced = false; + let mut next_deadline = None; + + for item in &mut active { + if item.current_step >= item.steps { + continue; + } + if now >= item.next_at { + item.current_step += 1; + let t = item.current_step as f64 / item.steps as f64; + let ix = item.drag.from_x + + ((item.drag.to_x - item.drag.from_x) as f64 * t).round() as i32; + let iy = item.drag.from_y + + ((item.drag.to_y - item.drag.from_y) as f64 * t).round() as i32; + let dx = ix - item.last_x; + let dy = iy - item.last_y; + if dx != 0 || dy != 0 { + let mut device = item.device.lock().unwrap(); + emit_relative_motion(&mut device, dx, dy)?; + // Keep the agent cursor overlay tracking the drag so + // the gesture is visible, not just its endpoints. + crate::overlay::send_command_for( + item.cursor_id.clone(), + cursor_overlay::OverlayCommand::SnapTo { + x: ix as f64, + y: iy as f64, + heading_radians: Some((dy as f64).atan2(dx as f64)), + }, + ); + } + item.last_x = ix; + item.last_y = iy; + item.next_at = now + item.step_delay; + advanced = true; + } + if item.current_step < item.steps { + next_deadline = Some(match next_deadline { + Some(deadline) => std::cmp::min(deadline, item.next_at), + None => item.next_at, + }); + } + } + + if !advanced { + if let Some(deadline) = next_deadline { + let now = std::time::Instant::now(); + if deadline > now { + std::thread::sleep(deadline - now); + } + } + } + } + + for item in &active { + let mut device = item.device.lock().unwrap(); + emit_button(&mut device, item.drag.button, false)?; + } + + // Remove the shields now that the drag is done. The button is only + // grabbed for ButtonPress, so the shield is dormant during motion and + // release; this just stops it matching the next gesture's press. + for item in &active { + if shielded.contains(&item.ids.pointer_id) { + remove_shield_grab( + display, + item.ids.pointer_id, + item.drag.target_window as x11::xlib::Window, + item.drag.button, + ); + } + } + Ok(()) + })(); + // Remove the per-session masters before handing focus back: non-MPX-aware + // WMs (xfwm4, openbox) desync their focus bookkeeping while foreign + // master keyboards linger, and the next call recreates masters cheaply. + for (cursor_id, _) in drags { + forget_master_pointer(cursor_id); + } + restore_focus_state(display, &saved_focus); + unsafe { + x11::xlib::XCloseDisplay(display); + } + result +} + +/// Pre-drag focus snapshot: the EWMH active window when a conforming WM is +/// running, plus the core input focus as a WM-agnostic fallback. +struct SavedFocus { + ewmh_active: Option, + core_focus: x11::xlib::Window, + core_revert_to: std::os::raw::c_int, +} + +fn save_focus_state(display: *mut x11::xlib::Display) -> SavedFocus { + let mut core_focus: x11::xlib::Window = 0; + let mut core_revert_to: std::os::raw::c_int = 0; + unsafe { + x11::xlib::XGetInputFocus(display, &mut core_focus, &mut core_revert_to); + } + SavedFocus { + ewmh_active: ewmh_active_window(display), + core_focus, + core_revert_to, + } +} + +unsafe extern "C" fn ignore_x_error( + _display: *mut x11::xlib::Display, + _event: *mut x11::xlib::XErrorEvent, +) -> std::os::raw::c_int { + 0 +} + +fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) { + // Let the release/focus events from the drag settle before reading the + // post-drag state, so we don't race the WM's own focus update. + unsafe { x11::xlib::XSync(display, 0) }; + + if let Some(prev) = saved.ewmh_active { + // EWMH path: ask the WM to re-activate, so its active-window + // bookkeeping (decorations, stacking) stays consistent. The WM + // processes its own click-to-focus for the drag asynchronously and + // can re-activate the dragged window even after one re-activation of + // ours has landed — so don't stop at first success: require the + // active window to hold stable for consecutive checks, re-sending on + // every regression, within a bounded budget. + sleep(Duration::from_millis(300)); + let mut stable = 0; + for attempt in 0..15 { + let now = ewmh_active_window(display); + if now == Some(prev) { + stable += 1; + if stable >= 3 { + return; + } + } else { + stable = 0; + // MPX clicks can leave a core-protocol WM believing the + // dragged window is focused while the core focus never moved + // there: its XSetInputFocus for our activation is then a + // no-op, no FocusIn arrives, and its bookkeeping never + // updates. Bounce the core focus onto the window the WM + // believes active so the activation produces a real focus + // transition the WM can observe. + if attempt >= 2 { + if let Some(now_win) = now { + unsafe { + let prev_handler = + x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xlib::XSetInputFocus( + display, + now_win, + x11::xlib::RevertToParent, + x11::xlib::CurrentTime, + ); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev_handler); + } + sleep(Duration::from_millis(100)); + } + } + ewmh_activate_window(display, prev, now.unwrap_or(0)); + } + sleep(Duration::from_millis(200)); + } + if stable == 0 { + tracing::warn!("focus restore: WM did not re-activate 0x{prev:x}"); + } + return; + } + + // No EWMH WM (bare X / minimal WM): restore the core input focus + // directly. The saved window may have been destroyed meanwhile, and + // Xlib's default error handler exits the process on BadWindow, so the + // restore runs under a scoped ignore-errors handler. + if saved.core_focus == 0 { + return; + } + unsafe { + let mut now_focus: x11::xlib::Window = 0; + let mut now_revert: std::os::raw::c_int = 0; + x11::xlib::XGetInputFocus(display, &mut now_focus, &mut now_revert); + if now_focus == saved.core_focus { + return; + } + let prev_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xlib::XSetInputFocus( + display, + saved.core_focus, + saved.core_revert_to, + x11::xlib::CurrentTime, + ); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev_handler); + } +} + #[derive(Clone, Copy, Debug)] struct EventTarget { window: Window, diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs index 01dd261263..2ae7df7eba 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs @@ -17,49 +17,188 @@ //! What stays here is the X11 window plumbing: connection setup, //! override-redirect visual, ShapeInput passthrough, and the XPutImage paint. +use std::collections::{HashMap, HashSet}; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; -use cursor_overlay::{CursorConfig, OverlayCommand, RenderStateCore}; +use cursor_overlay::{ + CursorConfig, CursorKey, KeyedOverlayCommand, OverlayCommand, OverlayMsg, Palette, + RenderStateCore, +}; #[cfg(target_os = "linux")] use cursor_overlay::ZOrderEnforcer; // ── Global channel ──────────────────────────────────────────────────────── -static CMD_TX: OnceLock> = OnceLock::new(); -static CMD_RX_CELL: Mutex>> = Mutex::new(None); -static RENDER: Mutex> = Mutex::new(None); -static ARRIVAL_TX: Mutex>> = Mutex::new(None); +static CMD_TX: OnceLock> = OnceLock::new(); +static CMD_RX_CELL: Mutex>> = Mutex::new(None); +static RENDER: Mutex> = Mutex::new(None); +static ARRIVAL_TX: Mutex>>> = + Mutex::new(None); + +fn arrival_register(key: CursorKey, tx: tokio::sync::oneshot::Sender<()>) { + let mut guard = ARRIVAL_TX.lock().unwrap(); + let map = guard.get_or_insert_with(HashMap::new); + if let Some(old_tx) = map.insert(key, tx) { + let _ = old_tx.send(()); + } +} + +fn arrival_fire(key: &CursorKey) { + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(map) = guard.as_mut() { + if let Some(tx) = map.remove(key) { + let _ = tx.send(()); + } + } + } +} + +struct RenderMap { + cursors: HashMap, + scr_w: u32, + scr_h: u32, + template: CursorConfig, + ended: HashSet, + last_active: Option, +} + +fn render_state_for_key(template: &CursorConfig, key: &str) -> RenderState { + let mut rs = RenderState::new(template.clone()); + rs.core.palette = Palette::for_instance(key); + rs +} + +fn apply_msg(map: &mut RenderMap, msg: OverlayMsg) -> Option { + match msg { + OverlayMsg::Remove(key) => { + if key != "default" { + map.cursors.remove(&key); + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(arrivals) = guard.as_mut() { + arrivals.remove(&key); + } + } + if map.last_active.as_deref() == Some(key.as_str()) { + map.last_active = None; + } + map.ended.insert(key); + } + None + } + OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }) => { + if map.ended.contains(&key) { + return None; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key) + .or_insert_with(|| render_state_for_key(&template, &k)); + rs.apply_command(cmd); + Some(k) + } + } +} pub fn init(cfg: CursorConfig) { let (tx, rx) = std::sync::mpsc::sync_channel(4096); let _ = CMD_TX.set(tx); *CMD_RX_CELL.lock().unwrap() = Some(rx); - *RENDER.lock().unwrap() = Some(RenderState::new(cfg)); + *ARRIVAL_TX.lock().unwrap() = Some(HashMap::new()); + let mut cursors = HashMap::new(); + cursors.insert("default".to_owned(), RenderState::new(cfg.clone())); + *RENDER.lock().unwrap() = Some(RenderMap { + cursors, + scr_w: 1920, + scr_h: 1080, + template: cfg, + ended: HashSet::new(), + last_active: None, + }); } pub fn send_command(cmd: OverlayCommand) { + send_command_for("default".to_owned(), cmd); +} + +pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) { + if key.is_empty() { + return; + } if let Some(tx) = CMD_TX.get() { - let _ = tx.try_send(cmd); + let _ = tx.try_send(OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd })); } } pub fn is_enabled() -> bool { + is_enabled_for("default") +} + +pub fn is_enabled_for(key: &str) -> bool { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.visible)) + .and_then(|g| { + g.as_ref().and_then(|m| { + m.cursors + .get(key) + .or_else(|| m.cursors.get("default")) + .map(|rs| rs.core.visible) + }) + }) .unwrap_or(false) } pub fn current_position() -> (f64, f64) { + current_position_for("default") +} + +pub fn current_position_for(key: &str) -> (f64, f64) { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.pos)) + .and_then(|g| g.as_ref().and_then(|m| m.cursors.get(key)).map(|rs| rs.core.pos)) .unwrap_or((-200.0, -200.0)) } +fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool { + const SEED_OFFSET: f64 = 140.0; + let mut guard = RENDER.lock().unwrap(); + let Some(map) = guard.as_mut() else { return false }; + if map.ended.contains(key) { + return false; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key.clone()) + .or_insert_with(|| render_state_for_key(&template, &k)); + if !(rs.core.cfg.enabled && rs.core.pos.0 < -50.0) { + return false; + } + let max_x = map.scr_w.max(2) as f64 - 2.0; + let max_y = map.scr_h.max(2) as f64 - 2.0; + let mut sx = (target_x - SEED_OFFSET).clamp(2.0, max_x); + let mut sy = (target_y - SEED_OFFSET).clamp(2.0, max_y); + if (sx - target_x).abs() < 8.0 && (sy - target_y).abs() < 8.0 { + sx = (target_x + SEED_OFFSET).clamp(2.0, max_x); + sy = (target_y + SEED_OFFSET).clamp(2.0, max_y); + } + rs.core.pos = (sx, sy); + true +} + pub async fn animate_cursor_to(x: f64, y: f64) { + animate_cursor_to_for("default".to_owned(), x, y).await; +} + +pub async fn animate_cursor_to_for(key: CursorKey, x: f64, y: f64) { + if key.is_empty() { + return; + } + seed_start_if_sentinel(&key, x, y); let should_animate = { let guard = RENDER.lock().unwrap(); - match guard.as_ref() { + match guard.as_ref().and_then(|m| m.cursors.get(&key)) { Some(rs) if rs.core.cfg.enabled && rs.core.visible && rs.core.pos.0 > -50.0 => true, _ => false, } @@ -69,15 +208,9 @@ pub async fn animate_cursor_to(x: f64, y: f64) { } let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - { - let mut guard = ARRIVAL_TX.lock().unwrap(); - if let Some(old_tx) = guard.take() { - let _ = old_tx.send(()); - } - *guard = Some(tx); - } + arrival_register(key.clone(), tx); - send_command(OverlayCommand::MoveTo { + send_command_for(key, OverlayCommand::MoveTo { x, y, end_heading_radians: std::f64::consts::FRAC_PI_4, @@ -86,6 +219,15 @@ pub async fn animate_cursor_to(x: f64, y: f64) { let _ = rx.await; } +pub fn remove_cursor(key: CursorKey) { + if key.is_empty() { + return; + } + if let Some(tx) = CMD_TX.get() { + let _ = tx.try_send(OverlayMsg::Remove(key)); + } +} + /// Spawn the overlay on a dedicated thread. Non-blocking. pub fn run_on_thread() { let rx = match CMD_RX_CELL.lock().unwrap().take() { @@ -96,7 +238,7 @@ pub fn run_on_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.core.cfg.clone(), + Some(map) => map.template.clone(), None => return, } }; @@ -121,17 +263,12 @@ pub fn run_on_thread() { struct RenderState { core: RenderStateCore, - /// X11 screen dimensions in pixels (populated after XOpenDisplay). - scr_w: u32, - scr_h: u32, } impl RenderState { fn new(cfg: CursorConfig) -> Self { RenderState { core: RenderStateCore::new(cfg), - scr_w: 1920, - scr_h: 1080, } } @@ -153,14 +290,14 @@ impl RenderState { // ── X11 thread ──────────────────────────────────────────────────────────── #[cfg(target_os = "linux")] -fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { +fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { use x11rb::connection::Connection; - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; - use x11rb::protocol::shape::*; - use x11rb::protocol::shape::ConnectionExt as _; - use x11rb::wrapper::ConnectionExt as _; - use x11rb::COPY_FROM_PARENT; + use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK, SO}; + use x11rb::protocol::xproto::{ + AtomEnum, ColormapAlloc, CreateWindowAux, EventMask, PropMode, WindowClass, + }; + use x11rb::protocol::xproto::ConnectionExt as XprotoConnectionExt; + use x11rb::wrapper::ConnectionExt as WrapperConnectionExt; // Connect to X11. let (conn, screen_num) = match x11rb::connect(None) { @@ -179,9 +316,9 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver= Duration::from_millis(80) { last_ztick = Instant::now(); - let pinned_wid = { - let guard = RENDER.lock().unwrap(); - guard.as_ref().and_then(|rs| rs.core.pinned_wid) - }; z_enforcer.reassert(pinned_wid); } @@ -332,8 +475,7 @@ struct X11ZOrderEnforcer<'a, C: x11rb::connection::Connection> { #[cfg(target_os = "linux")] impl<'a, C: x11rb::connection::Connection> ZOrderEnforcer for X11ZOrderEnforcer<'a, C> { fn reassert(&self, target: Option) { - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; + use x11rb::protocol::xproto::{ConfigureWindowAux, ConnectionExt as XprotoConnectionExt, StackMode}; // Per the ZOrderEnforcer trait contract, a stale `target` (window // gone) should fall back to the `None` behavior — top of the @@ -396,8 +538,7 @@ fn paint_x11( _visual_id: u32, pm: &tiny_skia::Pixmap, ) { - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; + use x11rb::protocol::xproto::{ConnectionExt as XprotoConnectionExt, CreateGCAux, ImageFormat}; if pm.width() == 0 || pm.height() == 0 { return; } // Create a GC for the window if we don't have one. @@ -436,4 +577,4 @@ fn paint_x11( } #[cfg(not(target_os = "linux"))] -fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) {} +fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) {} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index a28772d4c9..dda4a553b4 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -1,7 +1,11 @@ //! Real Linux tool implementations (compiled only on Linux). use async_trait::async_trait; -use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}}; +use cua_driver_core::{ + protocol::ToolResult, + tool::{Tool, ToolDef, ToolRegistry}, + tool_args::ArgsExt, +}; use serde_json::{json, Value}; use std::fs; use std::path::PathBuf; @@ -65,12 +69,13 @@ pub struct ToolState { pub cursor_registry: Arc, pub resize_registry: Arc, pub zoom_registry: Arc, - pub mouse_hold: std::sync::Mutex>, + pub mouse_hold: std::sync::Mutex>, pub config: Arc>, } #[derive(Clone, Debug)] pub struct MouseHoldState { + pub cursor_id: String, pub pid: u32, pub xid: u64, pub button: u8, @@ -85,7 +90,7 @@ impl ToolState { cursor_registry: Arc::new(CursorRegistry::new()), resize_registry: Arc::new(ResizeRegistry::new()), zoom_registry: Arc::new(ZoomRegistry::new()), - mouse_hold: std::sync::Mutex::new(None), + mouse_hold: std::sync::Mutex::new(Default::default()), config: Arc::new(RwLock::new(DriverConfig::default())), }) } @@ -592,9 +597,21 @@ fn mouse_button_name(button: u8) -> &'static str { } } -fn mouse_hold_json(hold: Option<&MouseHoldState>) -> Value { +fn resolve_cursor_key(args: &Value) -> String { + for key in ["session", "cursor_id"] { + if let Some(v) = args.get(key).and_then(|v| v.as_str()) { + if !v.is_empty() { + return v.to_owned(); + } + } + } + "default".to_owned() +} + +fn mouse_hold_json(cursor_id: &str, hold: Option<&MouseHoldState>) -> Value { match hold { Some(hold) => json!({ + "cursor_id": cursor_id, "held": true, "pid": hold.pid, "window_id": hold.xid, @@ -603,6 +620,7 @@ fn mouse_hold_json(hold: Option<&MouseHoldState>) -> Value { "y": hold.y, }), None => json!({ + "cursor_id": cursor_id, "held": false, "pid": Value::Null, "window_id": Value::Null, @@ -613,16 +631,72 @@ fn mouse_hold_json(hold: Option<&MouseHoldState>) -> Value { } } +fn held_target_mismatch(args: &Value, cursor_id: &str, hold: &MouseHoldState) -> Option { + match args.opt_u32("pid") { + Ok(Some(pid)) if pid != hold.pid => { + return Some( + ToolResult::error(format!( + "Cursor '{cursor_id}' is holding a button for pid {}, not pid {pid}.", + hold.pid + )) + .with_structured(mouse_hold_json(cursor_id, Some(hold))), + ); + } + Err(err) => return Some(err.with_structured(mouse_hold_json(cursor_id, Some(hold)))), + _ => {} + } + + match args.opt_u64("window_id") { + Some(xid) if xid != hold.xid => Some( + ToolResult::error(format!( + "Cursor '{cursor_id}' is holding a button for window_id {}, not {xid}.", + hold.xid + )) + .with_structured(mouse_hold_json(cursor_id, Some(hold))), + ), + _ => None, + } +} + async fn overlay_glide_to(sx: f64, sy: f64) { - if !crate::overlay::is_enabled() { + overlay_glide_to_for("default", sx, sy).await; +} + +fn overlay_snap_to_for(cursor_id: &str, sx: f64, sy: f64, heading: Option) { + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::SnapTo { + x: sx, + y: sy, + heading_radians: heading, + }, + ); +} + +fn overlay_move_to_for(cursor_id: &str, sx: f64, sy: f64, heading: Option) { + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::MoveTo { + x: sx, + y: sy, + end_heading_radians: heading.unwrap_or(std::f64::consts::FRAC_PI_4), + }, + ); +} + +async fn overlay_glide_to_for(cursor_id: &str, sx: f64, sy: f64) { + if !crate::overlay::is_enabled_for(cursor_id) { return; } - let pos = crate::overlay::current_position(); + let pos = crate::overlay::current_position_for(cursor_id); if pos.0 < 0.0 && pos.1 < 0.0 { - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); return; } - crate::overlay::animate_cursor_to(sx, sy).await; + crate::overlay::animate_cursor_to_for(cursor_id.to_owned(), sx, sy).await; } fn process_name(pid: u32) -> Option { @@ -769,6 +843,8 @@ impl Tool for ClickTool { back to full-window space.".into(), input_schema: json!({ "type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -784,7 +860,7 @@ impl Tool for ClickTool { } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; let count = args.u64_or("count", 1) as usize; let button = parse_mouse_button(args.str_or("button", "left").as_str()); @@ -814,10 +890,16 @@ impl Tool for ClickTool { return match result { Ok(Ok((xid, x, y))) => { if xid != 0 { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); } - overlay_glide_to(x, y).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y }); + overlay_glide_to_for(&cursor_id, x, y).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x, y }, + ); ToolResult::text(format!("Clicked element [{idx}] (pid {pid}).")) } Ok(Err(e)) => ToolResult::error(format!("AT-SPI element click failed: {e}")), @@ -845,12 +927,18 @@ impl Tool for ClickTool { y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); @@ -1269,6 +1357,8 @@ impl Tool for DoubleClickTool { No focus steal. Provide either (window_id + x/y) or (pid + element_index). \ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(), input_schema: json!({"type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1281,6 +1371,7 @@ impl Tool for DoubleClickTool { } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; @@ -1291,9 +1382,15 @@ impl Tool for DoubleClickTool { return match result { Ok(Ok((xid, lx, ly))) => { if let Ok((sx, sy)) = element_screen_center(pid, idx) { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await { Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")), @@ -1322,12 +1419,18 @@ impl Tool for DoubleClickTool { x *= ratio; y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await; @@ -1355,6 +1458,8 @@ impl Tool for RightClickTool { No focus steal. Provide either (window_id + x/y) or (pid + element_index). \ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(), input_schema: json!({"type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1368,6 +1473,7 @@ impl Tool for RightClickTool { } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; @@ -1378,9 +1484,15 @@ impl Tool for RightClickTool { return match result { Ok(Ok((xid, lx, ly))) => { if let Ok((sx, sy)) = element_screen_center(pid, idx) { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await { Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")), @@ -1409,12 +1521,18 @@ impl Tool for RightClickTool { x *= ratio; y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await; @@ -1442,6 +1560,8 @@ impl Tool for DragTool { window-local screenshot pixels via XSendEvent (ButtonPress + MotionNotify × steps + ButtonRelease). \ duration_ms (default 500), steps (default 20). No focus steal.".into(), input_schema: json!({"type":"object","required":["pid","from_x","from_y","to_x","to_y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer","description":"Target window XID. Required."}, "from_x":{"type":"number"}, @@ -1458,7 +1578,7 @@ impl Tool for DragTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("window_id is required on Linux."), @@ -1492,46 +1612,119 @@ impl Tool for DragTool { to_x *= ratio; to_y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx_from, sy_from))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await { - overlay_glide_to(sx_from, sy_from).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { - x: sx_from, - y: sy_from, - }); + overlay_glide_to_for(&cursor_id, sx_from, sy_from).await; + self.state.cursor_registry.update_position(&cursor_id, sx_from, sy_from); + overlay_snap_to_for(&cursor_id, sx_from, sy_from, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx_from, y: sy_from }, + ); } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); - let result = tokio::task::spawn_blocking(move || { - crate::input::send_drag( - xid, - from_x as i32, from_y as i32, - to_x as i32, to_y as i32, - duration_ms, steps, button, - ) + let press_result = tokio::task::spawn_blocking(move || { + crate::input::send_button_down(xid, from_x.round() as i32, from_y.round() as i32, button) }).await; + let mut result: anyhow::Result<()> = match press_result { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::anyhow!("Task error: {e}")), + }; - if matches!(&result, Ok(Ok(()))) { + if result.is_ok() { + let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; + let mut prev_x = from_x; + let mut prev_y = from_y; + for i in 1..=steps { + let t = i as f64 / steps.max(1) as f64; + let ix = from_x + (to_x - from_x) * t; + let iy = from_y + (to_y - from_y) * t; + let motion_result = tokio::task::spawn_blocking(move || { + crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button)) + }).await; + match motion_result { + Ok(Ok(())) => { + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await + { + let heading = if (ix - prev_x).abs() > f64::EPSILON + || (iy - prev_y).abs() > f64::EPSILON + { + Some((iy - prev_y).atan2(ix - prev_x)) + } else { + None + }; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_move_to_for(&cursor_id, sx, sy, heading); + } + prev_x = ix; + prev_y = iy; + if step_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms)).await; + } + } + Ok(Err(e)) => { + result = Err(e); + break; + } + Err(e) => { + result = Err(anyhow::anyhow!("Task error: {e}")); + break; + } + } + } + } + + let release_result = tokio::task::spawn_blocking(move || { + crate::input::send_button_up(xid, to_x.round() as i32, to_y.round() as i32, button) + }).await; + if result.is_ok() { + result = match release_result { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::anyhow!("Task error: {e}")), + }; + } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(false), + ); + + if result.is_ok() { if let Ok(Ok((sx_to, sy_to))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await { - overlay_glide_to(sx_to, sy_to).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { - x: sx_to, - y: sy_to, - }); + self.state.cursor_registry.update_position(&cursor_id, sx_to, sy_to); + overlay_snap_to_for( + &cursor_id, + sx_to, + sy_to, + Some((to_y - from_y).atan2(to_x - from_x)), + ); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx_to, y: sy_to }, + ); } } match result { - Ok(Ok(())) => ToolResult::text(format!( + Ok(()) => ToolResult::text(format!( "✅ Posted drag ({button_str}) to pid {pid} \ from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \ in {duration_ms}ms / {steps} steps." )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), + Err(e) => ToolResult::error(e.to_string()), } } } @@ -1552,6 +1745,8 @@ impl Tool for MouseButtonDownTool { Does not release the button; pair with mouse_drag / mouse_button_up. \ Returns the current held-button state.".into(), input_schema: json!({"type":"object","required":["pid","window_id","x","y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1564,11 +1759,12 @@ impl Tool for MouseButtonDownTool { } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; - if self.state.mouse_hold.lock().unwrap().is_some() { - let held = self.state.mouse_hold.lock().unwrap().clone(); - return ToolResult::error("A mouse button is already held. Call mouse_button_up first.") - .with_structured(mouse_hold_json(held.as_ref())); + let cursor_id = resolve_cursor_key(&args); + if let Some(held) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() { + return ToolResult::error(format!( + "Cursor '{cursor_id}' already has a held mouse button. Call mouse_button_up first." + )) + .with_structured(mouse_hold_json(&cursor_id, Some(&held))); } let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; @@ -1590,10 +1786,16 @@ impl Tool for MouseButtonDownTool { y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let xi = x as i32; @@ -1601,18 +1803,28 @@ impl Tool for MouseButtonDownTool { let result = tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await; match result { Ok(Ok(())) => { - let hold = MouseHoldState { pid, xid, button, x, y }; - *self.state.mouse_hold.lock().unwrap() = Some(hold.clone()); + let hold = MouseHoldState { cursor_id: cursor_id.clone(), pid, xid, button, x, y }; + self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone()); + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await + { + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); + } ToolResult::text(format!( - "✅ Held {} button down at ({x:.1}, {y:.1}).", - mouse_button_name(button) + "✅ Cursor '{cursor_id}' held {} button down at ({x:.1}, {y:.1}).", + mouse_button_name(button), )) - .with_structured(mouse_hold_json(Some(&hold))) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) } Ok(Err(e)) => ToolResult::error(e.to_string()) - .with_structured(mouse_hold_json(self.state.mouse_hold.lock().unwrap().as_ref())), + .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))), Err(e) => ToolResult::error(format!("Task error: {e}")) - .with_structured(mouse_hold_json(self.state.mouse_hold.lock().unwrap().as_ref())), + .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))), } } } @@ -1631,6 +1843,8 @@ impl Tool for MouseDragTool { Requires an active mouse_button_down state; does not release the button. \ Returns the updated held-button state.".into(), input_schema: json!({"type":"object","required":["x","y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1644,11 +1858,16 @@ impl Tool for MouseDragTool { } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; - let Some(mut hold) = self.state.mouse_hold.lock().unwrap().clone() else { - return ToolResult::error("No mouse button is currently held. Call mouse_button_down first.") - .with_structured(mouse_hold_json(None)); + let cursor_id = resolve_cursor_key(&args); + let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else { + return ToolResult::error(format!( + "No mouse button is currently held for cursor '{cursor_id}'. Call mouse_button_down first." + )) + .with_structured(mouse_hold_json(&cursor_id, None)); }; + if let Some(err) = held_target_mismatch(&args, &cursor_id, &hold) { + return err; + } let mut to_x = args.f64_or("x", 0.0); let mut to_y = args.f64_or("y", 0.0); @@ -1656,65 +1875,96 @@ impl Tool for MouseDragTool { match self.state.zoom_registry.get(hold.pid) { Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(to_x, to_y); to_x = wx; to_y = wy; } None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid)) - .with_structured(mouse_hold_json(Some(&hold))), + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), } } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { to_x *= ratio; to_y *= ratio; } - let xid = args.opt_u64("window_id").unwrap_or(hold.xid); - if xid != hold.xid { - return ToolResult::error(format!( - "mouse_drag window_id {xid} does not match held window {}.", - hold.xid - )) - .with_structured(mouse_hold_json(Some(&hold))); - } + let xid = hold.xid; let from_x = hold.x; let from_y = hold.y; let duration_ms = args.u64_or("duration_ms", 500); let steps = args.u64_or("steps", 20).max(1) as usize; - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await { - overlay_glide_to(sx, sy).await; + overlay_glide_to_for(&cursor_id, sx, sy).await; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); } let button = hold.button; - let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; - for i in 1..=steps { - let t = i as f64 / steps as f64; - let ix = from_x + (to_x - from_x) * t; - let iy = from_y + (to_y - from_y) * t; - crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))?; - if step_delay_ms > 0 { - std::thread::sleep(std::time::Duration::from_millis(step_delay_ms)); + let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; + let mut result: anyhow::Result<()> = Ok(()); + let mut prev_x = from_x; + let mut prev_y = from_y; + for i in 1..=steps { + let t = i as f64 / steps as f64; + let ix = from_x + (to_x - from_x) * t; + let iy = from_y + (to_y - from_y) * t; + let move_result = tokio::task::spawn_blocking(move || { + crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button)) + }).await; + match move_result { + Ok(Ok(())) => { + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await + { + let heading = if (ix - prev_x).abs() > f64::EPSILON || (iy - prev_y).abs() > f64::EPSILON { + Some((iy - prev_y).atan2(ix - prev_x)) + } else { + None + }; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_move_to_for(&cursor_id, sx, sy, heading); + } + prev_x = ix; + prev_y = iy; + if step_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms)).await; + } + } + Ok(Err(e)) => { + result = Err(e); + break; + } + Err(e) => { + result = Err(anyhow::anyhow!("Task error: {e}")); + break; } } - Ok(()) - }).await; + } match result { - Ok(Ok(())) => { + Ok(()) => { hold.x = to_x; hold.y = to_y; - *self.state.mouse_hold.lock().unwrap() = Some(hold.clone()); + self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone()); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, Some((to_y - from_y).atan2(to_x - from_x))); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } ToolResult::text(format!( - "✅ Dragged held {} button to ({to_x:.1}, {to_y:.1}).", - mouse_button_name(hold.button) + "✅ Cursor '{cursor_id}' dragged held {} button to ({to_x:.1}, {to_y:.1}).", + mouse_button_name(hold.button), )) - .with_structured(mouse_hold_json(Some(&hold))) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) } - Ok(Err(e)) => ToolResult::error(e.to_string()) - .with_structured(mouse_hold_json(Some(&hold))), - Err(e) => ToolResult::error(format!("Task error: {e}")) - .with_structured(mouse_hold_json(Some(&hold))), + Err(e) => ToolResult::error(e.to_string()) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), } } } @@ -1732,6 +1982,8 @@ impl Tool for MouseButtonUpTool { description: "Release a previously-held mouse button via background X11 delivery. \ If x/y are omitted, releases at the last held position. Returns the current held-button state.".into(), input_schema: json!({"type":"object","properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1743,37 +1995,36 @@ impl Tool for MouseButtonUpTool { } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; - let Some(mut hold) = self.state.mouse_hold.lock().unwrap().clone() else { - return ToolResult::error("No mouse button is currently held.") - .with_structured(mouse_hold_json(None)); + let cursor_id = resolve_cursor_key(&args); + let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else { + return ToolResult::error(format!("No mouse button is currently held for cursor '{cursor_id}'.")) + .with_structured(mouse_hold_json(&cursor_id, None)); }; - - let xid = args.opt_u64("window_id").unwrap_or(hold.xid); - if xid != hold.xid { - return ToolResult::error(format!( - "mouse_button_up window_id {xid} does not match held window {}.", - hold.xid - )) - .with_structured(mouse_hold_json(Some(&hold))); + if let Some(err) = held_target_mismatch(&args, &cursor_id, &hold) { + return err; } + let xid = hold.xid; + let mut x = args.opt_f64("x").unwrap_or(hold.x); let mut y = args.opt_f64("y").unwrap_or(hold.y); if args.bool_or("from_zoom", false) { match self.state.zoom_registry.get(hold.pid) { Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid)) - .with_structured(mouse_hold_json(Some(&hold))), + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), } } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { x *= ratio; y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; + overlay_glide_to_for(&cursor_id, sx, sy).await; } let button = hold.button; @@ -1784,18 +2035,167 @@ impl Tool for MouseButtonUpTool { Ok(Ok(())) => { hold.x = x; hold.y = y; - *self.state.mouse_hold.lock().unwrap() = None; - let cleared = mouse_hold_json(None); + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await + { + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(false), + ); + self.state.mouse_hold.lock().unwrap().remove(&cursor_id); + let cleared = mouse_hold_json(&cursor_id, None); ToolResult::text(format!( - "✅ Released held {} button at ({x:.1}, {y:.1}).", - mouse_button_name(button) + "✅ Cursor '{cursor_id}' released held {} button at ({x:.1}, {y:.1}).", + mouse_button_name(button), )) .with_structured(cleared) } Ok(Err(e)) => ToolResult::error(e.to_string()) - .with_structured(mouse_hold_json(Some(&hold))), + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), Err(e) => ToolResult::error(format!("Task error: {e}")) - .with_structured(mouse_hold_json(Some(&hold))), + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + } + } +} + +pub struct ParallelMouseDragTool { + state: Arc, +} +static PMDRAG_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for ParallelMouseDragTool { + fn def(&self) -> &ToolDef { + PMDRAG_DEF.get_or_init(|| ToolDef { + name: "parallel_mouse_drag".into(), + description: "Run multiple mouse press-drag-release gestures concurrently via Linux MPX/XI2 virtual master pointers. \ + Each drag item is executed on its own session-scoped master pointer, allowing true same-window concurrent line draws on X11.".into(), + input_schema: json!({"type":"object","required":["drags"],"properties":{ + "drags":{"type":"array","minItems":2,"items":{"type":"object","required":["session","window_id","from_x","from_y","to_x","to_y"],"properties":{ + "session":{"type":"string","description":"Session/cursor id; also keys the virtual master pointer."}, + "window_id":{"type":"integer"}, + "from_x":{"type":"number"}, + "from_y":{"type":"number"}, + "to_x":{"type":"number"}, + "to_y":{"type":"number"}, + "button":{"type":"string","enum":["left","right","middle"],"description":"Default: left."}, + "duration_ms":{"type":"integer","minimum":0,"maximum":10000,"description":"Default: 500."}, + "steps":{"type":"integer","minimum":1,"maximum":300,"description":"Default: 20."} + },"additionalProperties":false}} + },"additionalProperties":false}), + read_only: false, destructive: true, idempotent: false, open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await { + Ok(Ok(())) => {} + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + } + + let Some(items) = args.get("drags").and_then(|v| v.as_array()) else { + return ToolResult::error("drags[] is required."); + }; + if items.len() < 2 { + return ToolResult::error("parallel_mouse_drag requires at least two drag items."); + } + + let mut drags = Vec::with_capacity(items.len()); + for item in items { + let Some(session) = item.get("session").and_then(|v| v.as_str()) else { + return ToolResult::error("each drag item requires session."); + }; + let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else { + return ToolResult::error("each drag item requires window_id."); + }; + let Some(from_x) = item.get("from_x").and_then(|v| v.as_f64()) else { + return ToolResult::error("each drag item requires from_x."); + }; + let Some(from_y) = item.get("from_y").and_then(|v| v.as_f64()) else { + return ToolResult::error("each drag item requires from_y."); + }; + let Some(to_x) = item.get("to_x").and_then(|v| v.as_f64()) else { + return ToolResult::error("each drag item requires to_x."); + }; + let Some(to_y) = item.get("to_y").and_then(|v| v.as_f64()) else { + return ToolResult::error("each drag item requires to_y."); + }; + + let button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left")); + let duration_ms = item.get("duration_ms").and_then(|v| v.as_u64()).unwrap_or(500); + let steps = item.get("steps").and_then(|v| v.as_u64()).unwrap_or(20).max(1) as usize; + + let from = match tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await { + Ok(Ok(coords)) => coords, + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + }; + let to = match tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await { + Ok(Ok(coords)) => coords, + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + }; + + self.state.cursor_registry.update_position(session, from.0, from.1); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo { + x: from.0, + y: from.1, + heading_radians: None, + }); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(true)); + + drags.push(( + session.to_owned(), + crate::input::VirtualPointerDrag { + target_window: xid, + button, + from_x: from.0.round() as i32, + from_y: from.1.round() as i32, + to_x: to.0.round() as i32, + to_y: to.1.round() as i32, + duration_ms, + steps, + }, + )); + } + + let drags_for_task = drags.clone(); + let result = tokio::task::spawn_blocking(move || crate::input::send_parallel_virtual_pointer_drags(&drags_for_task)).await; + match result { + Ok(Ok(())) => { + for (session, drag) in &drags { + self.state.cursor_registry.update_position(session, drag.to_x as f64, drag.to_y as f64); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo { + x: drag.to_x as f64, + y: drag.to_y as f64, + heading_radians: Some(((drag.to_y - drag.from_y) as f64).atan2((drag.to_x - drag.from_x) as f64)), + }); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { + x: drag.to_x as f64, + y: drag.to_y as f64, + }); + } + ToolResult::text(format!("✅ Ran {} MPX drag gesture(s) concurrently.", drags.len())) + .with_structured(json!({"count": drags.len()})) + } + Ok(Err(e)) => { + for (session, _) in &drags { + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + } + ToolResult::error(e.to_string()) + } + Err(e) => { + for (session, _) in &drags { + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + } + ToolResult::error(format!("Task error: {e}")) + } } } } @@ -1894,7 +2294,7 @@ impl Tool for MoveCursorTool { name: "move_cursor".into(), description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor.".into(), input_schema: json!({"type":"object","required":["x","y"],"properties":{ - "x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"} + "x":{"type":"number"},"y":{"type":"number"},"session":{"type":"string"},"cursor_id":{"type":"string"} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: false, }) @@ -1903,14 +2303,19 @@ impl Tool for MoveCursorTool { use cua_driver_core::tool_args::ArgsExt; let x = args.f64_or("x", 0.0); let y = args.f64_or("y", 0.0); - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.update_position(&cursor_id, x, y); // End pointing upper-left (45°) — matches Swift's // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so the // overlay arrow settles to the natural macOS-style pose. - crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { - x, y, end_heading_radians: std::f64::consts::FRAC_PI_4, - }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::MoveTo { + x, + y, + end_heading_radians: std::f64::consts::FRAC_PI_4, + }, + ); ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).")) } } @@ -1930,7 +2335,7 @@ impl Tool for SetAgentCursorEnabledTool { name: "set_agent_cursor_enabled".into(), description: "Show or hide the agent cursor overlay.".into(), input_schema: json!({"type":"object","required":["enabled"],"properties":{ - "enabled":{"type":"boolean"},"cursor_id":{"type":"string"} + "enabled":{"type":"boolean"},"session":{"type":"string"},"cursor_id":{"type":"string"} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: false, }) @@ -1938,9 +2343,12 @@ impl Tool for SetAgentCursorEnabledTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let enabled = match args.require_bool("enabled") { Ok(v) => v, Err(e) => return e }; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.set_enabled(&cursor_id, enabled); - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetEnabled(enabled), + ); ToolResult::text(format!("Agent cursor '{cursor_id}' {}.", if enabled { "enabled" } else { "disabled" })) } } @@ -1967,6 +2375,7 @@ impl Tool for SetAgentCursorMotionTool { - cursor_opacity: 0.0–1.0 (default=0.85)".into(), input_schema: json!({ "type":"object","properties":{ + "session":{"type":"string"}, "cursor_id":{"type":"string"}, "cursor_icon":{"type":"string"}, "cursor_color":{"type":"string"}, @@ -1979,8 +2388,7 @@ impl Tool for SetAgentCursorMotionTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.update_config(&cursor_id, |cfg| { if let Some(v) = args.opt_str("cursor_icon") { cfg.cursor_icon = Some(v); } if let Some(v) = args.opt_str("cursor_color") { cfg.cursor_color = Some(v); } @@ -2006,12 +2414,17 @@ impl Tool for GetAgentCursorStateTool { GCSTATE_DEF.get_or_init(|| ToolDef { name: "get_agent_cursor_state".into(), description: "Return the current state of all agent cursor instances.".into(), - input_schema: json!({"type":"object","properties":{},"additionalProperties":false}), + input_schema: json!({"type":"object","properties":{"session":{"type":"string"},"cursor_id":{"type":"string"}},"additionalProperties":false}), read_only: true, destructive: false, idempotent: true, open_world: false, }) } - async fn invoke(&self, _args: Value) -> ToolResult { - let states = self.state.cursor_registry.all_states(); + async fn invoke(&self, args: Value) -> ToolResult { + let cursor_id = resolve_cursor_key(&args); + let states = if args.get("session").is_some() || args.get("cursor_id").is_some() { + vec![self.state.cursor_registry.get_or_create(&cursor_id)] + } else { + self.state.cursor_registry.all_states() + }; let json = serde_json::to_value(&states).unwrap_or_default(); ToolResult::text(format!("{} cursor instance(s).", states.len())) .with_structured(json!({ "cursors": json })) @@ -2046,6 +2459,10 @@ impl Tool for SetAgentCursorStyleTool { input_schema: json!({ "type": "object", "properties": { + "session": { + "type": "string", + "description": "Optional multi-cursor session id; takes precedence over cursor_id." + }, "cursor_id": { "type": "string", "description": "Cursor instance. Default: 'default'." @@ -2072,7 +2489,7 @@ impl Tool for SetAgentCursorStyleTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); // image_path let image_path = args.get("image_path").and_then(|v| v.as_str()); @@ -2131,15 +2548,18 @@ impl Tool for SetAgentCursorStyleTool { // Dispatch to overlay if let Some(cmd) = shape_cmd { - crate::overlay::send_command(cmd); + crate::overlay::send_command_for(cursor_id.clone(), cmd); } let gradient_provided = args.get("gradient_colors").is_some(); let bloom_provided = args.get("bloom_color").is_some(); if gradient_provided || bloom_provided { - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient { - gradient_colors, - bloom_color: bloom_color.flatten(), - }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetGradient { + gradient_colors, + bloom_color: bloom_color.flatten(), + }, + ); } let grad_str = args.get("gradient_colors") @@ -2611,6 +3031,16 @@ impl Tool for BringToFrontTool { pub fn build_registry(compat: bool) -> ToolRegistry { let state = ToolState::new(); + { + let cursor_registry = state.cursor_registry.clone(); + let state_for_session_end = state.clone(); + cua_driver_core::session::register_session_end_hook(move |session_id| { + cursor_registry.remove(session_id); + crate::overlay::remove_cursor(session_id.to_owned()); + state_for_session_end.mouse_hold.lock().unwrap().remove(session_id); + crate::input::forget_master_pointer(session_id); + }); + } let mut r = ToolRegistry::new(); r.register(Box::new(ListAppsTool)); r.register(Box::new(ListWindowsTool)); @@ -2625,6 +3055,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry { r.register(Box::new(MouseButtonDownTool { state: state.clone() })); r.register(Box::new(MouseDragTool { state: state.clone() })); r.register(Box::new(MouseButtonUpTool { state: state.clone() })); + r.register(Box::new(ParallelMouseDragTool { state: state.clone() })); r.register(Box::new(TypeTextTool)); r.register(Box::new(PressKeyTool)); r.register(Box::new(HotkeyTool)); diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix index d34ffb3302..1a338075e8 100644 --- a/nix/cua-driver/package.nix +++ b/nix/cua-driver/package.nix @@ -25,7 +25,7 @@ pkgs.rustPlatform.buildRustPackage { # gracefully via `cargo vendor`. # Bumped when the dependency set changes (added `atspi`/zbus for native # AT-SPI). If this mismatches, the nix build prints the expected value. - cargoHash = "sha256-P+f+ma8ZDWhhk1TTCGgbLTp4zU/uuh4vHYYQMIjlCbU="; + cargoHash = "sha256-3oz8KeW8a6ak8uOLqPCmb4Sf59f2c4NXr6PTti8eS/Q="; # Build only the main binary crate. The workspace also contains # platform-macos, platform-windows, cua-driver-uia, and focus-monitor-win @@ -34,13 +34,20 @@ pkgs.rustPlatform.buildRustPackage { cargoBuildFlags = [ "-p" "cua-driver" ]; cargoTestFlags = [ "-p" "cua-driver" ]; - # The entire Linux dependency chain is pure Rust: + # Mostly pure Rust: # x11rb -> RustConnection (no libxcb C binding) # ureq -> rustls (no openssl) # tiny-skia -> pure Rust 2D graphics # ring -> compiles own C/asm via stdenv's cc - nativeBuildInputs = [ ]; - buildInputs = [ ]; + # Except the `x11` crate (raw Xlib FFI for MPX multi-cursor drags), whose + # build.rs locates libX11/libXi/libXtst via pkg-config. + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = with pkgs; [ + libx11 + libxi + libxtst + libxext + ]; # Skip tests that require a running X11 display or AT-SPI bus doCheck = false;