diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 8d090365e0..fc7d24198c 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -55,6 +55,17 @@ jobs: visual: true result_link: result-linux-background-terminal-gif artifact_name: cua-driver-linux-background-terminal-gif + - name: Linux parallel multi-cursor drag test + check_attr: cua-driver-linux-parallel-drag-xserver + timeout_minutes: 20 + visual: true + result_link: result-linux-parallel-drag-xserver + artifact_name: cua-driver-linux-parallel-drag-xserver + # NOTE: the older "parallel multi-cursor drag GIF" scenario + # (linux-parallel-drag-gif.nix) is intentionally not in this matrix — + # it hand-launches Xorg, which can't get a VT/seat in the emulated + # nixos-test VM. The services.xserver entry above supersedes it; the + # old scenario file is kept for local/real-X runs. # Full entries (CDP / Tk focus-free-write overrides) — kept as-is. - name: Linux background GUI test (chromium) check_attr: cua-driver-linux-background-gui-chromium @@ -278,6 +289,7 @@ jobs: const artifactNames = [ 'cua-driver-linux-cursor-click-gif', 'cua-driver-linux-background-terminal-gif', + 'cua-driver-linux-parallel-drag-xserver', 'cua-driver-linux-background-gui-chromium', 'cua-driver-linux-background-gui-tk', 'cua-driver-linux-background-gui-gtk3-gedit', diff --git a/flake.nix b/flake.nix index f3425c74ef..dbf8b5ce4c 100644 --- a/flake.nix +++ b/flake.nix @@ -79,6 +79,29 @@ services.cua-driver.package = cuaDriverPackage; }; }; + + # Multi-cursor (MPX) parallel-drag test on a REAL Xorg brought up + # by NixOS services.xserver (dummy video + libinput, on a seat via + # a display manager). This is the CI-viable replacement for the + # hand-launched-Xorg linux-parallel-drag-gif.nix (which timed out + # because a self-launched Xorg couldn't get a VT/seat in the + # emulated nixos-test VM). Proves uinput slaves enumerate as X + # devices, two cursors draw concurrent window-targeted events, and + # the shield grab keeps focus off the drag. + cua-driver-linux-parallel-drag-xserver = import ./nix/cua-driver/tests/linux-parallel-drag-xserver.nix { + inherit pkgs; + inherit (pkgs) lib; + cuaDriverModule = { + imports = [ ./nix/cua-driver/module.nix ]; + services.cua-driver.package = cuaDriverPackage; + }; + }; + + # NOTE: cua-driver-linux-parallel-drag-gif (nix/cua-driver/tests/ + # linux-parallel-drag-gif.nix) is intentionally NOT a flake check — + # it hand-launches Xorg, which can't get a VT/seat in the emulated + # GHA nixos-test VM. It is superseded by the services.xserver test + # above and kept only for local/real-X manual runs. } // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( # Background GUI input coverage — one independent matrix job per diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 751f8f6e3d..2fccd8ebd2 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -265,6 +265,18 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -680,6 +692,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "evdev" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6055a93a963297befb0f4f6e18f314aec9767a4bbe88b151126df2433610a7" +dependencies = [ + "bitvec", + "cfg-if", + "libc", + "nix", + "thiserror", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -748,6 +773,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "focus-monitor-win" version = "0.5.1" @@ -824,6 +855,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-core" version = "0.3.32" @@ -1222,6 +1259,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -1231,6 +1277,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "meval" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79496a5651c8d57cd033c5add8ca7ee4e3d5f7587a4777484640d9cb60392d9" +dependencies = [ + "fnv", + "nom", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1279,6 +1335,25 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nom" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1544,8 +1619,10 @@ dependencies = [ "base64", "cua-driver-core", "cursor-overlay", + "evdev", "image", "libc", + "meval", "pip-preview", "serde", "serde_json", @@ -1553,6 +1630,7 @@ dependencies = [ "tiny-skia", "tokio", "tracing", + "x11", "x11rb", ] @@ -1733,6 +1811,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.6" @@ -2177,6 +2261,12 @@ dependencies = [ "syn", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.45" @@ -2536,7 +2626,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ - "memoffset", + "memoffset 0.9.1", "tempfile", "windows-sys 0.61.2", ] @@ -3312,6 +3402,25 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x11rb" version = "0.13.2" diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/ffmpeg_install.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/ffmpeg_install.rs new file mode 100644 index 0000000000..c02646973b --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/ffmpeg_install.rs @@ -0,0 +1,141 @@ +//! Best-effort ffmpeg installer for the `install_ffmpeg` tool. +//! +//! The Linux/Windows video-recording backend shells out to the ffmpeg +//! *binary* (macOS records natively via ScreenCaptureKit and needs no +//! ffmpeg). When ffmpeg is absent, `start_recording(record_video: true)` +//! can't produce an mp4. This module detects the platform package manager, +//! reports the exact install command, and — only on explicit confirmation — +//! runs it. We never link ffmpeg; this installs the same user-provided +//! binary the subprocess backend already looks for via `find_ffmpeg`. + +use std::process::Command; + +/// A resolved install action: a human-readable manager name + the argv to run. +pub struct InstallPlan { + pub manager: String, + pub argv: Vec, +} + +impl InstallPlan { + pub fn display(&self) -> String { + self.argv.join(" ") + } +} + +/// Is `name` an executable on PATH? +fn cmd_exists(name: &str) -> bool { + #[cfg(target_os = "windows")] + let probe = Command::new("where").arg(name).output(); + #[cfg(not(target_os = "windows"))] + let probe = Command::new("sh") + .arg("-c") + .arg(format!("command -v {name}")) + .output(); + probe.map(|o| o.status.success()).unwrap_or(false) +} + +#[cfg(target_os = "linux")] +fn is_root() -> bool { + // Avoid a libc dependency: ask `id -u`. + Command::new("id") + .arg("-u") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim() == "0") + .unwrap_or(false) +} + +/// Detect how to install ffmpeg on this platform, or `None` if no supported +/// package manager is available (caller should then tell the user to install +/// ffmpeg manually). +pub fn install_plan() -> Option { + #[cfg(target_os = "linux")] + { + // (manager binary, install argv) — first match on PATH wins. + let candidates: &[(&str, &[&str])] = &[ + ("apt-get", &["apt-get", "install", "-y", "ffmpeg"]), + ("dnf", &["dnf", "install", "-y", "ffmpeg"]), + ("yum", &["yum", "install", "-y", "ffmpeg"]), + ("zypper", &["zypper", "--non-interactive", "install", "ffmpeg"]), + ("pacman", &["pacman", "-S", "--noconfirm", "ffmpeg"]), + ("apk", &["apk", "add", "ffmpeg"]), + ("snap", &["snap", "install", "ffmpeg"]), + ]; + for (bin, argv) in candidates { + if !cmd_exists(bin) { + continue; + } + let mut full: Vec = Vec::new(); + // System package managers need root. If we're not root and sudo + // is available, run it non-interactively — a password-required + // sudo fails fast rather than hanging the (TTY-less) daemon. + if !is_root() && cmd_exists("sudo") { + full.push("sudo".into()); + full.push("-n".into()); + } + full.extend(argv.iter().map(|s| (*s).to_string())); + return Some(InstallPlan { + manager: (*bin).into(), + argv: full, + }); + } + None + } + #[cfg(target_os = "macos")] + { + if cmd_exists("brew") { + return Some(InstallPlan { + manager: "brew".into(), + argv: vec!["brew".into(), "install".into(), "ffmpeg".into()], + }); + } + None + } + #[cfg(target_os = "windows")] + { + if cmd_exists("winget") { + return Some(InstallPlan { + manager: "winget".into(), + argv: [ + "winget", "install", "-e", "--id", "Gyan.FFmpeg", + "--accept-package-agreements", "--accept-source-agreements", + ] + .iter() + .map(|s| (*s).to_string()) + .collect(), + }); + } + if cmd_exists("choco") { + return Some(InstallPlan { + manager: "choco".into(), + argv: ["choco", "install", "ffmpeg", "-y"] + .iter() + .map(|s| (*s).to_string()) + .collect(), + }); + } + None + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + { + None + } +} + +/// Run an install plan. Returns `(command_succeeded, combined_output_tail)`. +pub fn run_install(plan: &InstallPlan) -> anyhow::Result<(bool, String)> { + let out = Command::new(&plan.argv[0]) + .args(&plan.argv[1..]) + .output() + .map_err(|e| anyhow::anyhow!("failed to spawn `{}`: {e}", plan.argv[0]))?; + let mut buf = String::new(); + buf.push_str(&String::from_utf8_lossy(&out.stdout)); + buf.push_str(&String::from_utf8_lossy(&out.stderr)); + let tail = if buf.len() > 3000 { + format!("…{}", &buf[buf.len() - 3000..]) + } else { + buf + }; + Ok((out.status.success(), tail)) +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs index 8243c26f1a..db9c2e8e37 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod cdp; pub mod element_cache; +pub mod ffmpeg_install; pub mod image_utils; pub mod page; pub mod pip_hook; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs index 7493ea2582..c0f87d3a1b 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs @@ -137,7 +137,13 @@ impl Tool for StartRecordingTool { " (video → recording.mp4)".to_string() } else if video_failed { let err = state.last_error.clone().unwrap_or_else(|| "unknown".into()); - format!("\n\n⚠️ Video capture failed (per-turn JSON+screenshot still running):\n{err}") + let hint = if crate::video_ffmpeg::find_ffmpeg().is_none() { + "\n\nffmpeg was not found. Call install_ffmpeg (then again with \ + confirm=true) to install it, then restart recording." + } else { + "" + }; + format!("\n\n⚠️ Video capture failed (per-turn JSON+screenshot still running):\n{err}{hint}") } else { String::new() }; let msg = format!("✅ Recording started -> {}{}", state.output_dir.as_deref().unwrap_or("?"), @@ -475,3 +481,91 @@ fn parse_action_json(path: &std::path::Path) -> anyhow::Result<(String, Value)> let tool_args = obj.get("arguments").cloned().unwrap_or(Value::Object(Default::default())); Ok((tool, tool_args)) } + +// ── install_ffmpeg ──────────────────────────────────────────────────────────── +// +// Confirmation-gated installer for the ffmpeg binary that the Linux/Windows +// video backend shells out to. Called without `confirm` it only REPORTS the +// command it would run (read-only preview); `confirm: true` runs it. Marked +// destructive + open_world so conforming MCP clients also gate it behind a +// human approval. ffmpeg is invoked as a separate process, never linked. + +pub struct InstallFfmpegTool; +static INSTALL_FFMPEG_DEF: OnceLock = OnceLock::new(); + +#[async_trait] +impl Tool for InstallFfmpegTool { + fn def(&self) -> &ToolDef { + INSTALL_FFMPEG_DEF.get_or_init(|| ToolDef { + name: "install_ffmpeg".into(), + description: "Install the ffmpeg binary used by start_recording's video \ + capture (Linux/Windows; macOS records natively and needs no ffmpeg). \ + Two-step and confirmed: called without `confirm` it only REPORTS the \ + exact install command for this platform's package manager; pass \ + `confirm: true` to actually run it. No-op if ffmpeg is already on PATH. \ + ffmpeg is run as a separate process, never linked into the driver." + .into(), + input_schema: json!({"type":"object","properties":{ + "confirm":{"type":"boolean","description":"Run the install command. Without it, only the planned command is reported."} + },"additionalProperties":false}), + read_only: false, + destructive: true, + idempotent: false, + open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + use crate::tool_args::ArgsExt; + + if let Some(path) = crate::video_ffmpeg::find_ffmpeg() { + return ToolResult::text(format!( + "✅ ffmpeg already available ({}). Nothing to install.", + path.display() + )) + .with_structured(json!({ + "installed": true, "ran": false, "path": path.display().to_string() + })); + } + + let Some(plan) = crate::ffmpeg_install::install_plan() else { + return ToolResult::error( + "ffmpeg is not installed and no supported package manager was found to \ + install it automatically. Install ffmpeg manually and put it on PATH \ + (Linux: apt/dnf/pacman/zypper/apk/snap; macOS: `brew install ffmpeg`; \ + Windows: `winget install Gyan.FFmpeg`).", + ); + }; + + if !args.bool_or("confirm", false) { + return ToolResult::text(format!( + "ffmpeg is not installed. To install it via {}, re-call install_ffmpeg \ + with confirm=true.\n\nCommand that will run:\n {}", + plan.manager, + plan.display() + )) + .with_structured(json!({ + "installed": false, "ran": false, + "manager": plan.manager, "command": plan.display() + })); + } + + let display = plan.display(); + let result = tokio::task::spawn_blocking(move || crate::ffmpeg_install::run_install(&plan)).await; + match result { + Ok(Ok((cmd_ok, output))) => match crate::video_ffmpeg::find_ffmpeg() { + Some(path) => ToolResult::text(format!("✅ ffmpeg installed via `{display}`.")) + .with_structured(json!({ + "installed": true, "ran": true, + "command": display, "path": path.display().to_string() + })), + None => ToolResult::error(format!( + "Ran the install command but ffmpeg is still not found.\n\ + Command: {display}\ncommand_succeeded={cmd_ok}\nOutput tail:\n{output}" + )), + }, + Ok(Err(e)) => ToolResult::error(format!("ffmpeg install failed: {e}\nCommand: {display}")), + Err(e) => ToolResult::error(format!("install task error: {e}")), + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs index 7ec6ce17fa..926ac40411 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -85,6 +85,7 @@ impl ToolRegistry { self.register(Box::new(StopRecordingTool::new(session.clone()))); self.register(Box::new(GetRecordingStateTool::new(session))); self.register(Box::new(ReplayTrajectoryTool)); + self.register(Box::new(crate::recording_tools::InstallFfmpegTool)); } /// Register the platform-independent session-lifecycle tools diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs index ae2f4cf3e9..7d395b8e88 100644 --- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs +++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs @@ -283,8 +283,12 @@ pub enum OverlayMsg { pub enum OverlayCommand { /// Animate the cursor to a new screen position. MoveTo { x: f64, y: f64, end_heading_radians: f64 }, + /// Snap the cursor immediately to a screen position, optionally updating heading. + SnapTo { x: f64, y: f64, heading_radians: Option }, /// Start the click-press visual. ClickPulse { x: f64, y: f64 }, + /// Toggle the held-button visual state. + SetPressed(bool), /// Show or hide the overlay. SetEnabled(bool), /// Update the motion/timing config live. diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs index 3d3579a054..34a07183a2 100644 --- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs +++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs @@ -9,7 +9,7 @@ //! //! - [`RenderStateCore`] — the platform-agnostic animation fields //! (`cfg`, `palette`, `motion`, `pos`, `heading`, `path`, `dist`, `spring`, -//! `spring_tgt`, `click_t`, `shape`, `visible`, `idle_secs`, `idle_alpha`, +//! `spring_tgt`, `click_t`, `pressed`, `shape`, `visible`, `idle_secs`, `idle_alpha`, //! `pinned_wid`, `gradient_colors`, `bloom_override`). //! - [`RenderStateCore::tick_motion`] — speed-profile + spring physics + //! click-pulse + idle-fade using runtime [`MotionConfig`] (Windows + Linux). @@ -68,6 +68,8 @@ pub struct RenderStateCore { pub spring_tgt: Option<(f64, f64, f64)>, /// Click-pulse phase 0..1; `None` = no pulse in flight. pub click_t: Option, + /// Whether a button is currently being held for this cursor. + pub pressed: bool, /// Custom cursor shape; `None` = built-in gradient arrow. pub shape: Option, /// User-controlled visibility. @@ -108,6 +110,7 @@ impl RenderStateCore { spring: None, spring_tgt: None, click_t: None, + pressed: false, visible: true, idle_secs: 0.0, idle_alpha: 1.0, @@ -417,6 +420,23 @@ impl RenderStateCore { self.idle_alpha = 1.0; true } + OverlayCommand::SnapTo { + x, + y, + heading_radians, + } => { + self.pos = (x, y); + if let Some(heading) = heading_radians { + self.heading = heading; + } + self.path = None; + self.dist = 0.0; + self.spring = None; + self.spring_tgt = None; + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } OverlayCommand::ClickPulse { x, y } => { if click_pulse_sentinel_only { // macOS: only snap position on first placement (sentinel state). @@ -438,6 +458,12 @@ impl RenderStateCore { self.idle_alpha = 1.0; true } + OverlayCommand::SetPressed(v) => { + self.pressed = v; + self.idle_secs = 0.0; + self.idle_alpha = 1.0; + true + } OverlayCommand::SetEnabled(v) => { self.visible = v; true @@ -534,7 +560,7 @@ pub fn paint_cursor( let alpha_scale = core.idle_alpha as f32; // --- Bloom (radial gradient behind the arrow) --- - let bloom_r: f32 = 22.0; + let bloom_r: f32 = if core.pressed { 34.0 } else { 22.0 }; // Use runtime bloom_override if set, otherwise fall back to palette. let (br, bg, bb) = if let Some([r, g, b, _]) = core.bloom_override { (r, g, b) @@ -580,6 +606,46 @@ pub fn paint_cursor( pm.fill_rect(r, &bloom_paint, tiny_skia::Transform::identity(), None); } + if core.pressed { + let [pr, pg, pb, _] = core.palette.cursor_mid; + let ring_color = + tiny_skia::Color::from_rgba8(pr, pg, pb, (210.0 * alpha_scale) as u8); + let mut ring_paint = tiny_skia::Paint::default(); + ring_paint.shader = tiny_skia::Shader::SolidColor(ring_color); + ring_paint.anti_alias = true; + let stroke = tiny_skia::Stroke { + width: 3.0, + ..Default::default() + }; + let core_fill = + tiny_skia::Color::from_rgba8(pr, pg, pb, (110.0 * alpha_scale) as u8); + let mut fill_paint = tiny_skia::Paint::default(); + fill_paint.shader = tiny_skia::Shader::SolidColor(core_fill); + fill_paint.anti_alias = true; + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_circle(px as f32, py as f32, 6.5); + if let Some(path) = pb.finish() { + pm.fill_path( + &path, + &fill_paint, + tiny_skia::FillRule::Winding, + tiny_skia::Transform::identity(), + None, + ); + } + let mut pb = tiny_skia::PathBuilder::new(); + pb.push_circle(px as f32, py as f32, 13.0); + if let Some(path) = pb.finish() { + pm.stroke_path( + &path, + &ring_paint, + &stroke, + tiny_skia::Transform::identity(), + None, + ); + } + } + // --- Focus rect highlight (macOS only — others pass None) --- // Cyan glow border + faint fill, matching Swift AgentCursor.showFocusRect. if let Some(fr) = focus_rect { diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml index 1d7cdfa48f..7d530b4874 100644 --- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml +++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml @@ -20,6 +20,7 @@ tiny-skia = { version = "0.11", default-features = false, features = ["std"] } [target.'cfg(target_os = "linux")'.dependencies] # X11 background input + window enumeration x11rb = { version = "0.13", features = ["xinput", "randr", "xfixes", "composite", "shape", "xtest"] } +x11 = { version = "2.21", features = ["xlib", "xinput", "xtest"] } base64 = { workspace = true } image = { workspace = true } # kill(2) for the kill_app tool — SIGKILL via libc::kill. @@ -29,3 +30,6 @@ libc = "0.2" # typelibs at runtime. `tokio` matches the driver's async runtime; `zbus` # re-exports the bus types we need (fdo::DBusProxy for pid resolution). atspi = { version = "0.30", features = ["tokio", "zbus"] } +evdev = "0.12" +# Math-expression evaluator for parallel_mouse_drag `fn` paths (y = f(x)). +meval = "0.2" diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs index 0616d59d6a..adcbad7d7a 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs @@ -11,9 +11,17 @@ //! them, because XTest delivers to the *focused* window and would break the //! no-focus-steal contract. -use anyhow::Result; +use anyhow::{anyhow, bail, Context, Result}; +use std::collections::HashMap; +use std::ffi::{CStr, CString}; +use std::fs; +use std::ptr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::sleep; use std::time::Duration; +use evdev::uinput::VirtualDevice; +use evdev::{AttributeSet, EventType, InputEvent, Key, RelativeAxisType}; use x11rb::connection::Connection; use x11rb::protocol::xproto::*; use x11rb::rust_connection::RustConnection; @@ -21,6 +29,1117 @@ use x11rb::rust_connection::RustConnection; const CLICK_DELAY_MS: u64 = 35; const KEY_DELAY_MS: u64 = 10; +#[derive(Clone, Debug)] +pub struct VirtualPointerDrag { + pub target_window: u64, + pub button: u8, + /// Screen-coordinate waypoints. The pointer presses once at `path[0]`, + /// glides through every waypoint (arc-length interpolated), and releases + /// once at the last point — a single continuous held drag, so a curved + /// path (e.g. a sampled y = f(x)) draws as one smooth stroke rather than + /// a chain of press/release dabs. Must contain >= 2 points. + pub path: Vec<(i32, i32)>, + pub duration_ms: u64, + pub steps: usize, +} + +/// Cumulative segment lengths along `path` and its total length. +fn path_cumulative(path: &[(i32, i32)]) -> (Vec, f64) { + let mut cum = Vec::with_capacity(path.len()); + let mut total = 0.0; + cum.push(0.0); + for w in path.windows(2) { + let dx = (w[1].0 - w[0].0) as f64; + let dy = (w[1].1 - w[0].1) as f64; + total += (dx * dx + dy * dy).sqrt(); + cum.push(total); + } + (cum, total) +} + +/// Sample `y = f(x)` over `[x_from, x_to]` into `samples` window-local +/// waypoints. Evaluated with meval (sin/cos/^/etc.); non-finite outputs +/// (ln of a negative, 1/0, …) are dropped. Errors on a bad expression or +/// fewer than 2 finite points. +pub fn sample_function(expr: &str, x_from: f64, x_to: f64, samples: u64) -> Result> { + let parsed: meval::Expr = expr + .parse() + .map_err(|e| anyhow!("invalid fn '{expr}': {e}"))?; + let f = parsed + .bind("x") + .map_err(|e| anyhow!("fn must be in terms of x: {e}"))?; + let n = samples.max(2); + let mut pts = Vec::with_capacity(n as usize); + for i in 0..n { + let x = x_from + (x_to - x_from) * (i as f64) / ((n - 1) as f64); + let y = f(x); + if x.is_finite() && y.is_finite() { + pts.push((x, y)); + } + } + if pts.len() < 2 { + bail!("fn produced fewer than 2 finite points over the domain"); + } + Ok(pts) +} + +/// Point at arc-length fraction `t` (0..1) along `path`. +fn point_on_path(path: &[(i32, i32)], cum: &[f64], total: f64, t: f64) -> (i32, i32) { + if path.len() == 1 || total <= 0.0 { + return *path.last().unwrap(); + } + let d = t.clamp(0.0, 1.0) * total; + let mut i = match cum.binary_search_by(|v| v.partial_cmp(&d).unwrap_or(std::cmp::Ordering::Less)) { + Ok(i) => i, + Err(i) => i.saturating_sub(1), + }; + if i >= path.len() - 1 { + i = path.len() - 2; + } + let seg = cum[i + 1] - cum[i]; + let f = if seg > 0.0 { (d - cum[i]) / seg } else { 0.0 }; + let x = path[i].0 as f64 + (path[i + 1].0 - path[i].0) as f64 * f; + let y = path[i].1 as f64 + (path[i + 1].1 - path[i].1) as f64 * f; + (x.round() as i32, y.round() as i32) +} + +#[derive(Clone, Copy, Debug)] +struct MasterPointerIds { + pointer_id: i32, + keyboard_id: i32, + slave_pointer_id: i32, +} + +static MPX_POINTERS: OnceLock>> = OnceLock::new(); +static UINPUT_POINTERS: OnceLock>>>> = OnceLock::new(); +static XLIB_THREADS_READY: OnceLock> = OnceLock::new(); +static MPX_NAME_COUNTER: AtomicU64 = AtomicU64::new(1); + +fn mpx_pointers() -> &'static Mutex> { + MPX_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn uinput_pointers() -> &'static Mutex>>> { + UINPUT_POINTERS.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn master_pointer_name(cursor_id: &str) -> String { + let nonce = MPX_NAME_COUNTER.fetch_add(1, Ordering::Relaxed); + format!("CUA {cursor_id} mp-{}-{nonce}", std::process::id()) +} + +fn slave_pointer_name(master_name: &str) -> String { + format!("{master_name} uinput pointer") +} + +fn master_pointer_device_name(master_name: &str) -> String { + format!("{master_name} pointer") +} + +fn master_keyboard_device_name(master_name: &str) -> String { + format!("{master_name} keyboard") +} + +fn open_display() -> Result<*mut x11::xlib::Display> { + match XLIB_THREADS_READY.get_or_init(|| { + let rc = unsafe { x11::xlib::XInitThreads() }; + if rc == 0 { + Err("XInitThreads failed".to_owned()) + } else { + Ok(()) + } + }) { + Ok(()) => {} + Err(err) => bail!("{err}"), + } + let display = unsafe { x11::xlib::XOpenDisplay(ptr::null()) }; + if display.is_null() { + bail!("XOpenDisplay returned null"); + } + Ok(display) +} + +fn xi2_query_devices( + display: *mut x11::xlib::Display, +) -> Result> { + let mut count = 0; + let ptr = unsafe { x11::xinput2::XIQueryDevice(display, x11::xinput2::XIAllDevices, &mut count) }; + if ptr.is_null() { + bail!("XIQueryDevice returned null"); + } + let mut out = Vec::new(); + for i in 0..count { + let info = unsafe { *ptr.add(i as usize) }; + let name = if info.name.is_null() { + String::new() + } else { + unsafe { CStr::from_ptr(info.name) }.to_string_lossy().into_owned() + }; + out.push((info.deviceid, info._use, name)); + } + unsafe { x11::xinput2::XIFreeDeviceInfo(ptr) }; + Ok(out) +} + +fn x_server_vendor(display: *mut x11::xlib::Display) -> String { + let ptr = unsafe { x11::xlib::XServerVendor(display) }; + if ptr.is_null() { + return String::new(); + } + unsafe { CStr::from_ptr(ptr) }.to_string_lossy().into_owned() +} + +fn supports_parallel_pointer_injection(display: *mut x11::xlib::Display) -> Result<()> { + let vendor = x_server_vendor(display); + if vendor.to_ascii_lowercase().contains("tigervnc") { + bail!( + "parallel_mouse_drag is not supported on this X server ('{vendor}'). \ + Xtigervnc exposes only its built-in VNC/XTEST devices, so Linux uinput/libinput \ + pointers cannot become real X input devices here." + ); + } + if is_xtigervnc_process_running() { + let display_name = std::env::var("DISPLAY").unwrap_or_else(|_| "".to_owned()); + bail!( + "parallel_mouse_drag is not supported on display {display_name} because the active X server is Xtigervnc. \ + Xtigervnc exposes only its built-in VNC/XTEST devices, so Linux uinput/libinput pointers \ + cannot become real X input devices in this environment." + ); + } + Ok(()) +} + +fn is_xtigervnc_process_running() -> bool { + let display = std::env::var("DISPLAY").unwrap_or_default(); + // Extract display number from DISPLAY (e.g., ":0" -> "0", "host:1.0" -> "1") + let display_num = display + .rsplit(':') + .next() + .unwrap_or("") + .split('.') + .next() + .unwrap_or("") + .trim(); + if display_num.is_empty() { + return false; + } + // The X server writes its PID to the standard lock file /tmp/.X{N}-lock + let lock_path = format!("/tmp/.X{display_num}-lock"); + let Ok(contents) = fs::read_to_string(&lock_path) else { + return false; + }; + let pid = contents.trim(); + if pid.is_empty() || !pid.bytes().all(|b| b.is_ascii_digit()) { + return false; + } + // Check the executable path of the X server process directly + if let Ok(exe) = fs::read_link(format!("/proc/{pid}/exe")) { + return exe.file_name().and_then(|n| n.to_str()) == Some("Xtigervnc"); + } + // Fallback: check the process name via comm (limited to 15 chars, but "Xtigervnc" fits) + if let Ok(comm) = fs::read_to_string(format!("/proc/{pid}/comm")) { + return comm.trim() == "Xtigervnc"; + } + false +} + +pub fn check_parallel_pointer_support() -> Result<()> { + let display = open_display()?; + let result = supports_parallel_pointer_injection(display); + unsafe { x11::xlib::XCloseDisplay(display) }; + result +} + +fn ensure_master_pointer(cursor_id: &str) -> Result { + if let Some(ids) = mpx_pointers().lock().unwrap().get(cursor_id).copied() { + return Ok(ids); + } + + let display = open_display()?; + let mut major = 2; + let mut minor = 3; + let rc = unsafe { x11::xinput2::XIQueryVersion(display, &mut major, &mut minor) }; + if rc != 0 { + unsafe { x11::xlib::XCloseDisplay(display) }; + bail!("XIQueryVersion failed with status {rc}"); + } + + let base = master_pointer_name(cursor_id); + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + let name = CString::new(base.clone())?; + unsafe { + let add = change.add(); + (*add)._type = x11::xinput2::XIAddMaster; + (*add).name = name.as_ptr() as *mut _; + // Core events stay on so core-only apps (xterm, Tk, …) receive the + // drags too. Note this is not what makes the WM focus the dragged + // window — XI2-aware WMs grab buttons for XIAllMasterDevices — see + // the active-window save/restore in send_parallel_virtual_pointer_drags. + (*add).send_core = 1; + (*add).enable = 1; + } + let rc = unsafe { x11::xinput2::XIChangeHierarchy(display, &mut change, 1) }; + unsafe { + x11::xlib::XSync(display, 0); + } + if rc != 0 { + unsafe { x11::xlib::XCloseDisplay(display) }; + bail!("XIChangeHierarchy(XIAddMaster) failed with status {rc}"); + } + + let devices = xi2_query_devices(display)?; + let mut pointer_id = None; + let mut keyboard_id = None; + let pointer_name = master_pointer_device_name(&base); + let keyboard_name = master_keyboard_device_name(&base); + for (device_id, use_, device_name) in devices { + if use_ == x11::xinput2::XIMasterPointer && device_name == pointer_name { + pointer_id = Some(device_id); + } else if use_ == x11::xinput2::XIMasterKeyboard && device_name == keyboard_name { + keyboard_id = Some(device_id); + } + } + + let pointer_id = pointer_id.ok_or_else(|| anyhow!("failed to locate created master pointer for '{cursor_id}'"))?; + let keyboard_id = keyboard_id.ok_or_else(|| anyhow!("failed to locate created master keyboard for '{cursor_id}'"))?; + + let device_name = slave_pointer_name(&base); + let uinput_device = create_uinput_pointer(&device_name)?; + let slave_pointer_id = wait_for_slave_pointer_id(display, &device_name)?; + attach_slave_to_master(display, slave_pointer_id, pointer_id)?; + set_flat_pointer_accel(display, slave_pointer_id); + unsafe { x11::xlib::XCloseDisplay(display) }; + + let ids = MasterPointerIds { pointer_id, keyboard_id, slave_pointer_id }; + mpx_pointers().lock().unwrap().insert(cursor_id.to_owned(), ids); + uinput_pointers() + .lock() + .unwrap() + .insert(cursor_id.to_owned(), Arc::new(Mutex::new(uinput_device))); + Ok(ids) +} + +pub fn forget_master_pointer(cursor_id: &str) { + uinput_pointers().lock().unwrap().remove(cursor_id); + let Some(ids) = mpx_pointers().lock().unwrap().remove(cursor_id) else { + return; + }; + + let Ok(display) = open_display() else { + return; + }; + + let Ok(devices) = xi2_query_devices(display) else { + unsafe { x11::xlib::XCloseDisplay(display) }; + return; + }; + + let mut virtual_core_pointer = None; + let mut virtual_core_keyboard = None; + for (device_id, use_, device_name) in devices { + if device_name == "Virtual core pointer" && use_ == x11::xinput2::XIMasterPointer { + virtual_core_pointer = Some(device_id); + } else if device_name == "Virtual core keyboard" && use_ == x11::xinput2::XIMasterKeyboard { + virtual_core_keyboard = Some(device_id); + } + } + + let (Some(return_pointer), Some(return_keyboard)) = (virtual_core_pointer, virtual_core_keyboard) else { + unsafe { x11::xlib::XCloseDisplay(display) }; + return; + }; + + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + unsafe { + let remove = change.remove(); + (*remove)._type = x11::xinput2::XIRemoveMaster; + (*remove).deviceid = ids.pointer_id; + (*remove).return_mode = x11::xinput2::XIAttachToMaster; + (*remove).return_pointer = return_pointer; + (*remove).return_keyboard = return_keyboard; + let _ = x11::xinput2::XIChangeHierarchy(display, &mut change, 1); + x11::xlib::XSync(display, 0); + x11::xlib::XCloseDisplay(display); + } +} + +fn create_uinput_pointer(name: &str) -> Result { + let mut keys = AttributeSet::::new(); + keys.insert(Key::BTN_LEFT); + keys.insert(Key::BTN_RIGHT); + keys.insert(Key::BTN_MIDDLE); + + let mut rel_axes = AttributeSet::::new(); + rel_axes.insert(RelativeAxisType::REL_X); + rel_axes.insert(RelativeAxisType::REL_Y); + rel_axes.insert(RelativeAxisType::REL_WHEEL); + + Ok( + evdev::uinput::VirtualDeviceBuilder::new()? + .name(name) + .with_keys(&keys)? + .with_relative_axes(&rel_axes)? + .build()?, + ) +} + +fn wait_for_slave_pointer_id(display: *mut x11::xlib::Display, device_name: &str) -> Result { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + for (device_id, use_, seen_name) in xi2_query_devices(display)? { + if use_ == x11::xinput2::XISlavePointer && seen_name == device_name { + return Ok(device_id); + } + } + if std::time::Instant::now() >= deadline { + bail!("timed out waiting for X input slave pointer '{device_name}'"); + } + sleep(Duration::from_millis(50)); + } +} + +fn attach_slave_to_master(display: *mut x11::xlib::Display, slave_pointer_id: i32, master_pointer_id: i32) -> Result<()> { + let mut change = x11::xinput2::XIAnyHierarchyChangeInfo::default(); + unsafe { + let attach = change.attach(); + (*attach)._type = x11::xinput2::XIAttachSlave; + (*attach).deviceid = slave_pointer_id; + (*attach).new_master = master_pointer_id; + } + let rc = unsafe { x11::xinput2::XIChangeHierarchy(display, &mut change, 1) }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIChangeHierarchy(XIAttachSlave) failed with status {rc}"); + } + Ok(()) +} + +fn set_flat_pointer_accel(display: *mut x11::xlib::Display, slave_pointer_id: i32) { + // Pin libinput's accel profile to flat so relative deltas map 1:1 onto + // cursor movement — the default adaptive profile rescales small deltas + // and makes drag endpoints drift off-target by a few pixels. + // Best-effort: the property only exists under xf86-input-libinput. + unsafe { + let prop = x11::xlib::XInternAtom( + display, + c"libinput Accel Profile Enabled".as_ptr(), + x11::xlib::True, + ); + if prop == 0 { + return; + } + let mut type_ret: x11::xlib::Atom = 0; + let mut format_ret: std::os::raw::c_int = 0; + let mut num_items: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut data: *mut std::os::raw::c_uchar = std::ptr::null_mut(); + let rc = x11::xinput2::XIGetProperty( + display, + slave_pointer_id, + prop, + 0, + 16, + x11::xlib::False, + x11::xlib::AnyPropertyType as x11::xlib::Atom, + &mut type_ret, + &mut format_ret, + &mut num_items, + &mut bytes_after, + &mut data, + ); + if rc != x11::xlib::Success as i32 || data.is_null() { + return; + } + // Profile order is (adaptive, flat[, custom]); enable flat only. + if format_ret == 8 && (2..=8).contains(&num_items) { + let mut values = vec![0u8; num_items as usize]; + values[1] = 1; + x11::xinput2::XIChangeProperty( + display, + slave_pointer_id, + prop, + type_ret, + 8, + x11::xlib::PropModeReplace, + values.as_mut_ptr(), + num_items as std::os::raw::c_int, + ); + x11::xlib::XSync(display, 0); + } + x11::xlib::XFree(data as *mut _); + } +} + +fn warp_master_pointer(display: *mut x11::xlib::Display, ids: MasterPointerIds, x: i32, y: i32) -> Result<()> { + let root = unsafe { x11::xlib::XDefaultRootWindow(display) }; + let rc = unsafe { + x11::xinput2::XIWarpPointer( + display, + ids.pointer_id, + 0, + root, + 0.0, + 0.0, + 0, + 0, + x as f64, + y as f64, + ) + }; + // XSync (not XFlush): the button press that follows is emitted through + // uinput on a separate kernel pipeline, and races ahead of a merely + // queued warp request. Once XSync returns the server has executed the + // warp, so the press lands at the warped position. + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIWarpPointer failed with status {rc}"); + } + Ok(()) +} + +/// XIAnyModifier (1u32 << 31). The x11 crate doesn't export it. +const XI_ANY_MODIFIER: std::os::raw::c_int = 0x8000_0000u32 as std::os::raw::c_int; + +fn xi_mask_len() -> usize { + (x11::xinput2::XI_LASTEVENT as usize >> 3) + 1 +} + +/// Look up the XInputExtension major opcode so we can recognise its +/// GenericEvent cookies on the display connection. +fn xinput_opcode(display: *mut x11::xlib::Display) -> Option { + let name = match CString::new("XInputExtension") { + Ok(n) => n, + Err(_) => return None, + }; + let mut opcode = 0; + let mut event = 0; + let mut error = 0; + let present = unsafe { + x11::xlib::XQueryExtension(display, name.as_ptr(), &mut opcode, &mut event, &mut error) + }; + if present != 0 { + Some(opcode) + } else { + None + } +} + +/// Install a device-specific XI2 synchronous passive button grab on `window` +/// for `device_id`. This shields the drag: the grab is newer than (and thus +/// checked before) the window manager's click-to-focus grab on the same +/// window, and being device-specific it does not conflict with the WM's +/// core/all-master grabs. The matching press freezes the device and is +/// delivered to us; replaying it (XIReplayDevice) re-checks grabs only +/// *below* this window and then delivers the event normally to the app, so +/// the WM never sees the press and never steals focus. +fn install_shield_grab( + display: *mut x11::xlib::Display, + device_id: i32, + window: x11::xlib::Window, + button: u8, +) -> Result<()> { + let mut mask_bits = vec![0u8; xi_mask_len()]; + x11::xinput2::XISetMask(&mut mask_bits, x11::xinput2::XI_ButtonPress); + let mut evmask = x11::xinput2::XIEventMask { + deviceid: device_id, + mask_len: mask_bits.len() as std::os::raw::c_int, + mask: mask_bits.as_mut_ptr(), + }; + let mut mods = x11::xinput2::XIGrabModifiers { + modifiers: XI_ANY_MODIFIER, + status: 0, + }; + let rc = unsafe { + x11::xinput2::XIGrabButton( + display, + device_id, + button as std::os::raw::c_int, + window, + 0, // cursor: None + x11::xinput2::XIGrabModeSync, // freeze the pointer on press + x11::xinput2::XIGrabModeAsync, // leave the paired keyboard alone + x11::xlib::False, // owner_events: deliver to us + &mut evmask, + 1, + &mut mods, + ) + }; + unsafe { x11::xlib::XSync(display, 0) }; + if rc != 0 { + bail!("XIGrabButton(shield) failed with status {rc}"); + } + Ok(()) +} + +fn remove_shield_grab(display: *mut x11::xlib::Display, device_id: i32, window: x11::xlib::Window, button: u8) { + let mut mods = x11::xinput2::XIGrabModifiers { + modifiers: XI_ANY_MODIFIER, + status: 0, + }; + unsafe { + let prev = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xinput2::XIUngrabButton(display, device_id, button as std::os::raw::c_int, window, 1, &mut mods); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev); + } +} + +/// Drain the frozen shield presses for `pending_devices` and replay each so +/// it continues to the application. Returns the set of device ids we failed +/// to see within the timeout (their drags still proceed; the focus-restore +/// safety net covers any leak). +fn replay_shielded_presses( + display: *mut x11::xlib::Display, + xi_opcode: std::os::raw::c_int, + pending_devices: &mut std::collections::HashSet, + timeout: Duration, +) { + let deadline = std::time::Instant::now() + timeout; + while !pending_devices.is_empty() && std::time::Instant::now() < deadline { + // Only block on XNextEvent when something is queued, so a missing + // press can't hang us past the deadline. + if unsafe { x11::xlib::XPending(display) } == 0 { + sleep(Duration::from_millis(2)); + continue; + } + let mut ev: x11::xlib::XEvent = unsafe { std::mem::zeroed() }; + unsafe { x11::xlib::XNextEvent(display, &mut ev) }; + if unsafe { ev.type_ } != x11::xlib::GenericEvent { + continue; + } + let mut cookie = unsafe { ev.generic_event_cookie }; + if cookie.extension != xi_opcode || cookie.evtype != x11::xinput2::XI_ButtonPress { + continue; + } + if unsafe { x11::xlib::XGetEventData(display, &mut cookie) } == 0 { + continue; + } + let de = cookie.data as *const x11::xinput2::XIDeviceEvent; + if !de.is_null() { + let device_id = unsafe { (*de).deviceid }; + let time = unsafe { (*de).time }; + if pending_devices.remove(&device_id) { + unsafe { + x11::xinput2::XIAllowEvents(display, device_id, x11::xinput2::XIReplayDevice, time); + x11::xlib::XSync(display, 0); + } + } + } + unsafe { x11::xlib::XFreeEventData(display, &mut cookie) }; + } +} + +fn ewmh_active_window(display: *mut x11::xlib::Display) -> Option { + unsafe { + let atom = x11::xlib::XInternAtom( + display, + c"_NET_ACTIVE_WINDOW".as_ptr(), + x11::xlib::True, + ); + if atom == 0 { + return None; + } + let root = x11::xlib::XDefaultRootWindow(display); + let mut type_ret: x11::xlib::Atom = 0; + let mut format_ret: std::os::raw::c_int = 0; + let mut nitems: std::os::raw::c_ulong = 0; + let mut bytes_after: std::os::raw::c_ulong = 0; + let mut data: *mut std::os::raw::c_uchar = std::ptr::null_mut(); + let rc = x11::xlib::XGetWindowProperty( + display, + root, + atom, + 0, + 1, + x11::xlib::False, + x11::xlib::XA_WINDOW, + &mut type_ret, + &mut format_ret, + &mut nitems, + &mut bytes_after, + &mut data, + ); + if rc != x11::xlib::Success as i32 || data.is_null() { + return None; + } + let window = if nitems >= 1 && format_ret == 32 { + Some(*(data as *const std::os::raw::c_ulong) as x11::xlib::Window) + } else { + None + }; + x11::xlib::XFree(data as *mut _); + window.filter(|w| *w != 0) + } +} + +/// Current X server time via the standard PropertyNotify round-trip. +/// EWMH activation requests stamped CurrentTime(0) lose to the WM's +/// focus-stealing prevention whenever any newer input exists. +fn x_server_time(display: *mut x11::xlib::Display) -> x11::xlib::Time { + unsafe { + let root = x11::xlib::XDefaultRootWindow(display); + let win = x11::xlib::XCreateSimpleWindow(display, root, -1, -1, 1, 1, 0, 0, 0); + x11::xlib::XSelectInput(display, win, x11::xlib::PropertyChangeMask); + let atom = x11::xlib::XInternAtom(display, c"CUA_TIME_PROBE".as_ptr(), x11::xlib::False); + x11::xlib::XChangeProperty( + display, + win, + atom, + x11::xlib::XA_STRING, + 8, + x11::xlib::PropModeReplace, + [0u8].as_ptr(), + 0, + ); + x11::xlib::XSync(display, 0); + let mut time: x11::xlib::Time = x11::xlib::CurrentTime; + let mut ev: x11::xlib::XEvent = std::mem::zeroed(); + while x11::xlib::XCheckWindowEvent( + display, + win, + x11::xlib::PropertyChangeMask, + &mut ev, + ) != 0 + { + if ev.get_type() == x11::xlib::PropertyNotify { + time = ev.property.time; + } + } + x11::xlib::XDestroyWindow(display, win); + x11::xlib::XFlush(display); + time + } +} + +fn ewmh_activate_window( + display: *mut x11::xlib::Display, + window: x11::xlib::Window, + current_active: x11::xlib::Window, +) { + unsafe { + let atom = x11::xlib::XInternAtom( + display, + c"_NET_ACTIVE_WINDOW".as_ptr(), + x11::xlib::True, + ); + if atom == 0 { + return; + } + let root = x11::xlib::XDefaultRootWindow(display); + let mut ev: x11::xlib::XClientMessageEvent = std::mem::zeroed(); + ev.type_ = x11::xlib::ClientMessage; + ev.window = window; + ev.message_type = atom; + ev.format = 32; + ev.data.set_long(0, 2); // source indication: pager/tool + ev.data.set_long(1, x_server_time(display) as std::os::raw::c_long); + ev.data.set_long(2, current_active as std::os::raw::c_long); + x11::xlib::XSendEvent( + display, + root, + x11::xlib::False, + x11::xlib::SubstructureRedirectMask | x11::xlib::SubstructureNotifyMask, + &mut ev as *mut _ as *mut x11::xlib::XEvent, + ); + x11::xlib::XSync(display, 0); + } +} + +fn button_code(button: u8) -> Result { + match button { + 1 => Ok(Key::BTN_LEFT), + 2 => Ok(Key::BTN_MIDDLE), + 3 => Ok(Key::BTN_RIGHT), + _ => bail!("unsupported button {button} for uinput pointer"), + } +} + +fn emit_button(device: &mut VirtualDevice, button: u8, press: bool) -> Result<()> { + let code = button_code(button)?; + device.emit(&[InputEvent::new(EventType::KEY, code.0, if press { 1 } else { 0 })])?; + Ok(()) +} + +fn emit_relative_motion(device: &mut VirtualDevice, dx: i32, dy: i32) -> Result<()> { + let mut events = Vec::with_capacity(2); + if dx != 0 { + events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_X.0, dx)); + } + if dy != 0 { + events.push(InputEvent::new(EventType::RELATIVE, RelativeAxisType::REL_Y.0, dy)); + } + if events.is_empty() { + return Ok(()); + } + device.emit(&events)?; + Ok(()) +} + +pub fn send_parallel_virtual_pointer_drags( + drags: &[(String, VirtualPointerDrag)], +) -> Result<()> { + let display = open_display()?; + supports_parallel_pointer_injection(display)?; + let xi_opcode = xinput_opcode(display); + + struct ActiveDrag { + cursor_id: String, + ids: MasterPointerIds, + device: Arc>, + drag: VirtualPointerDrag, + cum: Vec, + total: f64, + steps: usize, + step_delay: Duration, + current_step: usize, + next_at: std::time::Instant, + last_x: i32, + last_y: i32, + } + + let start_at = std::time::Instant::now() + Duration::from_millis(120); + let mut active = Vec::with_capacity(drags.len()); + + // Click-to-focus WMs grab buttons for XIAllMasterDevices, so the drag's + // press activates the target window exactly like a user click would. + // Remember the focus state and hand it back afterwards so parallel + // drags don't steal it. + let saved_focus = save_focus_state(display); + + let result = (|| -> Result<()> { + for (cursor_id, drag) in drags { + let ids = ensure_master_pointer(cursor_id)?; + let device = uinput_pointers() + .lock() + .unwrap() + .get(cursor_id) + .cloned() + .ok_or_else(|| anyhow!("missing uinput pointer for '{cursor_id}'"))?; + let (cum, total) = path_cumulative(&drag.path); + let start = *drag.path.first().unwrap_or(&(0, 0)); + active.push(ActiveDrag { + cursor_id: cursor_id.clone(), + ids, + device, + drag: drag.clone(), + cum, + total, + steps: drag.steps.max(1), + step_delay: if drag.steps.max(1) > 1 { + Duration::from_millis(drag.duration_ms / drag.steps.max(1) as u64) + } else { + Duration::from_millis(drag.duration_ms) + }, + current_step: 0, + next_at: start_at, + last_x: start.0, + last_y: start.1, + }); + } + + let now = std::time::Instant::now(); + if start_at > now { + std::thread::sleep(start_at - now); + } + + // Shield each drag from the WM's click-to-focus grab, then press. + // Per item: install a device-specific sync grab on the target window, + // warp, press, and immediately replay the frozen press so it reaches + // the app while the WM stays blind to it. We replay each press before + // emitting the next so only ONE device is ever frozen at a time — the + // X server drops replayed presses when several devices are frozen on + // the same window and replayed together. The few-ms stagger this adds + // to the presses is invisible; the concurrency that matters is motion. + // Shielding is mandatory: if install/replay fails, abort instead of + // continuing with a drag that could steal focus and rely on restore. + let mut shielded = std::collections::HashSet::new(); + for item in &active { + let opcode = xi_opcode.ok_or_else(|| { + anyhow!("parallel_mouse_drag requires XInput/XI2 shield grabs for no-focus-steal operation") + })?; + install_shield_grab( + display, + item.ids.pointer_id, + item.drag.target_window as x11::xlib::Window, + item.drag.button, + ) + .with_context(|| format!("shield grab failed for '{}'", item.cursor_id))?; + shielded.insert(item.ids.pointer_id); + let start = *item.drag.path.first().unwrap_or(&(0, 0)); + warp_master_pointer(display, item.ids, start.0, start.1)?; + { + let mut device = item.device.lock().unwrap(); + emit_button(&mut device, item.drag.button, true)?; + } + let mut pending = std::collections::HashSet::from([item.ids.pointer_id]); + replay_shielded_presses(display, opcode, &mut pending, Duration::from_millis(1000)); + if !pending.is_empty() { + return Err(anyhow!( + "shield replay timed out before XI_ButtonPress arrived for '{}'", + item.cursor_id + )); + } + } + + while active.iter().any(|item| item.current_step < item.steps) { + let now = std::time::Instant::now(); + let mut advanced = false; + let mut next_deadline = None; + + for item in &mut active { + if item.current_step >= item.steps { + continue; + } + if now >= item.next_at { + item.current_step += 1; + let t = item.current_step as f64 / item.steps as f64; + let (ix, iy) = point_on_path(&item.drag.path, &item.cum, item.total, t); + let dx = ix - item.last_x; + let dy = iy - item.last_y; + if dx != 0 || dy != 0 { + let mut device = item.device.lock().unwrap(); + emit_relative_motion(&mut device, dx, dy)?; + // Keep the agent cursor overlay tracking the drag so + // the gesture is visible, not just its endpoints. + crate::overlay::send_command_for( + item.cursor_id.clone(), + cursor_overlay::OverlayCommand::SnapTo { + x: ix as f64, + y: iy as f64, + heading_radians: Some((dy as f64).atan2(dx as f64)), + }, + ); + } + item.last_x = ix; + item.last_y = iy; + item.next_at = now + item.step_delay; + advanced = true; + } + if item.current_step < item.steps { + next_deadline = Some(match next_deadline { + Some(deadline) => std::cmp::min(deadline, item.next_at), + None => item.next_at, + }); + } + } + + if !advanced { + if let Some(deadline) = next_deadline { + let now = std::time::Instant::now(); + if deadline > now { + std::thread::sleep(deadline - now); + } + } + } + } + + for item in &active { + let mut device = item.device.lock().unwrap(); + emit_button(&mut device, item.drag.button, false)?; + } + + // Remove the shields now that the drag is done. The button is only + // grabbed for ButtonPress, so the shield is dormant during motion and + // release; this just stops it matching the next gesture's press. + for item in &active { + if shielded.contains(&item.ids.pointer_id) { + remove_shield_grab( + display, + item.ids.pointer_id, + item.drag.target_window as x11::xlib::Window, + item.drag.button, + ); + } + } + Ok(()) + })(); + // Remove the per-session masters before handing focus back: non-MPX-aware + // WMs (xfwm4, openbox) desync their focus bookkeeping while foreign + // master keyboards linger, and the next call recreates masters cheaply. + for (cursor_id, _) in drags { + forget_master_pointer(cursor_id); + } + restore_focus_state(display, &saved_focus); + unsafe { + x11::xlib::XCloseDisplay(display); + } + result +} + +/// Pre-drag focus snapshot: the EWMH active window when a conforming WM is +/// running, plus the core input focus as a WM-agnostic fallback. +struct SavedFocus { + ewmh_active: Option, + core_focus: x11::xlib::Window, + core_revert_to: std::os::raw::c_int, +} + +fn save_focus_state(display: *mut x11::xlib::Display) -> SavedFocus { + let mut core_focus: x11::xlib::Window = 0; + let mut core_revert_to: std::os::raw::c_int = 0; + unsafe { + x11::xlib::XGetInputFocus(display, &mut core_focus, &mut core_revert_to); + } + SavedFocus { + ewmh_active: ewmh_active_window(display), + core_focus, + core_revert_to, + } +} + +unsafe extern "C" fn ignore_x_error( + _display: *mut x11::xlib::Display, + _event: *mut x11::xlib::XErrorEvent, +) -> std::os::raw::c_int { + 0 +} + +fn restore_focus_state(display: *mut x11::xlib::Display, saved: &SavedFocus) { + // Let the release/focus events from the drag settle before reading the + // post-drag state, so we don't race the WM's own focus update. + unsafe { x11::xlib::XSync(display, 0) }; + + if let Some(prev) = saved.ewmh_active { + // EWMH path: ask the WM to re-activate, so its active-window + // bookkeeping (decorations, stacking) stays consistent. The WM + // processes its own click-to-focus for the drag asynchronously and + // can re-activate the dragged window even after one re-activation of + // ours has landed — so don't stop at first success: require the + // active window to hold stable for consecutive checks, re-sending on + // every regression, within a bounded budget. + sleep(Duration::from_millis(300)); + let mut stable = 0; + for attempt in 0..15 { + let now = ewmh_active_window(display); + if now == Some(prev) { + stable += 1; + if stable >= 3 { + return; + } + } else { + stable = 0; + // MPX clicks can leave a core-protocol WM believing the + // dragged window is focused while the core focus never moved + // there: its XSetInputFocus for our activation is then a + // no-op, no FocusIn arrives, and its bookkeeping never + // updates. Bounce the core focus onto the window the WM + // believes active so the activation produces a real focus + // transition the WM can observe. + if attempt >= 2 { + if let Some(now_win) = now { + unsafe { + let prev_handler = + x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xlib::XSetInputFocus( + display, + now_win, + x11::xlib::RevertToParent, + x11::xlib::CurrentTime, + ); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev_handler); + } + sleep(Duration::from_millis(100)); + } + } + ewmh_activate_window(display, prev, now.unwrap_or(0)); + } + sleep(Duration::from_millis(200)); + } + if stable == 0 { + tracing::warn!("focus restore: WM did not re-activate 0x{prev:x}"); + } + return; + } + + // No EWMH WM (bare X / minimal WM): restore the core input focus + // directly. The saved window may have been destroyed meanwhile, and + // Xlib's default error handler exits the process on BadWindow, so the + // restore runs under a scoped ignore-errors handler. + if saved.core_focus == 0 { + return; + } + unsafe { + let mut now_focus: x11::xlib::Window = 0; + let mut now_revert: std::os::raw::c_int = 0; + x11::xlib::XGetInputFocus(display, &mut now_focus, &mut now_revert); + if now_focus == saved.core_focus { + return; + } + let prev_handler = x11::xlib::XSetErrorHandler(Some(ignore_x_error)); + x11::xlib::XSetInputFocus( + display, + saved.core_focus, + saved.core_revert_to, + x11::xlib::CurrentTime, + ); + x11::xlib::XSync(display, 0); + x11::xlib::XSetErrorHandler(prev_handler); + } +} + +#[derive(Clone, Copy, Debug)] +struct EventTarget { + window: Window, + local_x: i16, + local_y: i16, + root_x: i16, + root_y: i16, +} + +fn point_in_rect(x: i32, y: i32, geom: &GetGeometryReply) -> bool { + x >= geom.x as i32 + && y >= geom.y as i32 + && x < geom.x as i32 + geom.width as i32 + && y < geom.y as i32 + geom.height as i32 +} + +fn deepest_child_at_point( + conn: &RustConnection, + window: Window, + local_x: i32, + local_y: i32, +) -> Result<(Window, i32, i32)> { + let tree = conn.query_tree(window)?.reply()?; + for child in tree.children.iter().rev() { + let Ok(geom) = conn.get_geometry(*child)?.reply() else { + continue; + }; + if !point_in_rect(local_x, local_y, &geom) { + continue; + } + let child_x = local_x - geom.x as i32; + let child_y = local_y - geom.y as i32; + return deepest_child_at_point(conn, *child, child_x, child_y); + } + Ok((window, local_x, local_y)) +} + +fn resolve_event_target(conn: &RustConnection, xid: u64, x: i32, y: i32) -> Result { + let top = xid as Window; + let root = conn.setup().roots[0].root; + let root_pos = conn.translate_coordinates(top, root, 0, 0)?.reply()?; + let (window, local_x, local_y) = deepest_child_at_point(conn, top, x, y)?; + Ok(EventTarget { + window, + local_x: local_x as i16, + local_y: local_y as i16, + root_x: (root_pos.dst_x as i32 + x) as i16, + root_y: (root_pos.dst_y as i32 + y) as i16, + }) +} + +fn button_state_mask(button: u8) -> KeyButMask { + match button { + 1 => KeyButMask::BUTTON1, + 2 => KeyButMask::BUTTON2, + 3 => KeyButMask::BUTTON3, + 4 => KeyButMask::BUTTON4, + 5 => KeyButMask::BUTTON5, + _ => KeyButMask::from(0u16), + } +} + /// Send a synthetic FocusIn event to a window without changing the actual X11 input focus. /// This can trigger toolkit-level focus handlers (e.g., Qt5's AT-SPI bridge) without /// moving the window manager's active window. Use with send_focus_out to restore state. @@ -62,23 +1181,22 @@ pub fn send_focus_out(xid: u64) -> Result<()> { /// Send a button click (down + up) to a window at window-local coordinates. pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> { let (conn, _) = RustConnection::connect(None)?; - let window = xid as u32; - - // Get the root window for the display. let root = conn.setup().roots[0].root; for _ in 0..count { + let target = resolve_event_target(&conn, xid, x, y)?; let press = ButtonPressEvent { response_type: BUTTON_PRESS_EVENT, detail: button, sequence: 0, time: x11rb::CURRENT_TIME, root, - event: window, + event: target.window, child: x11rb::NONE, - root_x: 0, root_y: 0, - event_x: x as i16, - event_y: y as i16, + root_x: target.root_x, + root_y: target.root_y, + event_x: target.local_x, + event_y: target.local_y, state: KeyButMask::from(0u16), same_screen: true, }; @@ -89,18 +1207,19 @@ pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result< sequence: 0, time: x11rb::CURRENT_TIME, root, - event: window, + event: target.window, child: x11rb::NONE, - root_x: 0, root_y: 0, - event_x: x as i16, - event_y: y as i16, - state: KeyButMask::from(0u16), + root_x: target.root_x, + root_y: target.root_y, + event_x: target.local_x, + event_y: target.local_y, + state: button_state_mask(button), same_screen: true, }; - conn.send_event(false, window, EventMask::BUTTON_PRESS, &press)?; + conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?; sleep(Duration::from_millis(CLICK_DELAY_MS)); - conn.send_event(false, window, EventMask::BUTTON_RELEASE, &release)?; + conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?; conn.flush()?; if count > 1 { @@ -126,10 +1245,10 @@ pub fn send_drag( button: u8, ) -> Result<()> { let (conn, _) = RustConnection::connect(None)?; - let window = xid as u32; let root = conn.setup().roots[0].root; let steps = steps.max(1); let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; + let press_target = resolve_event_target(&conn, xid, from_x, from_y)?; // ButtonPress at start. let press = ButtonPressEvent { @@ -137,13 +1256,13 @@ pub fn send_drag( detail: button, sequence: 0, time: x11rb::CURRENT_TIME, - root, event: window, child: x11rb::NONE, - root_x: 0, root_y: 0, - event_x: from_x as i16, event_y: from_y as i16, + root, event: press_target.window, child: x11rb::NONE, + root_x: press_target.root_x, root_y: press_target.root_y, + event_x: press_target.local_x, event_y: press_target.local_y, state: KeyButMask::from(0u16), same_screen: true, }; - conn.send_event(false, window, EventMask::BUTTON_PRESS, &press)?; + conn.send_event(false, press_target.window, EventMask::BUTTON_PRESS, &press)?; conn.flush()?; sleep(Duration::from_millis(CLICK_DELAY_MS)); @@ -152,18 +1271,19 @@ pub fn send_drag( let t = i as f64 / steps as f64; let ix = from_x + ((to_x - from_x) as f64 * t).round() as i32; let iy = from_y + ((to_y - from_y) as f64 * t).round() as i32; + let target = resolve_event_target(&conn, xid, ix, iy)?; let motion = MotionNotifyEvent { response_type: MOTION_NOTIFY_EVENT, detail: Motion::NORMAL, sequence: 0, time: x11rb::CURRENT_TIME, - root, event: window, child: x11rb::NONE, - root_x: 0, root_y: 0, - event_x: ix as i16, event_y: iy as i16, - state: KeyButMask::from(0u16), + root, event: target.window, child: x11rb::NONE, + root_x: target.root_x, root_y: target.root_y, + event_x: target.local_x, event_y: target.local_y, + state: button_state_mask(button), same_screen: true, }; - conn.send_event(false, window, EventMask::POINTER_MOTION, &motion)?; + conn.send_event(false, target.window, EventMask::POINTER_MOTION, &motion)?; conn.flush()?; if step_delay_ms > 0 { sleep(Duration::from_millis(step_delay_ms)); @@ -171,18 +1291,91 @@ pub fn send_drag( } // ButtonRelease at end. + let release_target = resolve_event_target(&conn, xid, to_x, to_y)?; let release = ButtonReleaseEvent { response_type: BUTTON_RELEASE_EVENT, detail: button, sequence: 0, time: x11rb::CURRENT_TIME, - root, event: window, child: x11rb::NONE, - root_x: 0, root_y: 0, - event_x: to_x as i16, event_y: to_y as i16, + root, event: release_target.window, child: x11rb::NONE, + root_x: release_target.root_x, root_y: release_target.root_y, + event_x: release_target.local_x, event_y: release_target.local_y, + state: button_state_mask(button), + same_screen: true, + }; + conn.send_event(false, release_target.window, EventMask::BUTTON_RELEASE, &release)?; + conn.flush()?; + Ok(()) +} + +pub fn send_button_down(xid: u64, x: i32, y: i32, button: u8) -> Result<()> { + let (conn, _) = RustConnection::connect(None)?; + let root = conn.setup().roots[0].root; + let target = resolve_event_target(&conn, xid, x, y)?; + let press = ButtonPressEvent { + response_type: BUTTON_PRESS_EVENT, + detail: button, + sequence: 0, + time: x11rb::CURRENT_TIME, + root, + event: target.window, + child: x11rb::NONE, + root_x: target.root_x, + root_y: target.root_y, + event_x: target.local_x, + event_y: target.local_y, state: KeyButMask::from(0u16), same_screen: true, }; - conn.send_event(false, window, EventMask::BUTTON_RELEASE, &release)?; + conn.send_event(false, target.window, EventMask::BUTTON_PRESS, &press)?; + conn.flush()?; + Ok(()) +} + +pub fn send_motion(xid: u64, x: i32, y: i32, button: Option) -> Result<()> { + let (conn, _) = RustConnection::connect(None)?; + let root = conn.setup().roots[0].root; + let target = resolve_event_target(&conn, xid, x, y)?; + let motion = MotionNotifyEvent { + response_type: MOTION_NOTIFY_EVENT, + detail: Motion::NORMAL, + sequence: 0, + time: x11rb::CURRENT_TIME, + root, + event: target.window, + child: x11rb::NONE, + root_x: target.root_x, + root_y: target.root_y, + event_x: target.local_x, + event_y: target.local_y, + state: button.map(button_state_mask).unwrap_or_else(|| KeyButMask::from(0u16)), + same_screen: true, + }; + conn.send_event(false, target.window, EventMask::POINTER_MOTION, &motion)?; + conn.flush()?; + Ok(()) +} + +pub fn send_button_up(xid: u64, x: i32, y: i32, button: u8) -> Result<()> { + let (conn, _) = RustConnection::connect(None)?; + let root = conn.setup().roots[0].root; + let target = resolve_event_target(&conn, xid, x, y)?; + let release = ButtonReleaseEvent { + response_type: BUTTON_RELEASE_EVENT, + detail: button, + sequence: 0, + time: x11rb::CURRENT_TIME, + root, + event: target.window, + child: x11rb::NONE, + root_x: target.root_x, + root_y: target.root_y, + event_x: target.local_x, + event_y: target.local_y, + state: button_state_mask(button), + same_screen: true, + }; + conn.send_event(false, target.window, EventMask::BUTTON_RELEASE, &release)?; conn.flush()?; Ok(()) } @@ -449,3 +1642,92 @@ exit 0"#, } } } + +#[cfg(test)] +mod path_tests { + use super::{path_cumulative, point_on_path, sample_function}; + + #[test] + fn sample_linear_function() { + let pts = sample_function("x", 0.0, 10.0, 11).unwrap(); + assert_eq!(pts.len(), 11); + assert_eq!(pts.first().unwrap(), &(0.0, 0.0)); + assert_eq!(pts.last().unwrap(), &(10.0, 10.0)); + assert!((pts[5].0 - 5.0).abs() < 1e-9 && (pts[5].1 - 5.0).abs() < 1e-9); + } + + #[test] + fn sample_affine_and_trig() { + let pts = sample_function("2*x+1", 0.0, 4.0, 5).unwrap(); + for (x, y) in pts { + assert!((y - (2.0 * x + 1.0)).abs() < 1e-9); + } + // sin(x) parses and yields finite, bounded values. + let s = sample_function("100+50*sin(x)", 0.0, 6.28, 40).unwrap(); + assert!(s.iter().all(|(_, y)| (49.9..=150.1).contains(y))); + } + + #[test] + fn invalid_expression_errors() { + assert!(sample_function("x +", 0.0, 1.0, 4).is_err()); + assert!(sample_function("3*z", 0.0, 1.0, 4).is_err()); // unknown var + } + + #[test] + fn non_finite_points_are_dropped() { + // ln(x) is -inf/NaN for x<=0; the finite tail must still sample. + let pts = sample_function("ln(x)", -2.0, 5.0, 50).unwrap(); + assert!(pts.iter().all(|(_, y)| y.is_finite())); + assert!(pts.len() >= 2); + } + + #[test] + fn cumulative_lengths_and_total() { + // 3-4-5 triangle then a zero-length repeat. + let path = [(0, 0), (3, 4), (3, 4)]; + let (cum, total) = path_cumulative(&path); + assert_eq!(cum.len(), 3); + assert!((cum[0] - 0.0).abs() < 1e-9); + assert!((cum[1] - 5.0).abs() < 1e-9); + assert!((cum[2] - 5.0).abs() < 1e-9); + assert!((total - 5.0).abs() < 1e-9); + } + + #[test] + fn straight_segment_interpolates_by_fraction() { + let path = [(0, 0), (10, 0)]; + let (cum, total) = path_cumulative(&path); + assert_eq!(point_on_path(&path, &cum, total, 0.0), (0, 0)); + assert_eq!(point_on_path(&path, &cum, total, 0.5), (5, 0)); + assert_eq!(point_on_path(&path, &cum, total, 1.0), (10, 0)); + } + + #[test] + fn multi_segment_follows_arc_length() { + // L-shape: (0,0)->(10,0)->(10,10), total length 20. + let path = [(0, 0), (10, 0), (10, 10)]; + let (cum, total) = path_cumulative(&path); + assert!((total - 20.0).abs() < 1e-9); + // Halfway by arc length lands exactly on the corner. + assert_eq!(point_on_path(&path, &cum, total, 0.5), (10, 0)); + // 3/4 of the way is 5px down the second segment. + assert_eq!(point_on_path(&path, &cum, total, 0.75), (10, 5)); + } + + #[test] + fn fraction_is_clamped_and_endpoints_exact() { + let path = [(2, 2), (8, 2), (8, 8)]; + let (cum, total) = path_cumulative(&path); + // t past the ends clamps to the terminal points (no overshoot). + assert_eq!(point_on_path(&path, &cum, total, -0.5), (2, 2)); + assert_eq!(point_on_path(&path, &cum, total, 2.0), (8, 8)); + } + + #[test] + fn degenerate_path_returns_last_point() { + let path = [(5, 5), (5, 5)]; + let (cum, total) = path_cumulative(&path); + assert!((total - 0.0).abs() < 1e-9); + assert_eq!(point_on_path(&path, &cum, total, 0.3), (5, 5)); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs index 01dd261263..2ae7df7eba 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs @@ -17,49 +17,188 @@ //! What stays here is the X11 window plumbing: connection setup, //! override-redirect visual, ShapeInput passthrough, and the XPutImage paint. +use std::collections::{HashMap, HashSet}; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; -use cursor_overlay::{CursorConfig, OverlayCommand, RenderStateCore}; +use cursor_overlay::{ + CursorConfig, CursorKey, KeyedOverlayCommand, OverlayCommand, OverlayMsg, Palette, + RenderStateCore, +}; #[cfg(target_os = "linux")] use cursor_overlay::ZOrderEnforcer; // ── Global channel ──────────────────────────────────────────────────────── -static CMD_TX: OnceLock> = OnceLock::new(); -static CMD_RX_CELL: Mutex>> = Mutex::new(None); -static RENDER: Mutex> = Mutex::new(None); -static ARRIVAL_TX: Mutex>> = Mutex::new(None); +static CMD_TX: OnceLock> = OnceLock::new(); +static CMD_RX_CELL: Mutex>> = Mutex::new(None); +static RENDER: Mutex> = Mutex::new(None); +static ARRIVAL_TX: Mutex>>> = + Mutex::new(None); + +fn arrival_register(key: CursorKey, tx: tokio::sync::oneshot::Sender<()>) { + let mut guard = ARRIVAL_TX.lock().unwrap(); + let map = guard.get_or_insert_with(HashMap::new); + if let Some(old_tx) = map.insert(key, tx) { + let _ = old_tx.send(()); + } +} + +fn arrival_fire(key: &CursorKey) { + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(map) = guard.as_mut() { + if let Some(tx) = map.remove(key) { + let _ = tx.send(()); + } + } + } +} + +struct RenderMap { + cursors: HashMap, + scr_w: u32, + scr_h: u32, + template: CursorConfig, + ended: HashSet, + last_active: Option, +} + +fn render_state_for_key(template: &CursorConfig, key: &str) -> RenderState { + let mut rs = RenderState::new(template.clone()); + rs.core.palette = Palette::for_instance(key); + rs +} + +fn apply_msg(map: &mut RenderMap, msg: OverlayMsg) -> Option { + match msg { + OverlayMsg::Remove(key) => { + if key != "default" { + map.cursors.remove(&key); + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(arrivals) = guard.as_mut() { + arrivals.remove(&key); + } + } + if map.last_active.as_deref() == Some(key.as_str()) { + map.last_active = None; + } + map.ended.insert(key); + } + None + } + OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }) => { + if map.ended.contains(&key) { + return None; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key) + .or_insert_with(|| render_state_for_key(&template, &k)); + rs.apply_command(cmd); + Some(k) + } + } +} pub fn init(cfg: CursorConfig) { let (tx, rx) = std::sync::mpsc::sync_channel(4096); let _ = CMD_TX.set(tx); *CMD_RX_CELL.lock().unwrap() = Some(rx); - *RENDER.lock().unwrap() = Some(RenderState::new(cfg)); + *ARRIVAL_TX.lock().unwrap() = Some(HashMap::new()); + let mut cursors = HashMap::new(); + cursors.insert("default".to_owned(), RenderState::new(cfg.clone())); + *RENDER.lock().unwrap() = Some(RenderMap { + cursors, + scr_w: 1920, + scr_h: 1080, + template: cfg, + ended: HashSet::new(), + last_active: None, + }); } pub fn send_command(cmd: OverlayCommand) { + send_command_for("default".to_owned(), cmd); +} + +pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) { + if key.is_empty() { + return; + } if let Some(tx) = CMD_TX.get() { - let _ = tx.try_send(cmd); + let _ = tx.try_send(OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd })); } } pub fn is_enabled() -> bool { + is_enabled_for("default") +} + +pub fn is_enabled_for(key: &str) -> bool { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.visible)) + .and_then(|g| { + g.as_ref().and_then(|m| { + m.cursors + .get(key) + .or_else(|| m.cursors.get("default")) + .map(|rs| rs.core.visible) + }) + }) .unwrap_or(false) } pub fn current_position() -> (f64, f64) { + current_position_for("default") +} + +pub fn current_position_for(key: &str) -> (f64, f64) { RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.pos)) + .and_then(|g| g.as_ref().and_then(|m| m.cursors.get(key)).map(|rs| rs.core.pos)) .unwrap_or((-200.0, -200.0)) } +fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool { + const SEED_OFFSET: f64 = 140.0; + let mut guard = RENDER.lock().unwrap(); + let Some(map) = guard.as_mut() else { return false }; + if map.ended.contains(key) { + return false; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key.clone()) + .or_insert_with(|| render_state_for_key(&template, &k)); + if !(rs.core.cfg.enabled && rs.core.pos.0 < -50.0) { + return false; + } + let max_x = map.scr_w.max(2) as f64 - 2.0; + let max_y = map.scr_h.max(2) as f64 - 2.0; + let mut sx = (target_x - SEED_OFFSET).clamp(2.0, max_x); + let mut sy = (target_y - SEED_OFFSET).clamp(2.0, max_y); + if (sx - target_x).abs() < 8.0 && (sy - target_y).abs() < 8.0 { + sx = (target_x + SEED_OFFSET).clamp(2.0, max_x); + sy = (target_y + SEED_OFFSET).clamp(2.0, max_y); + } + rs.core.pos = (sx, sy); + true +} + pub async fn animate_cursor_to(x: f64, y: f64) { + animate_cursor_to_for("default".to_owned(), x, y).await; +} + +pub async fn animate_cursor_to_for(key: CursorKey, x: f64, y: f64) { + if key.is_empty() { + return; + } + seed_start_if_sentinel(&key, x, y); let should_animate = { let guard = RENDER.lock().unwrap(); - match guard.as_ref() { + match guard.as_ref().and_then(|m| m.cursors.get(&key)) { Some(rs) if rs.core.cfg.enabled && rs.core.visible && rs.core.pos.0 > -50.0 => true, _ => false, } @@ -69,15 +208,9 @@ pub async fn animate_cursor_to(x: f64, y: f64) { } let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - { - let mut guard = ARRIVAL_TX.lock().unwrap(); - if let Some(old_tx) = guard.take() { - let _ = old_tx.send(()); - } - *guard = Some(tx); - } + arrival_register(key.clone(), tx); - send_command(OverlayCommand::MoveTo { + send_command_for(key, OverlayCommand::MoveTo { x, y, end_heading_radians: std::f64::consts::FRAC_PI_4, @@ -86,6 +219,15 @@ pub async fn animate_cursor_to(x: f64, y: f64) { let _ = rx.await; } +pub fn remove_cursor(key: CursorKey) { + if key.is_empty() { + return; + } + if let Some(tx) = CMD_TX.get() { + let _ = tx.try_send(OverlayMsg::Remove(key)); + } +} + /// Spawn the overlay on a dedicated thread. Non-blocking. pub fn run_on_thread() { let rx = match CMD_RX_CELL.lock().unwrap().take() { @@ -96,7 +238,7 @@ pub fn run_on_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.core.cfg.clone(), + Some(map) => map.template.clone(), None => return, } }; @@ -121,17 +263,12 @@ pub fn run_on_thread() { struct RenderState { core: RenderStateCore, - /// X11 screen dimensions in pixels (populated after XOpenDisplay). - scr_w: u32, - scr_h: u32, } impl RenderState { fn new(cfg: CursorConfig) -> Self { RenderState { core: RenderStateCore::new(cfg), - scr_w: 1920, - scr_h: 1080, } } @@ -153,14 +290,14 @@ impl RenderState { // ── X11 thread ──────────────────────────────────────────────────────────── #[cfg(target_os = "linux")] -fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { +fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { use x11rb::connection::Connection; - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; - use x11rb::protocol::shape::*; - use x11rb::protocol::shape::ConnectionExt as _; - use x11rb::wrapper::ConnectionExt as _; - use x11rb::COPY_FROM_PARENT; + use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK, SO}; + use x11rb::protocol::xproto::{ + AtomEnum, ColormapAlloc, CreateWindowAux, EventMask, PropMode, WindowClass, + }; + use x11rb::protocol::xproto::ConnectionExt as XprotoConnectionExt; + use x11rb::wrapper::ConnectionExt as WrapperConnectionExt; // Connect to X11. let (conn, screen_num) = match x11rb::connect(None) { @@ -179,9 +316,9 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver= Duration::from_millis(80) { last_ztick = Instant::now(); - let pinned_wid = { - let guard = RENDER.lock().unwrap(); - guard.as_ref().and_then(|rs| rs.core.pinned_wid) - }; z_enforcer.reassert(pinned_wid); } @@ -332,8 +475,7 @@ struct X11ZOrderEnforcer<'a, C: x11rb::connection::Connection> { #[cfg(target_os = "linux")] impl<'a, C: x11rb::connection::Connection> ZOrderEnforcer for X11ZOrderEnforcer<'a, C> { fn reassert(&self, target: Option) { - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; + use x11rb::protocol::xproto::{ConfigureWindowAux, ConnectionExt as XprotoConnectionExt, StackMode}; // Per the ZOrderEnforcer trait contract, a stale `target` (window // gone) should fall back to the `None` behavior — top of the @@ -396,8 +538,7 @@ fn paint_x11( _visual_id: u32, pm: &tiny_skia::Pixmap, ) { - use x11rb::protocol::xproto::*; - use x11rb::protocol::xproto::ConnectionExt as _; + use x11rb::protocol::xproto::{ConnectionExt as XprotoConnectionExt, CreateGCAux, ImageFormat}; if pm.width() == 0 || pm.height() == 0 { return; } // Create a GC for the window if we don't have one. @@ -436,4 +577,4 @@ fn paint_x11( } #[cfg(not(target_os = "linux"))] -fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) {} +fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) {} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 57d31539cf..a89d397ab6 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -1,7 +1,11 @@ //! Real Linux tool implementations (compiled only on Linux). use async_trait::async_trait; -use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}}; +use cua_driver_core::{ + protocol::ToolResult, + tool::{Tool, ToolDef, ToolRegistry}, + tool_args::ArgsExt, +}; use serde_json::{json, Value}; use std::fs; use std::path::PathBuf; @@ -65,9 +69,20 @@ pub struct ToolState { pub cursor_registry: Arc, pub resize_registry: Arc, pub zoom_registry: Arc, + pub mouse_hold: std::sync::Mutex>, pub config: Arc>, } +#[derive(Clone, Debug)] +pub struct MouseHoldState { + pub cursor_id: String, + pub pid: u32, + pub xid: u64, + pub button: u8, + pub x: f64, + pub y: f64, +} + impl ToolState { pub fn new() -> Arc { Arc::new(Self { @@ -75,6 +90,7 @@ impl ToolState { cursor_registry: Arc::new(CursorRegistry::new()), resize_registry: Arc::new(ResizeRegistry::new()), zoom_registry: Arc::new(ZoomRegistry::new()), + mouse_hold: std::sync::Mutex::new(Default::default()), config: Arc::new(RwLock::new(DriverConfig::default())), }) } @@ -565,16 +581,122 @@ fn window_local_to_screen(xid: u64, x: f64, y: f64) -> anyhow::Result<(f64, f64) Ok((reply.dst_x as f64 + x, reply.dst_y as f64 + y)) } +fn parse_mouse_button(name: &str) -> u8 { + match name { + "right" => 3, + "middle" => 2, + _ => 1, + } +} + +fn mouse_button_name(button: u8) -> &'static str { + match button { + 3 => "right", + 2 => "middle", + _ => "left", + } +} + +fn resolve_cursor_key(args: &Value) -> String { + for key in ["session", "cursor_id"] { + if let Some(v) = args.get(key).and_then(|v| v.as_str()) { + if !v.is_empty() { + return v.to_owned(); + } + } + } + "default".to_owned() +} + +fn mouse_hold_json(cursor_id: &str, hold: Option<&MouseHoldState>) -> Value { + match hold { + Some(hold) => json!({ + "cursor_id": cursor_id, + "held": true, + "pid": hold.pid, + "window_id": hold.xid, + "button": mouse_button_name(hold.button), + "x": hold.x, + "y": hold.y, + }), + None => json!({ + "cursor_id": cursor_id, + "held": false, + "pid": Value::Null, + "window_id": Value::Null, + "button": Value::Null, + "x": Value::Null, + "y": Value::Null, + }), + } +} + +fn held_target_mismatch(args: &Value, cursor_id: &str, hold: &MouseHoldState) -> Option { + match args.opt_u32("pid") { + Ok(Some(pid)) if pid != hold.pid => { + return Some( + ToolResult::error(format!( + "Cursor '{cursor_id}' is holding a button for pid {}, not pid {pid}.", + hold.pid + )) + .with_structured(mouse_hold_json(cursor_id, Some(hold))), + ); + } + Err(err) => return Some(err.with_structured(mouse_hold_json(cursor_id, Some(hold)))), + _ => {} + } + + match args.opt_u64("window_id") { + Some(xid) if xid != hold.xid => Some( + ToolResult::error(format!( + "Cursor '{cursor_id}' is holding a button for window_id {}, not {xid}.", + hold.xid + )) + .with_structured(mouse_hold_json(cursor_id, Some(hold))), + ), + _ => None, + } +} + async fn overlay_glide_to(sx: f64, sy: f64) { - if !crate::overlay::is_enabled() { + overlay_glide_to_for("default", sx, sy).await; +} + +fn overlay_snap_to_for(cursor_id: &str, sx: f64, sy: f64, heading: Option) { + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::SnapTo { + x: sx, + y: sy, + heading_radians: heading, + }, + ); +} + +fn overlay_move_to_for(cursor_id: &str, sx: f64, sy: f64, heading: Option) { + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::MoveTo { + x: sx, + y: sy, + end_heading_radians: heading.unwrap_or(std::f64::consts::FRAC_PI_4), + }, + ); +} + +async fn overlay_glide_to_for(cursor_id: &str, sx: f64, sy: f64) { + if !crate::overlay::is_enabled_for(cursor_id) { return; } - let pos = crate::overlay::current_position(); + let pos = crate::overlay::current_position_for(cursor_id); if pos.0 < 0.0 && pos.1 < 0.0 { - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.to_owned(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); return; } - crate::overlay::animate_cursor_to(sx, sy).await; + crate::overlay::animate_cursor_to_for(cursor_id.to_owned(), sx, sy).await; } fn process_name(pid: u32) -> Option { @@ -721,6 +843,8 @@ impl Tool for ClickTool { back to full-window space.".into(), input_schema: json!({ "type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -736,14 +860,10 @@ impl Tool for ClickTool { } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; let count = args.u64_or("count", 1) as usize; - let button: u8 = match args.str_or("button", "left").as_str() { - "right" => 3, - "middle" => 2, - _ => 1, - }; + let button = parse_mouse_button(args.str_or("button", "left").as_str()); if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; @@ -770,10 +890,16 @@ impl Tool for ClickTool { return match result { Ok(Ok((xid, x, y))) => { if xid != 0 { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); } - overlay_glide_to(x, y).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x, y }); + overlay_glide_to_for(&cursor_id, x, y).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x, y }, + ); ToolResult::text(format!("Clicked element [{idx}] (pid {pid}).")) } Ok(Err(e)) => ToolResult::error(format!("AT-SPI element click failed: {e}")), @@ -801,12 +927,18 @@ impl Tool for ClickTool { y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); @@ -1225,6 +1357,8 @@ impl Tool for DoubleClickTool { No focus steal. Provide either (window_id + x/y) or (pid + element_index). \ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(), input_schema: json!({"type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1237,6 +1371,7 @@ impl Tool for DoubleClickTool { } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; @@ -1247,9 +1382,15 @@ impl Tool for DoubleClickTool { return match result { Ok(Ok((xid, lx, ly))) => { if let Ok((sx, sy)) = element_screen_center(pid, idx) { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await { Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")), @@ -1278,12 +1419,18 @@ impl Tool for DoubleClickTool { x *= ratio; y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await; @@ -1311,6 +1458,8 @@ impl Tool for RightClickTool { No focus steal. Provide either (window_id + x/y) or (pid + element_index). \ After a zoom call, pass from_zoom=true to auto-translate zoom-image coords.".into(), input_schema: json!({"type":"object","required":["pid"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer"}, "x":{"type":"number"}, @@ -1324,6 +1473,7 @@ impl Tool for RightClickTool { } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; if let Some(idx) = args.opt_u64("element_index") { let idx = idx as usize; @@ -1334,9 +1484,15 @@ impl Tool for RightClickTool { return match result { Ok(Ok((xid, lx, ly))) => { if let Ok((sx, sy)) = element_screen_center(pid, idx) { - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await { Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")), @@ -1365,12 +1521,18 @@ impl Tool for RightClickTool { x *= ratio; y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); } let (xi, yi) = (x as i32, y as i32); let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await; @@ -1398,6 +1560,8 @@ impl Tool for DragTool { window-local screenshot pixels via XSendEvent (ButtonPress + MotionNotify × steps + ButtonRelease). \ duration_ms (default 500), steps (default 20). No focus steal.".into(), input_schema: json!({"type":"object","required":["pid","from_x","from_y","to_x","to_y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, "pid":{"type":"integer"}, "window_id":{"type":"integer","description":"Target window XID. Required."}, "from_x":{"type":"number"}, @@ -1414,7 +1578,7 @@ impl Tool for DragTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; + let cursor_id = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("window_id is required on Linux."), @@ -1431,7 +1595,7 @@ impl Tool for DragTool { let duration_ms = args.u64_or("duration_ms", 500); let steps = args.u64_or("steps", 20) as usize; let button_str = args.str_or("button", "left"); - let button: u8 = match button_str.as_str() { "right" => 3, "middle" => 2, _ => 1 }; + let button = parse_mouse_button(button_str.as_str()); let from_zoom = args.bool_or("from_zoom", false); if from_zoom { @@ -1448,46 +1612,636 @@ impl Tool for DragTool { to_x *= ratio; to_y *= ratio; } - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); if let Ok(Ok((sx_from, sy_from))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await { - overlay_glide_to(sx_from, sy_from).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { - x: sx_from, - y: sy_from, - }); + overlay_glide_to_for(&cursor_id, sx_from, sy_from).await; + self.state.cursor_registry.update_position(&cursor_id, sx_from, sy_from); + overlay_snap_to_for(&cursor_id, sx_from, sy_from, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx_from, y: sy_from }, + ); } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); - let result = tokio::task::spawn_blocking(move || { - crate::input::send_drag( - xid, - from_x as i32, from_y as i32, - to_x as i32, to_y as i32, - duration_ms, steps, button, - ) + let press_result = tokio::task::spawn_blocking(move || { + crate::input::send_button_down(xid, from_x.round() as i32, from_y.round() as i32, button) + }).await; + let mut result: anyhow::Result<()> = match press_result { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::anyhow!("Task error: {e}")), + }; + + if result.is_ok() { + let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; + let mut prev_x = from_x; + let mut prev_y = from_y; + for i in 1..=steps { + let t = i as f64 / steps.max(1) as f64; + let ix = from_x + (to_x - from_x) * t; + let iy = from_y + (to_y - from_y) * t; + let motion_result = tokio::task::spawn_blocking(move || { + crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button)) + }).await; + match motion_result { + Ok(Ok(())) => { + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await + { + let heading = if (ix - prev_x).abs() > f64::EPSILON + || (iy - prev_y).abs() > f64::EPSILON + { + Some((iy - prev_y).atan2(ix - prev_x)) + } else { + None + }; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_move_to_for(&cursor_id, sx, sy, heading); + } + prev_x = ix; + prev_y = iy; + if step_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms)).await; + } + } + Ok(Err(e)) => { + result = Err(e); + break; + } + Err(e) => { + result = Err(anyhow::anyhow!("Task error: {e}")); + break; + } + } + } + } + + let release_result = tokio::task::spawn_blocking(move || { + crate::input::send_button_up(xid, to_x.round() as i32, to_y.round() as i32, button) }).await; + if result.is_ok() { + result = match release_result { + Ok(Ok(())) => Ok(()), + Ok(Err(e)) => Err(e), + Err(e) => Err(anyhow::anyhow!("Task error: {e}")), + }; + } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(false), + ); - if matches!(&result, Ok(Ok(()))) { + if result.is_ok() { if let Ok(Ok((sx_to, sy_to))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await { - overlay_glide_to(sx_to, sy_to).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { - x: sx_to, - y: sy_to, - }); + self.state.cursor_registry.update_position(&cursor_id, sx_to, sy_to); + overlay_snap_to_for( + &cursor_id, + sx_to, + sy_to, + Some((to_y - from_y).atan2(to_x - from_x)), + ); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx_to, y: sy_to }, + ); } } match result { - Ok(Ok(())) => ToolResult::text(format!( + Ok(()) => ToolResult::text(format!( "✅ Posted drag ({button_str}) to pid {pid} \ from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \ in {duration_ms}ms / {steps} steps." )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), + Err(e) => ToolResult::error(e.to_string()), + } + } +} + +// ── mouse_button_down / mouse_drag / mouse_button_up ──────────────────────── + +pub struct MouseButtonDownTool { + state: Arc, +} +static MDOWN_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for MouseButtonDownTool { + fn def(&self) -> &ToolDef { + MDOWN_DEF.get_or_init(|| ToolDef { + name: "mouse_button_down".into(), + description: "Press and hold a mouse button at (x,y) via background X11 delivery. \ + Does not release the button; pair with mouse_drag / mouse_button_up. \ + Returns the current held-button state.".into(), + input_schema: json!({"type":"object","required":["pid","window_id","x","y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, + "pid":{"type":"integer"}, + "window_id":{"type":"integer"}, + "x":{"type":"number"}, + "y":{"type":"number"}, + "button":{"type":"string","enum":["left","right","middle"],"description":"Mouse button. Default: left."}, + "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."} + },"additionalProperties":false}), + read_only: false, destructive: true, idempotent: false, open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + let cursor_id = resolve_cursor_key(&args); + if let Some(held) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() { + return ToolResult::error(format!( + "Cursor '{cursor_id}' already has a held mouse button. Call mouse_button_up first." + )) + .with_structured(mouse_hold_json(&cursor_id, Some(&held))); + } + + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let xid = match args.opt_u64("window_id") { + Some(v) => v, + None => return ToolResult::error("window_id is required on Linux."), + }; + let button_name = args.str_or("button", "left"); + let button = parse_mouse_button(button_name.as_str()); + let mut x = args.f64_or("x", 0.0); + let mut y = args.f64_or("y", 0.0); + if args.bool_or("from_zoom", false) { + match self.state.zoom_registry.get(pid) { + Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } + None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {pid}. Call zoom first.")), + } + } else if let Some(ratio) = self.state.resize_registry.ratio(pid) { + x *= ratio; + y *= ratio; + } + + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { + overlay_glide_to_for(&cursor_id, sx, sy).await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); + } + + let xi = x as i32; + let yi = y as i32; + let result = tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await; + match result { + Ok(Ok(())) => { + let hold = MouseHoldState { cursor_id: cursor_id.clone(), pid, xid, button, x, y }; + self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone()); + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await + { + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); + } + ToolResult::text(format!( + "✅ Cursor '{cursor_id}' held {} button down at ({x:.1}, {y:.1}).", + mouse_button_name(button), + )) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) + } + Ok(Err(e)) => ToolResult::error(e.to_string()) + .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))), + Err(e) => ToolResult::error(format!("Task error: {e}")) + .with_structured(mouse_hold_json(&cursor_id, self.state.mouse_hold.lock().unwrap().get(&cursor_id))), + } + } +} + +pub struct MouseDragTool { + state: Arc, +} +static MDRAG_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for MouseDragTool { + fn def(&self) -> &ToolDef { + MDRAG_DEF.get_or_init(|| ToolDef { + name: "mouse_drag".into(), + description: "Move a previously-held mouse button to a new point via background X11 delivery. \ + Requires an active mouse_button_down state; does not release the button. \ + Returns the updated held-button state.".into(), + input_schema: json!({"type":"object","required":["x","y"],"properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, + "pid":{"type":"integer"}, + "window_id":{"type":"integer"}, + "x":{"type":"number"}, + "y":{"type":"number"}, + "duration_ms":{"type":"integer","minimum":0,"maximum":10000,"description":"Total drag duration. Default: 500."}, + "steps":{"type":"integer","minimum":1,"maximum":200,"description":"Intermediate MotionNotify events. Default: 20."}, + "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."} + },"additionalProperties":false}), + read_only: false, destructive: true, idempotent: false, open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + let cursor_id = resolve_cursor_key(&args); + let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else { + return ToolResult::error(format!( + "No mouse button is currently held for cursor '{cursor_id}'. Call mouse_button_down first." + )) + .with_structured(mouse_hold_json(&cursor_id, None)); + }; + if let Some(err) = held_target_mismatch(&args, &cursor_id, &hold) { + return err; + } + + let mut to_x = args.f64_or("x", 0.0); + let mut to_y = args.f64_or("y", 0.0); + if args.bool_or("from_zoom", false) { + match self.state.zoom_registry.get(hold.pid) { + Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(to_x, to_y); to_x = wx; to_y = wy; } + None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid)) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + } + } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { + to_x *= ratio; + to_y *= ratio; + } + + let xid = hold.xid; + + let from_x = hold.x; + let from_y = hold.y; + let duration_ms = args.u64_or("duration_ms", 500); + let steps = args.u64_or("steps", 20).max(1) as usize; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await { + overlay_glide_to_for(&cursor_id, sx, sy).await; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(true), + ); + } + + let button = hold.button; + let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms }; + let mut result: anyhow::Result<()> = Ok(()); + let mut prev_x = from_x; + let mut prev_y = from_y; + for i in 1..=steps { + let t = i as f64 / steps as f64; + let ix = from_x + (to_x - from_x) * t; + let iy = from_y + (to_y - from_y) * t; + let move_result = tokio::task::spawn_blocking(move || { + crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button)) + }).await; + match move_result { + Ok(Ok(())) => { + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, ix, iy)).await + { + let heading = if (ix - prev_x).abs() > f64::EPSILON || (iy - prev_y).abs() > f64::EPSILON { + Some((iy - prev_y).atan2(ix - prev_x)) + } else { + None + }; + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_move_to_for(&cursor_id, sx, sy, heading); + } + prev_x = ix; + prev_y = iy; + if step_delay_ms > 0 { + tokio::time::sleep(std::time::Duration::from_millis(step_delay_ms)).await; + } + } + Ok(Err(e)) => { + result = Err(e); + break; + } + Err(e) => { + result = Err(anyhow::anyhow!("Task error: {e}")); + break; + } + } + } + + match result { + Ok(()) => { + hold.x = to_x; + hold.y = to_y; + self.state.mouse_hold.lock().unwrap().insert(cursor_id.clone(), hold.clone()); + if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, to_x, to_y)).await { + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, Some((to_y - from_y).atan2(to_x - from_x))); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, + ); + } + ToolResult::text(format!( + "✅ Cursor '{cursor_id}' dragged held {} button to ({to_x:.1}, {to_y:.1}).", + mouse_button_name(hold.button), + )) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))) + } + Err(e) => ToolResult::error(e.to_string()) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + } + } +} + +pub struct MouseButtonUpTool { + state: Arc, +} +static MUP_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for MouseButtonUpTool { + fn def(&self) -> &ToolDef { + MUP_DEF.get_or_init(|| ToolDef { + name: "mouse_button_up".into(), + description: "Release a previously-held mouse button via background X11 delivery. \ + If x/y are omitted, releases at the last held position. Returns the current held-button state.".into(), + input_schema: json!({"type":"object","properties":{ + "session":{"type":"string","description":"Optional multi-cursor session id; takes precedence over cursor_id."}, + "cursor_id":{"type":"string","description":"Optional multi-cursor instance id. Default: 'default'."}, + "pid":{"type":"integer"}, + "window_id":{"type":"integer"}, + "x":{"type":"number"}, + "y":{"type":"number"}, + "from_zoom":{"type":"boolean","description":"Set true after a zoom call to auto-translate zoom-image pixel coordinates back to full-window space."} + },"additionalProperties":false}), + read_only: false, destructive: true, idempotent: false, open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + let cursor_id = resolve_cursor_key(&args); + let Some(mut hold) = self.state.mouse_hold.lock().unwrap().get(&cursor_id).cloned() else { + return ToolResult::error(format!("No mouse button is currently held for cursor '{cursor_id}'.")) + .with_structured(mouse_hold_json(&cursor_id, None)); + }; + if let Some(err) = held_target_mismatch(&args, &cursor_id, &hold) { + return err; + } + + let xid = hold.xid; + + let mut x = args.opt_f64("x").unwrap_or(hold.x); + let mut y = args.opt_f64("y").unwrap_or(hold.y); + if args.bool_or("from_zoom", false) { + match self.state.zoom_registry.get(hold.pid) { + Some(ctx) => { let (wx, wy) = ctx.zoom_to_window(x, y); x = wx; y = wy; } + None => return ToolResult::error(format!("from_zoom=true but no zoom context for pid {}. Call zoom first.", hold.pid)) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + } + } else if let Some(ratio) = self.state.resize_registry.ratio(hold.pid) { + x *= ratio; + y *= ratio; + } + + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::PinAbove(xid), + ); + if let Ok(Ok((sx, sy))) = tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await { + overlay_glide_to_for(&cursor_id, sx, sy).await; + } + + let button = hold.button; + let xi = x as i32; + let yi = y as i32; + let result = tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await; + match result { + Ok(Ok(())) => { + hold.x = x; + hold.y = y; + if let Ok(Ok((sx, sy))) = + tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)).await + { + self.state.cursor_registry.update_position(&cursor_id, sx, sy); + overlay_snap_to_for(&cursor_id, sx, sy, None); + } + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(false), + ); + self.state.mouse_hold.lock().unwrap().remove(&cursor_id); + let cleared = mouse_hold_json(&cursor_id, None); + ToolResult::text(format!( + "✅ Cursor '{cursor_id}' released held {} button at ({x:.1}, {y:.1}).", + mouse_button_name(button), + )) + .with_structured(cleared) + } + Ok(Err(e)) => ToolResult::error(e.to_string()) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + Err(e) => ToolResult::error(format!("Task error: {e}")) + .with_structured(mouse_hold_json(&cursor_id, Some(&hold))), + } + } +} + +pub struct ParallelMouseDragTool { + state: Arc, +} +static PMDRAG_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for ParallelMouseDragTool { + fn def(&self) -> &ToolDef { + PMDRAG_DEF.get_or_init(|| ToolDef { + name: "parallel_mouse_drag".into(), + description: "Run multiple mouse drag gestures concurrently via Linux MPX/XI2 virtual master pointers. \ + Each drag item runs on its own session-scoped master pointer (true same-window concurrent draws on X11). \ + Each item presses once, glides continuously through its whole path, and releases once — one smooth held \ + drag, not a chain of clicks. A path is given either as a straight segment (from_x/from_y → to_x/to_y) or \ + as a function `fn` = y(x) sampled over [x_from, x_to] in window-local pixels (e.g. fn:\"x\" is a diagonal, \ + fn:\"300+120*sin(x/40)\" a sine wave). Functions support + - * / ^, sin/cos/tan, sqrt, abs, exp, ln, pi, e.".into(), + input_schema: json!({"type":"object","required":["drags"],"properties":{ + "drags":{"type":"array","minItems":2,"items":{"type":"object","required":["session","window_id"],"properties":{ + "session":{"type":"string","description":"Session/cursor id; also keys the virtual master pointer."}, + "window_id":{"type":"integer"}, + "path":{"type":"array","items":{"type":"array","items":{"type":"number"}},"description":"Explicit window-local waypoints [[x,y],...] (>=2); pressed once, glided through, released once. Takes precedence over fn/from-to."}, + "fn":{"type":"string","description":"Expression y(x) in window-local pixels; sampled over [x_from,x_to]. Mutually exclusive with from_x/to_x."}, + "x_from":{"type":"number","description":"Domain start (window-local x) when `fn` is used."}, + "x_to":{"type":"number","description":"Domain end (window-local x) when `fn` is used."}, + "samples":{"type":"integer","minimum":2,"maximum":400,"description":"Waypoints sampled along `fn`. Default: 80."}, + "from_x":{"type":"number"}, + "from_y":{"type":"number"}, + "to_x":{"type":"number"}, + "to_y":{"type":"number"}, + "button":{"type":"string","enum":["left","right","middle"],"description":"Default: left."}, + "duration_ms":{"type":"integer","minimum":0,"maximum":10000,"description":"Default: 1500 for fn paths, 500 for straight."}, + "steps":{"type":"integer","minimum":1,"maximum":300,"description":"Motion sub-steps along the whole path. Default: scaled to path length."} + },"additionalProperties":false}} + },"additionalProperties":false}), + read_only: false, destructive: true, idempotent: false, open_world: true, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await { + Ok(Ok(())) => {} + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + } + + let Some(items) = args.get("drags").and_then(|v| v.as_array()) else { + return ToolResult::error("drags[] is required."); + }; + if items.len() < 2 { + return ToolResult::error("parallel_mouse_drag requires at least two drag items."); + } + + let mut drags = Vec::with_capacity(items.len()); + for item in items { + let Some(session) = item.get("session").and_then(|v| v.as_str()) else { + return ToolResult::error("each drag item requires session."); + }; + let Some(xid) = item.get("window_id").and_then(|v| v.as_u64()) else { + return ToolResult::error("each drag item requires window_id."); + }; + + // Build the window-local waypoint path from one of: an explicit + // `path` of [x,y] points, a function `fn` (y = f(x) sampled over + // [x_from, x_to]), or a straight from→to segment. + let is_fn = item.get("fn").and_then(|v| v.as_str()).is_some(); + let local: Vec<(f64, f64)> = if let Some(pts) = item.get("path").and_then(|v| v.as_array()) { + let mut out = Vec::with_capacity(pts.len()); + for p in pts { + let a = p.as_array(); + let (Some(px), Some(py)) = ( + a.and_then(|a| a.first()).and_then(|v| v.as_f64()), + a.and_then(|a| a.get(1)).and_then(|v| v.as_f64()), + ) else { + return ToolResult::error("each `path` entry must be [x, y]."); + }; + out.push((px, py)); + } + if out.len() < 2 { + return ToolResult::error("`path` needs at least 2 points."); + } + out + } else if let Some(expr_str) = item.get("fn").and_then(|v| v.as_str()) { + let Some(x_from) = item.get("x_from").and_then(|v| v.as_f64()) else { + return ToolResult::error("`fn` requires x_from."); + }; + let Some(x_to) = item.get("x_to").and_then(|v| v.as_f64()) else { + return ToolResult::error("`fn` requires x_to."); + }; + let samples = item.get("samples").and_then(|v| v.as_u64()).unwrap_or(80).clamp(2, 400); + match crate::input::sample_function(expr_str, x_from, x_to, samples) { + Ok(pts) => pts, + Err(e) => return ToolResult::error(e.to_string()), + } + } else { + let coerce = |k: &str| item.get(k).and_then(|v| v.as_f64()); + match (coerce("from_x"), coerce("from_y"), coerce("to_x"), coerce("to_y")) { + (Some(fx), Some(fy), Some(tx), Some(ty)) => vec![(fx, fy), (tx, ty)], + _ => return ToolResult::error("each drag item requires either `fn`+x_from+x_to, or from_x/from_y/to_x/to_y."), + } + }; + + let button = parse_mouse_button(item.get("button").and_then(|v| v.as_str()).unwrap_or("left")); + let duration_ms = item.get("duration_ms").and_then(|v| v.as_u64()) + .unwrap_or(if is_fn { 1500 } else { 500 }); + + // One translate gives the window origin; the path is a pure offset. + let origin = match tokio::task::spawn_blocking(move || window_local_to_screen(xid, 0.0, 0.0)).await { + Ok(Ok(o)) => o, + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + }; + let path: Vec<(i32, i32)> = local.iter() + .map(|(lx, ly)| ((origin.0 + lx).round() as i32, (origin.1 + ly).round() as i32)) + .collect(); + + // Default sub-step count scaled to path length (smooth glide), + // overridable via `steps`. + let total_len: f64 = path.windows(2) + .map(|w| (((w[1].0 - w[0].0) as f64).powi(2) + ((w[1].1 - w[0].1) as f64).powi(2)).sqrt()) + .sum(); + let steps = item.get("steps").and_then(|v| v.as_u64()) + .map(|s| (s as usize).clamp(1, 300)) + .unwrap_or_else(|| ((total_len / 3.0).round() as usize).clamp(24, 300)); + + let start = path[0]; + self.state.cursor_registry.update_position(session, start.0 as f64, start.1 as f64); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::PinAbove(xid)); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo { + x: start.0 as f64, + y: start.1 as f64, + heading_radians: None, + }); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(true)); + + drags.push(( + session.to_owned(), + crate::input::VirtualPointerDrag { + target_window: xid, + button, + path, + duration_ms, + steps, + }, + )); + } + + let drags_for_task = drags.clone(); + let result = tokio::task::spawn_blocking(move || crate::input::send_parallel_virtual_pointer_drags(&drags_for_task)).await; + match result { + Ok(Ok(())) => { + for (session, drag) in &drags { + let n = drag.path.len(); + let end = drag.path[n - 1]; + let prev = drag.path[n.saturating_sub(2)]; + self.state.cursor_registry.update_position(session, end.0 as f64, end.1 as f64); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SnapTo { + x: end.0 as f64, + y: end.1 as f64, + heading_radians: Some(((end.1 - prev.1) as f64).atan2((end.0 - prev.0) as f64)), + }); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { + x: end.0 as f64, + y: end.1 as f64, + }); + } + ToolResult::text(format!("✅ Ran {} MPX drag gesture(s) concurrently.", drags.len())) + .with_structured(json!({"count": drags.len()})) + } + Ok(Err(e)) => { + for (session, _) in &drags { + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + } + ToolResult::error(e.to_string()) + } + Err(e) => { + for (session, _) in &drags { + crate::overlay::send_command_for(session.to_owned(), cursor_overlay::OverlayCommand::SetPressed(false)); + } + ToolResult::error(format!("Task error: {e}")) + } } } } @@ -1586,7 +2340,7 @@ impl Tool for MoveCursorTool { name: "move_cursor".into(), description: "Move the agent cursor overlay to (x, y). Does NOT move the real mouse cursor.".into(), input_schema: json!({"type":"object","required":["x","y"],"properties":{ - "x":{"type":"number"},"y":{"type":"number"},"cursor_id":{"type":"string"} + "x":{"type":"number"},"y":{"type":"number"},"session":{"type":"string"},"cursor_id":{"type":"string"} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: false, }) @@ -1595,14 +2349,19 @@ impl Tool for MoveCursorTool { use cua_driver_core::tool_args::ArgsExt; let x = args.f64_or("x", 0.0); let y = args.f64_or("y", 0.0); - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.update_position(&cursor_id, x, y); // End pointing upper-left (45°) — matches Swift's // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so the // overlay arrow settles to the natural macOS-style pose. - crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { - x, y, end_heading_radians: std::f64::consts::FRAC_PI_4, - }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::MoveTo { + x, + y, + end_heading_radians: std::f64::consts::FRAC_PI_4, + }, + ); ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).")) } } @@ -1622,7 +2381,7 @@ impl Tool for SetAgentCursorEnabledTool { name: "set_agent_cursor_enabled".into(), description: "Show or hide the agent cursor overlay.".into(), input_schema: json!({"type":"object","required":["enabled"],"properties":{ - "enabled":{"type":"boolean"},"cursor_id":{"type":"string"} + "enabled":{"type":"boolean"},"session":{"type":"string"},"cursor_id":{"type":"string"} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: false, }) @@ -1630,9 +2389,12 @@ impl Tool for SetAgentCursorEnabledTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let enabled = match args.require_bool("enabled") { Ok(v) => v, Err(e) => return e }; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.set_enabled(&cursor_id, enabled); - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetEnabled(enabled), + ); ToolResult::text(format!("Agent cursor '{cursor_id}' {}.", if enabled { "enabled" } else { "disabled" })) } } @@ -1659,6 +2421,7 @@ impl Tool for SetAgentCursorMotionTool { - cursor_opacity: 0.0–1.0 (default=0.85)".into(), input_schema: json!({ "type":"object","properties":{ + "session":{"type":"string"}, "cursor_id":{"type":"string"}, "cursor_icon":{"type":"string"}, "cursor_color":{"type":"string"}, @@ -1671,8 +2434,7 @@ impl Tool for SetAgentCursorMotionTool { }) } async fn invoke(&self, args: Value) -> ToolResult { - use cua_driver_core::tool_args::ArgsExt; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); self.state.cursor_registry.update_config(&cursor_id, |cfg| { if let Some(v) = args.opt_str("cursor_icon") { cfg.cursor_icon = Some(v); } if let Some(v) = args.opt_str("cursor_color") { cfg.cursor_color = Some(v); } @@ -1698,12 +2460,17 @@ impl Tool for GetAgentCursorStateTool { GCSTATE_DEF.get_or_init(|| ToolDef { name: "get_agent_cursor_state".into(), description: "Return the current state of all agent cursor instances.".into(), - input_schema: json!({"type":"object","properties":{},"additionalProperties":false}), + input_schema: json!({"type":"object","properties":{"session":{"type":"string"},"cursor_id":{"type":"string"}},"additionalProperties":false}), read_only: true, destructive: false, idempotent: true, open_world: false, }) } - async fn invoke(&self, _args: Value) -> ToolResult { - let states = self.state.cursor_registry.all_states(); + async fn invoke(&self, args: Value) -> ToolResult { + let cursor_id = resolve_cursor_key(&args); + let states = if args.get("session").is_some() || args.get("cursor_id").is_some() { + vec![self.state.cursor_registry.get_or_create(&cursor_id)] + } else { + self.state.cursor_registry.all_states() + }; let json = serde_json::to_value(&states).unwrap_or_default(); ToolResult::text(format!("{} cursor instance(s).", states.len())) .with_structured(json!({ "cursors": json })) @@ -1738,6 +2505,10 @@ impl Tool for SetAgentCursorStyleTool { input_schema: json!({ "type": "object", "properties": { + "session": { + "type": "string", + "description": "Optional multi-cursor session id; takes precedence over cursor_id." + }, "cursor_id": { "type": "string", "description": "Cursor instance. Default: 'default'." @@ -1764,7 +2535,7 @@ impl Tool for SetAgentCursorStyleTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - let cursor_id = args.str_or("cursor_id", "default"); + let cursor_id = resolve_cursor_key(&args); // image_path let image_path = args.get("image_path").and_then(|v| v.as_str()); @@ -1823,15 +2594,18 @@ impl Tool for SetAgentCursorStyleTool { // Dispatch to overlay if let Some(cmd) = shape_cmd { - crate::overlay::send_command(cmd); + crate::overlay::send_command_for(cursor_id.clone(), cmd); } let gradient_provided = args.get("gradient_colors").is_some(); let bloom_provided = args.get("bloom_color").is_some(); if gradient_provided || bloom_provided { - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient { - gradient_colors, - bloom_color: bloom_color.flatten(), - }); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetGradient { + gradient_colors, + bloom_color: bloom_color.flatten(), + }, + ); } let grad_str = args.get("gradient_colors") @@ -2303,6 +3077,16 @@ impl Tool for BringToFrontTool { pub fn build_registry(compat: bool) -> ToolRegistry { let state = ToolState::new(); + { + let cursor_registry = state.cursor_registry.clone(); + let state_for_session_end = state.clone(); + cua_driver_core::session::register_session_end_hook(move |session_id| { + cursor_registry.remove(session_id); + crate::overlay::remove_cursor(session_id.to_owned()); + state_for_session_end.mouse_hold.lock().unwrap().remove(session_id); + crate::input::forget_master_pointer(session_id); + }); + } let mut r = ToolRegistry::new(); r.register(Box::new(ListAppsTool)); r.register(Box::new(ListWindowsTool)); @@ -2314,6 +3098,10 @@ pub fn build_registry(compat: bool) -> ToolRegistry { r.register(Box::new(DoubleClickTool { state: state.clone() })); r.register(Box::new(RightClickTool { state: state.clone() })); r.register(Box::new(DragTool { state: state.clone() })); + r.register(Box::new(MouseButtonDownTool { state: state.clone() })); + r.register(Box::new(MouseDragTool { state: state.clone() })); + r.register(Box::new(MouseButtonUpTool { state: state.clone() })); + r.register(Box::new(ParallelMouseDragTool { state: state.clone() })); r.register(Box::new(TypeTextTool)); r.register(Box::new(PressKeyTool)); r.register(Box::new(HotkeyTool)); diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix index d34ffb3302..50a41c973a 100644 --- a/nix/cua-driver/package.nix +++ b/nix/cua-driver/package.nix @@ -25,7 +25,7 @@ pkgs.rustPlatform.buildRustPackage { # gracefully via `cargo vendor`. # Bumped when the dependency set changes (added `atspi`/zbus for native # AT-SPI). If this mismatches, the nix build prints the expected value. - cargoHash = "sha256-P+f+ma8ZDWhhk1TTCGgbLTp4zU/uuh4vHYYQMIjlCbU="; + cargoHash = "sha256-8e/8inqyEUJA3s9lp/YGU5ckDXV1PnUVTyArwAM1e5g="; # Build only the main binary crate. The workspace also contains # platform-macos, platform-windows, cua-driver-uia, and focus-monitor-win @@ -34,13 +34,20 @@ pkgs.rustPlatform.buildRustPackage { cargoBuildFlags = [ "-p" "cua-driver" ]; cargoTestFlags = [ "-p" "cua-driver" ]; - # The entire Linux dependency chain is pure Rust: + # Mostly pure Rust: # x11rb -> RustConnection (no libxcb C binding) # ureq -> rustls (no openssl) # tiny-skia -> pure Rust 2D graphics # ring -> compiles own C/asm via stdenv's cc - nativeBuildInputs = [ ]; - buildInputs = [ ]; + # Except the `x11` crate (raw Xlib FFI for MPX multi-cursor drags), whose + # build.rs locates libX11/libXi/libXtst via pkg-config. + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = with pkgs; [ + libx11 + libxi + libxtst + libxext + ]; # Skip tests that require a running X11 display or AT-SPI bus doCheck = false; diff --git a/nix/cua-driver/tests/linux-parallel-drag-gif.nix b/nix/cua-driver/tests/linux-parallel-drag-gif.nix new file mode 100644 index 0000000000..dc2562211f --- /dev/null +++ b/nix/cua-driver/tests/linux-parallel-drag-gif.nix @@ -0,0 +1,387 @@ +# Linux parallel multi-cursor drag GIF test +# +# NOT WIRED INTO CI (flake checks / nix-build.yml). It needs a real Xorg +# (dummy video + libinput) so the MPX path's uinput slaves enumerate as X +# input devices, and that server does not start reliably in the emulated +# GHA nixos-test VM (the hand-launched Xorg times out before the socket +# appears). The feature's logic is covered by unit tests in +# platform-linux/src/input/mod.rs (arc-length path glide + fn sampling); +# this scenario is kept for local / real-X manual runs: +# nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-gif +# (re-add the flake check + matrix entry to run it where a real Xorg works). +# +# Pilots cua-driver through its Linux MPX `parallel_mouse_drag` path: two +# per-session master pointers drawing concurrent strokes into the SAME window, +# while a separate control window keeps the input focus. Proves the three +# guarantees the feature is built on: +# +# 1. Concurrent delivery — both masters' presses/motions/releases reach the +# target window as cooked, window-targeted XI2 events (an XI2 paint app +# logs every event with its device id; we assert two distinct devices). +# 2. No focus steal — the "shield grab" keeps the window manager blind to +# the presses, so the control window stays active throughout. +# 3. Pixel-exact endpoints — the recorded GIF shows two crossing strokes. +# +# Unlike the other Linux visual tests, this one needs a REAL Xorg server, not +# Xvfb: the MPX path attaches uinput slave devices to per-session master +# pointers, and only a real Xorg with the libinput input driver enumerates +# uinput devices as X input devices (Xvfb — like the old Xtigervnc setup — +# does not). So we launch Xorg with the `dummy` video driver + libinput, with +# the `uinput` kernel module loaded. +# +# To run: nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-gif +# +{ + pkgs, + lib ? pkgs.lib, + cuaDriverModule, + ... +}: + +let + # XI2 paint + event logger. Selects XI2 events for ALL master devices on its + # own window and, for each ButtonPress/Motion/ButtonRelease, appends a line + # " dev= x=<> y=<>" to a log file and paints a square (per-device + # colour). The log is what proves cooked, window-delivered, multi-device + # input — not just raw motion. Prints "READY 0x" on stdout at startup. + xi2paintSrc = pkgs.writeText "xi2paint.c" '' + #include + #include + #include + #include + #include + + int main(int argc, char **argv) { + const char *logpath = argc > 1 ? argv[1] : "/tmp/xi2paint-events.log"; + Display *dpy = XOpenDisplay(NULL); + if (!dpy) { fprintf(stderr, "no display\n"); return 1; } + int xi_opcode, ev, err; + if (!XQueryExtension(dpy, "XInputExtension", &xi_opcode, &ev, &err)) { + fprintf(stderr, "no XInputExtension\n"); return 1; + } + int major = 2, minor = 3; + XIQueryVersion(dpy, &major, &minor); + + int scr = DefaultScreen(dpy); + Window win = XCreateSimpleWindow(dpy, RootWindow(dpy, scr), 40, 40, 800, 600, + 1, BlackPixel(dpy, scr), WhitePixel(dpy, scr)); + XStoreName(dpy, win, "XI2 MPX Paint"); + XSelectInput(dpy, win, ExposureMask); + + unsigned char mask[XIMaskLen(XI_LASTEVENT)]; + memset(mask, 0, sizeof mask); + XISetMask(mask, XI_ButtonPress); + XISetMask(mask, XI_Motion); + XISetMask(mask, XI_ButtonRelease); + XISetMask(mask, XI_Enter); + XISetMask(mask, XI_FocusIn); + XIEventMask em; + em.deviceid = XIAllMasterDevices; + em.mask_len = sizeof mask; + em.mask = mask; + XISelectEvents(dpy, win, &em, 1); + XMapWindow(dpy, win); + XFlush(dpy); + GC gc = XCreateGC(dpy, win, 0, NULL); + + FILE *logf = fopen(logpath, "w"); + if (!logf) { fprintf(stderr, "cannot open log\n"); return 1; } + setvbuf(logf, NULL, _IOLBF, 0); + printf("READY 0x%lx\n", win); fflush(stdout); + fprintf(logf, "READY 0x%lx\n", win); + + int down[256]; memset(down, 0, sizeof down); + unsigned long colors[] = { 0xd00000, 0x0040d0, 0x00a000, 0xc08000 }; + for (;;) { + XEvent e; + XNextEvent(dpy, &e); + if (e.type == GenericEvent && e.xcookie.extension == xi_opcode && + XGetEventData(dpy, &e.xcookie)) { + int t = e.xcookie.evtype; + if (t == XI_ButtonPress || t == XI_Motion || t == XI_ButtonRelease) { + XIDeviceEvent *de = e.xcookie.data; + const char *n = t == XI_ButtonPress ? "PRESS" + : t == XI_Motion ? "MOTION" : "RELEASE"; + fprintf(logf, "%s dev=%d x=%.0f y=%.0f\n", + n, de->deviceid, de->event_x, de->event_y); + int d = de->deviceid & 255; + if (t == XI_ButtonPress) down[d] = 1; + if (t == XI_ButtonRelease) down[d] = 0; + if ((t == XI_Motion && down[d]) || t == XI_ButtonPress) { + XSetForeground(dpy, gc, colors[de->deviceid % 4]); + XFillRectangle(dpy, win, gc, (int)de->event_x - 3, + (int)de->event_y - 3, 6, 6); + XFlush(dpy); + } + } else if (t == XI_Enter || t == XI_FocusIn) { + XIEnterEvent *ee = e.xcookie.data; + fprintf(logf, "%s dev=%d\n", t == XI_Enter ? "ENTER" : "FOCUSIN", + ee->deviceid); + } + XFreeEventData(dpy, &e.xcookie); + } + } + return 0; + } + ''; + + xi2paint = pkgs.runCommandCC "xi2paint" { + buildInputs = [ pkgs.xorg.libX11 pkgs.xorg.libXi ]; + } '' + mkdir -p $out/bin + cc -O2 -o $out/bin/xi2paint ${xi2paintSrc} -lX11 -lXi + ''; + + # Real Xorg config: dummy video driver (software framebuffer) + libinput + # input hotplug so cua-driver's uinput slaves get enumerated as X devices. + xorgConf = pkgs.writeText "xorg-dummy.conf" '' + Section "ServerFlags" + Option "AutoAddDevices" "true" + Option "AutoEnableDevices" "true" + Option "DontVTSwitch" "true" + EndSection + Section "Device" + Identifier "dummy" + Driver "dummy" + VideoRam 256000 + EndSection + Section "Monitor" + Identifier "mon" + HorizSync 30.0 - 1000.0 + VertRefresh 30.0 - 200.0 + Modeline "1280x1024" 109.00 1280 1368 1496 1712 1024 1027 1034 1063 -hsync +vsync + EndSection + Section "Screen" + Identifier "screen" + Device "dummy" + Monitor "mon" + DefaultDepth 24 + SubSection "Display" + Depth 24 + Modes "1280x1024" + EndSubSection + EndSection + ''; + + # Manually-launched Xorg needs an explicit module path covering the server's + # own modules plus the separately-packaged dummy video + libinput drivers. + xorgModulePath = lib.concatStringsSep "," [ + "${pkgs.xorg.xorgserver}/lib/xorg/modules" + "${pkgs.xorg.xf86videodummy}/lib/xorg/modules/drivers" + "${pkgs.xorg.xf86inputlibinput}/lib/xorg/modules/input" + ]; + + mcpDragTest = pkgs.writeText "mcp-parallel-drag-test.py" '' + import json + import os + import subprocess + import sys + import threading + import time + + DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") + + def start_driver(): + proc = subprocess.Popen( + [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env={**os.environ}, + ) + + def drain_stderr(): + for line in proc.stderr: + sys.stderr.buffer.write(line) + sys.stderr.buffer.flush() + + threading.Thread(target=drain_stderr, daemon=True).start() + return proc + + def send(proc, method, params=None, req_id=None): + msg = {"jsonrpc": "2.0", "method": method} + if params is not None: + msg["params"] = params + if req_id is not None: + msg["id"] = req_id + proc.stdin.write((json.dumps(msg) + "\n").encode()) + proc.stdin.flush() + + def recv(proc, timeout=30): + result = [None] + + def reader(): + result[0] = proc.stdout.readline() + + thread = threading.Thread(target=reader) + thread.start() + thread.join(timeout) + if thread.is_alive(): + raise TimeoutError("No response within timeout") + line = result[0].decode().strip() + if not line: + raise RuntimeError("Driver returned an empty response") + return json.loads(line) + + def call_tool(proc, req_id, name, arguments, timeout=60): + send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id) + resp = recv(proc, timeout=timeout) + if "error" in resp and resp["error"] is not None: + raise RuntimeError(f"{name} failed: {resp}") + if resp.get("result", {}).get("isError"): + raise RuntimeError(f"{name} returned isError: {resp}") + return resp + + def main(): + with open("/tmp/paint-xid.txt", "r", encoding="utf-8") as f: + window_id = int(f.readline().strip()) + + proc = start_driver() + try: + send(proc, "initialize", { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "nixos-parallel-drag-test", "version": "1.0.0"}, + }, req_id=1) + recv(proc) + send(proc, "notifications/initialized", {}) + time.sleep(0.3) + + # Two concurrent strokes forming the left half of an "X" with two + # cursors, into a window that does NOT hold focus. + call_tool(proc, 2, "parallel_mouse_drag", {"drags": [ + {"session": "agent-1", "window_id": window_id, + "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, + "duration_ms": 2500, "steps": 80}, + {"session": "agent-2", "window_id": window_id, + "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, + "duration_ms": 2500, "steps": 80}, + ]}) + time.sleep(0.6) + + # Second pass completes both "X" shapes. + call_tool(proc, 3, "parallel_mouse_drag", {"drags": [ + {"session": "agent-1", "window_id": window_id, + "from_x": 100.0, "from_y": 420.0, "to_x": 380.0, "to_y": 120.0, + "duration_ms": 2500, "steps": 80}, + {"session": "agent-2", "window_id": window_id, + "from_x": 700.0, "from_y": 420.0, "to_x": 420.0, "to_y": 120.0, + "duration_ms": 2500, "steps": 80}, + ]}) + time.sleep(0.6) + print("parallel drag test complete", flush=True) + finally: + proc.stdin.close() + proc.terminate() + proc.wait(timeout=5) + + if __name__ == "__main__": + main() + ''; + + recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; +in + +pkgs.testers.nixosTest { + name = "cua-driver-linux-parallel-drag-gif-test"; + meta.maintainers = [ ]; + + nodes.machine = + { + pkgs, + ... + }: + { + imports = [ cuaDriverModule ]; + virtualisation = { + cores = 2; + memorySize = 2048; + }; + services.cua-driver.enable = true; + # The MPX path opens /dev/uinput and relies on Xorg+libinput enumerating + # the resulting devices, so the uinput module must be present. + boot.kernelModules = [ "uinput" ]; + environment.systemPackages = with pkgs; [ + xorg.xorgserver + xorg.xf86videodummy + xorg.xf86inputlibinput + xorg.xinput + xi2paint + xterm + openbox + picom + xdotool + imagemagick + python3 + jq + procps + ]; + }; + + testScript = '' + machine.start() + machine.wait_for_unit("multi-user.target") + machine.succeed("modprobe uinput && test -e /dev/uinput") + + with subtest("Start a real Xorg (dummy video + libinput) and a WM"): + machine.execute( + "Xorg :99 -ac -noreset -keeptty -nolisten tcp " + "-config ${xorgConf} -modulepath ${xorgModulePath} " + "-logfile /tmp/xorg.log >/tmp/xorg.out 2>&1 &" + ) + machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=30) + machine.wait_until_succeeds("DISPLAY=:99 xdpyinfo >/dev/null 2>&1", timeout=30) + machine.execute("DISPLAY=:99 openbox >/tmp/openbox.log 2>&1 &") + machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &") + + with subtest("Launch the XI2 paint target and a control window that holds focus"): + machine.execute( + "sh -lc \"DISPLAY=:99 xi2paint /tmp/xi2paint-events.log >/tmp/paint.log 2>&1 & echo \\$! >/tmp/paint-pid.txt\"" + ) + machine.execute( + "sh -lc \"DISPLAY=:99 xterm -T 'Control' -fa Monospace -fs 14 -geometry 50x12+980+120 >/tmp/control.log 2>&1 & echo \\$! >/tmp/control-pid.txt\"" + ) + machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --name 'XI2 MPX Paint' >/tmp/paint-xid.txt", timeout=20) + machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20) + # Give input focus to the CONTROL window — the drags target the paint + # window, which must never steal it. + machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") + machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)") + + with subtest("Record GIF and pilot cua-driver through parallel_mouse_drag"): + machine.copy_from_host("${mcpDragTest}", "/tmp/mcp-parallel-drag-test.py") + machine.execute( + "sh -lc '${recordGifScript} :99 /tmp/drag-frames /tmp/cua-driver-linux-parallel-drag.gif " + "/tmp/stop-drag-recorder /tmp/ffmpeg-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-drag.pid'" + ) + result = machine.succeed("timeout 90 env DISPLAY=:99 python3 /tmp/mcp-parallel-drag-test.py 2>&1") + machine.log(result) + assert "parallel drag test complete" in result, result + machine.succeed("touch /tmp/stop-drag-recorder") + machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-drag.pid) 2>/dev/null", timeout=60) + machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-drag.log || true'")) + + with subtest("Both cursors delivered cooked, window-targeted events"): + machine.log(machine.succeed("cat /tmp/xi2paint-events.log")) + # Four presses total (two strokes x two cursors). + machine.succeed("test \"$(grep -c '^PRESS ' /tmp/xi2paint-events.log)\" -eq 4") + # From two DISTINCT master pointer devices — i.e. genuinely concurrent + # multi-cursor input, not one pointer reused. + distinct = machine.succeed( + "grep '^PRESS ' /tmp/xi2paint-events.log | sed -n 's/.*dev=\\([0-9]*\\).*/\\1/p' | sort -u | wc -l" + ).strip() + assert int(distinct) == 2, f"expected 2 distinct devices, got {distinct}" + machine.succeed("test \"$(grep -c '^MOTION ' /tmp/xi2paint-events.log)\" -ge 4") + + with subtest("The drags did not steal focus from the control window"): + active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() + control = machine.succeed("head -1 /tmp/control-xid.txt").strip() + assert active == control, f"focus stolen: active={active} control={control}" + + with subtest("GIF artifact exists"): + machine.succeed("test -s /tmp/cua-driver-linux-parallel-drag.gif") + + with subtest("Copy GIF out of the VM"): + machine.copy_from_machine("/tmp/cua-driver-linux-parallel-drag.gif", "") + ''; +} diff --git a/nix/cua-driver/tests/linux-parallel-drag-xserver.nix b/nix/cua-driver/tests/linux-parallel-drag-xserver.nix new file mode 100644 index 0000000000..f812ca5482 --- /dev/null +++ b/nix/cua-driver/tests/linux-parallel-drag-xserver.nix @@ -0,0 +1,314 @@ +# Linux parallel multi-cursor drag test — real Xorg via services.xserver +# +# A CI-viable rewrite of linux-parallel-drag-gif.nix. The MPX path needs a +# REAL Xorg with the libinput input backend so cua-driver's uinput slaves +# enumerate as X input devices (Xvfb can't; a hand-launched Xorg couldn't get +# a VT/seat in the emulated nixos-test VM and timed out). Here NixOS's +# services.xserver brings up Xorg properly on a seat via a display manager, +# with the `dummy` video driver (software framebuffer, headless) and libinput. +# A normal user is auto-logged-in to an icewm session; the session runs +# `xhost +local:` so the root-run test driver / cua-driver can connect to :0. +# +# Proves: two per-session master pointers draw concurrent cooked, window- +# targeted XI2 events into one window (assert 2 distinct devices), the shield +# grab keeps focus on a separate control window, and a GIF is produced. +# +# To run: nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-xserver +{ + pkgs, + lib ? pkgs.lib, + cuaDriverModule, + ... +}: + +let + # XI2 paint + event logger (selects XI2 for all master devices; logs each + # press/motion/release with its device id and paints per-device squares). + xi2paintSrc = pkgs.writeText "xi2paint.c" '' + #include + #include + #include + #include + + int main(int argc, char **argv) { + const char *logpath = argc > 1 ? argv[1] : "/tmp/xi2paint-events.log"; + Display *dpy = XOpenDisplay(NULL); + if (!dpy) { fprintf(stderr, "no display\n"); return 1; } + int xi_opcode, ev, err; + if (!XQueryExtension(dpy, "XInputExtension", &xi_opcode, &ev, &err)) { + fprintf(stderr, "no XInputExtension\n"); return 1; + } + int major = 2, minor = 3; + XIQueryVersion(dpy, &major, &minor); + int scr = DefaultScreen(dpy); + Window win = XCreateSimpleWindow(dpy, RootWindow(dpy, scr), 40, 40, 800, 600, + 1, BlackPixel(dpy, scr), WhitePixel(dpy, scr)); + XStoreName(dpy, win, "XI2 MPX Paint"); + XSelectInput(dpy, win, ExposureMask); + unsigned char mask[XIMaskLen(XI_LASTEVENT)]; + memset(mask, 0, sizeof mask); + XISetMask(mask, XI_ButtonPress); + XISetMask(mask, XI_Motion); + XISetMask(mask, XI_ButtonRelease); + XIEventMask em; + em.deviceid = XIAllMasterDevices; + em.mask_len = sizeof mask; + em.mask = mask; + XISelectEvents(dpy, win, &em, 1); + XMapWindow(dpy, win); + XFlush(dpy); + GC gc = XCreateGC(dpy, win, 0, NULL); + FILE *logf = fopen(logpath, "w"); + if (!logf) { fprintf(stderr, "cannot open log\n"); return 1; } + setvbuf(logf, NULL, _IOLBF, 0); + printf("READY 0x%lx\n", win); fflush(stdout); + int down[256]; memset(down, 0, sizeof down); + unsigned long colors[] = { 0xd00000, 0x0040d0, 0x00a000, 0xc08000 }; + for (;;) { + XEvent e; + XNextEvent(dpy, &e); + if (e.type == GenericEvent && e.xcookie.extension == xi_opcode && + XGetEventData(dpy, &e.xcookie)) { + int t = e.xcookie.evtype; + if (t == XI_ButtonPress || t == XI_Motion || t == XI_ButtonRelease) { + XIDeviceEvent *de = e.xcookie.data; + const char *n = t == XI_ButtonPress ? "PRESS" + : t == XI_Motion ? "MOTION" : "RELEASE"; + fprintf(logf, "%s dev=%d x=%.0f y=%.0f\n", + n, de->deviceid, de->event_x, de->event_y); + int d = de->deviceid & 255; + if (t == XI_ButtonPress) down[d] = 1; + if (t == XI_ButtonRelease) down[d] = 0; + if ((t == XI_Motion && down[d]) || t == XI_ButtonPress) { + XSetForeground(dpy, gc, colors[de->deviceid % 4]); + XFillRectangle(dpy, win, gc, (int)de->event_x - 3, + (int)de->event_y - 3, 6, 6); + XFlush(dpy); + } + } + XFreeEventData(dpy, &e.xcookie); + } + } + return 0; + } + ''; + + xi2paint = pkgs.runCommandCC "xi2paint" { + buildInputs = [ pkgs.xorg.libX11 pkgs.xorg.libXi ]; + } '' + mkdir -p $out/bin + cc -O2 -o $out/bin/xi2paint ${xi2paintSrc} -lX11 -lXi + ''; + + mcpDragTest = pkgs.writeText "mcp-parallel-drag-test.py" '' + import json, os, subprocess, sys, threading, time + DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") + + def start_driver(): + proc = subprocess.Popen([DRIVER_BIN, "mcp", "--no-daemon-relaunch"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={**os.environ}) + def drain(): + for line in proc.stderr: + sys.stderr.buffer.write(line); sys.stderr.buffer.flush() + threading.Thread(target=drain, daemon=True).start() + return proc + + def send(proc, method, params=None, req_id=None): + msg = {"jsonrpc": "2.0", "method": method} + if params is not None: msg["params"] = params + if req_id is not None: msg["id"] = req_id + proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush() + + def recv(proc, timeout=30): + result = [None] + def reader(): result[0] = proc.stdout.readline() + th = threading.Thread(target=reader); th.start(); th.join(timeout) + if th.is_alive(): raise TimeoutError("no response") + line = result[0].decode().strip() + if not line: raise RuntimeError("empty response") + return json.loads(line) + + def call_tool(proc, rid, name, args, timeout=60): + send(proc, "tools/call", {"name": name, "arguments": args}, req_id=rid) + resp = recv(proc, timeout=timeout) + if resp.get("error"): raise RuntimeError(f"{name} failed: {resp}") + if resp.get("result", {}).get("isError"): raise RuntimeError(f"{name} isError: {resp}") + return resp + + def main(): + with open("/tmp/paint-xid.txt") as f: + wid = int(f.readline().strip()) + proc = start_driver() + try: + send(proc, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, + "clientInfo": {"name": "nixos-parallel-drag-xserver", "version": "1.0.0"}}, req_id=1) + recv(proc); send(proc, "notifications/initialized", {}); time.sleep(0.3) + # Two concurrent strokes (one per cursor) into an UNFOCUSED window; + # each is a single held-path drag (press once, glide, release once). + call_tool(proc, 2, "parallel_mouse_drag", {"drags": [ + {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, "duration_ms": 2200, "steps": 80}, + {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, "duration_ms": 2200, "steps": 80}, + ]}) + time.sleep(0.6) + call_tool(proc, 3, "parallel_mouse_drag", {"drags": [ + {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 420.0, "to_x": 380.0, "to_y": 120.0, "duration_ms": 2200, "steps": 80}, + {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 420.0, "to_x": 420.0, "to_y": 120.0, "duration_ms": 2200, "steps": 80}, + ]}) + time.sleep(0.6) + print("parallel drag test complete", flush=True) + finally: + proc.stdin.close(); proc.terminate(); proc.wait(timeout=5) + + if __name__ == "__main__": + main() + ''; + + recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; +in + +pkgs.testers.nixosTest { + name = "cua-driver-linux-parallel-drag-xserver-test"; + meta.maintainers = [ ]; + + nodes.machine = + { pkgs, ... }: + { + imports = [ cuaDriverModule ]; + virtualisation = { + cores = 2; + memorySize = 2048; + }; + services.cua-driver.enable = true; + boot.kernelModules = [ "uinput" ]; + # Root opens /dev/uinput directly; the group rule is belt-and-suspenders. + services.udev.extraRules = '' + KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" + ''; + + # Real Xorg on a seat (DM handles the VT), dummy video, libinput input. + services.xserver = { + enable = true; + videoDrivers = [ "dummy" ]; + deviceSection = ''VideoRam 256000''; + monitorSection = '' + HorizSync 30.0 - 1000.0 + VertRefresh 30.0 - 200.0 + Modeline "1280x1024" 109.00 1280 1368 1496 1712 1024 1027 1034 1063 -hsync +vsync + ''; + screenSection = '' + DefaultDepth 24 + SubSection "Display" + Depth 24 + Modes "1280x1024" + Virtual 1280 1024 + EndSubSection + ''; + windowManager.icewm.enable = true; + # A display manager is what actually starts Xorg on a seat/VT — the bit + # the hand-launched Xorg couldn't arrange in the emulated VM. lightdm is + # the lightest. (lightdm.enable still lives under xserver.displayManager; + # autoLogin/defaultSession moved to the top-level services.displayManager.) + displayManager.lightdm.enable = true; + # Disable X access control outright: this is a throwaway single-user + # test VM, and the root-run cua-driver / clients must reach :0. Relying + # on the session's `xhost +local:` proved unreliable across uids (the + # server's auth ACL only lists the autologin user), so `-ac` is the + # bulletproof grant. `xhost +local:` is kept as belt-and-suspenders. + displayManager.xserverArgs = [ "-ac" ]; + displayManager.sessionCommands = '' + ${pkgs.xorg.xhost}/bin/xhost +local: || true + ''; + }; + services.libinput.enable = true; + services.displayManager = { + defaultSession = "none+icewm"; + autoLogin = { + enable = true; + user = "cua"; + }; + }; + users.users.cua = { + isNormalUser = true; + extraGroups = [ "input" ]; + }; + + environment.systemPackages = with pkgs; [ + xorg.xinput + xorg.xhost + xorg.xdpyinfo # the :0 connectivity probe in the test script + xi2paint + xterm + xdotool + imagemagick + python3 + jq + procps + ]; + }; + + testScript = '' + machine.start() + machine.wait_for_unit("multi-user.target") + machine.succeed("modprobe uinput && test -e /dev/uinput") + + with subtest("Real Xorg up via the display manager"): + try: + machine.wait_for_x() + # With `-ac` the root-run clients can connect to :0 immediately. + machine.wait_until_succeeds("DISPLAY=:0 xdpyinfo >/dev/null 2>&1", timeout=30) + except Exception: + # Diagnostics: surface why X/lightdm did not come up or why root + # can't reach :0, so we don't burn a CI round-trip guessing. + machine.log(machine.execute("systemctl status display-manager.service --no-pager || true")[1]) + machine.log(machine.execute("journalctl -u display-manager.service --no-pager | tail -n 200 || true")[1]) + machine.log(machine.execute("ls -la /tmp/.X11-unix/ || true")[1]) + machine.log(machine.execute("cat /var/log/X.0.log 2>/dev/null | tail -n 200 || true")[1]) + machine.log(machine.execute("find / -name 'Xorg.0.log' 2>/dev/null | head; cat $(find / -name 'Xorg.0.log' 2>/dev/null | head -1) 2>/dev/null | tail -n 120 || true")[1]) + raise + machine.log(machine.succeed("DISPLAY=:0 xinput list --short || true")) + + with subtest("Launch XI2 paint target + a control window that holds focus"): + machine.execute( + "sh -lc \"DISPLAY=:0 xi2paint /tmp/xi2paint-events.log >/tmp/paint.log 2>&1 & echo \\$! >/tmp/paint-pid.txt\"" + ) + machine.execute( + "sh -lc \"DISPLAY=:0 xterm -T 'Control' -fa Monospace -fs 14 -geometry 50x12+980+120 >/tmp/control.log 2>&1 & echo \\$! >/tmp/control-pid.txt\"" + ) + machine.wait_until_succeeds("DISPLAY=:0 xdotool search --sync --name 'XI2 MPX Paint' >/tmp/paint-xid.txt", timeout=20) + machine.wait_until_succeeds("DISPLAY=:0 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20) + machine.succeed("DISPLAY=:0 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") + machine.succeed("DISPLAY=:0 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)") + + with subtest("Record GIF and pilot cua-driver through parallel_mouse_drag"): + machine.copy_from_host("${mcpDragTest}", "/tmp/mcp-parallel-drag-test.py") + machine.execute( + "sh -lc '${recordGifScript} :0 /tmp/drag-frames /tmp/cua-driver-linux-parallel-drag-xserver.gif " + "/tmp/stop-drag-recorder /tmp/ffmpeg-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-drag.pid'" + ) + result = machine.succeed("timeout 90 env DISPLAY=:0 python3 /tmp/mcp-parallel-drag-test.py 2>&1") + machine.log(result) + assert "parallel drag test complete" in result, result + machine.succeed("touch /tmp/stop-drag-recorder") + machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-drag.pid) 2>/dev/null", timeout=60) + + with subtest("Both cursors delivered cooked, window-targeted events"): + machine.log(machine.succeed("cat /tmp/xi2paint-events.log")) + machine.succeed("test \"$(grep -c '^PRESS ' /tmp/xi2paint-events.log)\" -eq 4") + distinct = machine.succeed( + "grep '^PRESS ' /tmp/xi2paint-events.log | sed -n 's/.*dev=\\([0-9]*\\).*/\\1/p' | sort -u | wc -l" + ).strip() + assert int(distinct) == 2, f"expected 2 distinct devices, got {distinct}" + machine.succeed("test \"$(grep -c '^MOTION ' /tmp/xi2paint-events.log)\" -ge 4") + + with subtest("The drags did not steal focus from the control window"): + active = machine.succeed("DISPLAY=:0 xdotool getactivewindow").strip() + control = machine.succeed("head -1 /tmp/control-xid.txt").strip() + assert active == control, f"focus stolen: active={active} control={control}" + + with subtest("GIF artifact exists"): + machine.succeed("test -s /tmp/cua-driver-linux-parallel-drag-xserver.gif") + + with subtest("Copy GIF out of the VM"): + machine.copy_from_machine("/tmp/cua-driver-linux-parallel-drag-xserver.gif", "") + ''; +}