diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 526d1cef56..65a714f577 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -275,7 +275,7 @@ dependencies = [ [[package]] name = "cua-driver" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "async-trait", @@ -305,7 +305,7 @@ dependencies = [ [[package]] name = "cua-driver-core" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "async-trait", @@ -320,7 +320,7 @@ dependencies = [ [[package]] name = "cua-driver-uia" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "cua-driver-core", @@ -335,7 +335,7 @@ dependencies = [ [[package]] name = "cursor-overlay" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "image", @@ -488,7 +488,7 @@ checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" [[package]] name = "focus-monitor-win" -version = "0.4.0" +version = "0.4.3" dependencies = [ "windows 0.58.0", ] @@ -1192,7 +1192,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pip-preview" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "serde_json", @@ -1207,7 +1207,7 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "platform-linux" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "async-trait", @@ -1228,7 +1228,7 @@ dependencies = [ [[package]] name = "platform-macos" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "async-trait", @@ -1261,7 +1261,7 @@ dependencies = [ [[package]] name = "platform-windows" -version = "0.4.0" +version = "0.4.3" dependencies = [ "anyhow", "async-trait", @@ -1269,6 +1269,7 @@ dependencies = [ "cua-driver-core", "cursor-overlay", "image", + "indexmap", "pip-preview", "serde", "serde_json", diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs index ff7fe23abc..ae2f4cf3e9 100644 --- a/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs +++ b/libs/cua-driver/rust/crates/cursor-overlay/src/lib.rs @@ -232,6 +232,16 @@ impl CursorRegistry { pub fn all_states(&self) -> Vec { self.inner.lock().unwrap().values().cloned().collect() } + + /// Drop a session's cursor metadata entry (fired from the `session_end` + /// hook). The `"default"` key backs the anonymous / one-shot path and is + /// guarded against removal; an empty or absent key is a harmless no-op. + pub fn remove(&self, cursor_id: &str) { + if cursor_id.is_empty() || cursor_id == "default" { + return; + } + self.inner.lock().unwrap().remove(cursor_id); + } } impl Default for CursorRegistry { diff --git a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml b/libs/cua-driver/rust/crates/platform-windows/Cargo.toml index 04f4fa34f3..c40c8eccef 100644 --- a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml +++ b/libs/cua-driver/rust/crates/platform-windows/Cargo.toml @@ -14,6 +14,10 @@ async-trait = "0.1" cua-driver-core = { path = "../cua-driver-core" } cursor-overlay = { path = "../cursor-overlay" } pip-preview = { path = "../pip-preview" } +# IndexMap gives the keyed cursor render collection deterministic +# insertion-ordered iteration = stable per-session cursor z-order frame to +# frame (mirrors platform-macos's per-session overlay). +indexmap = "2" # tiny-skia for cross-platform cursor rendering (used in overlay.rs on all targets) tiny-skia = { version = "0.11", default-features = false, features = ["std"] } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs index 4650c5107b..6cff851e57 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs @@ -9,103 +9,333 @@ //! - Z-ordering: every 80ms call `SetWindowPos` to stay just above the pinned target. //! - Idle-hide: fade out over 180ms once `idle_hide_ms` has elapsed with no activity. //! +//! ## Per-session cursors (2026-06 port from platform-macos #1779) +//! +//! Before this, the overlay was a process-wide singleton (one `RenderState`), +//! so concurrent MCP sessions clobbered each other last-writer-wins → one +//! shared cursor. It now keeps a keyed [`RenderMap`] (`IndexMap`): each declared `session` owns its own cursor with its own +//! palette, and the ~125 Hz tick composites them all into the single layered +//! window. `IndexMap` gives deterministic insertion-ordered iteration = stable +//! per-session z-order frame to frame. The lifecycle (lazy create, per-key +//! arrival isolation, `session_end` removal, resurrection tombstone) mirrors +//! `platform_macos::cursor::overlay` so the two platforms behave identically; +//! the shared `cursor_overlay::{CursorKey, KeyedOverlayCommand, OverlayMsg}` +//! types are the same ones macOS uses. +//! //! ## Cross-platform note (2026-05 dedup audit) //! //! Animation state + render pipeline live in `cursor_overlay::render_state` -//! (`RenderStateCore`, `tick_motion`, `apply_command_base`, `render_frame`). +//! (`RenderStateCore`, `tick_motion`, `apply_command_base`, `paint_cursor`). //! What stays here is purely the Win32 window plumbing: message loop, //! UpdateLayeredWindow paint, virtual-screen offset, z-order maintenance. #![allow(non_snake_case, non_upper_case_globals)] +use std::collections::{HashMap, HashSet}; use std::sync::{Mutex, OnceLock}; use std::time::Instant; use cursor_overlay::{ - CursorConfig, MotionConfig, OverlayCommand, RenderStateCore, ZOrderEnforcer, + CursorConfig, CursorKey, KeyedOverlayCommand, MotionConfig, OverlayCommand, OverlayMsg, + Palette, RenderStateCore, ZOrderEnforcer, }; +use indexmap::IndexMap; // ── Global channel ──────────────────────────────────────────────────────── -static CMD_TX: OnceLock> = OnceLock::new(); -static CMD_RX_CELL: Mutex>> = Mutex::new(None); -static RENDER: Mutex> = Mutex::new(None); +static CMD_TX: OnceLock> = OnceLock::new(); +static CMD_RX_CELL: Mutex>> = Mutex::new(None); +static RENDER: Mutex> = Mutex::new(None); -// ── Arrival-signal channel ──────────────────────────────────────────────── +// ── Arrival-signal channels (one waiter slot per cursor key) ────────────── // -// `animate_cursor_to` installs a oneshot sender here, the render thread's -// `WM_TIMER` handler fires it the tick the planned path ends. Mirrors -// macOS so click handlers can `.await` until the cursor visually lands -// before dispatching the actual UIA / PostMessage action. -static ARRIVAL_TX: Mutex>> = Mutex::new(None); +// Each session's `animate_cursor_to` registers an arrival oneshot keyed by its +// own cursor key. A new animation only supersedes the SAME key's prior waiter, +// so concurrent sessions never cross-cancel each other's arrivals. Mirrors +// macOS so click handlers can `.await` until the cursor visually lands before +// dispatching the actual UIA / PostMessage action. +static ARRIVAL_TX: Mutex>>> = + Mutex::new(None); + +fn arrival_register(key: CursorKey, tx: tokio::sync::oneshot::Sender<()>) { + let mut guard = ARRIVAL_TX.lock().unwrap(); + let map = guard.get_or_insert_with(HashMap::new); + // Cancel only the same key's previous waiter (superseded by new animation). + if let Some(old_tx) = map.insert(key, tx) { + let _ = old_tx.send(()); + } +} + +fn arrival_fire(key: &CursorKey) { + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(map) = guard.as_mut() { + if let Some(tx) = map.remove(key) { + let _ = tx.send(()); + } + } + } +} + +// ── Keyed render collection ─────────────────────────────────────────────── + +/// The keyed, insertion-ordered collection of owned cursors that the render +/// loop composites every tick. Insertion order = stable z-order (later keys +/// paint on top). Virtual-screen geometry + the `WM_TIMER` dt stamp are +/// hoisted here (screen-global, written once in `run_overlay_thread`). +struct RenderMap { + cursors: IndexMap, + /// Virtual screen dimensions set after window creation (Win32 DIPs). + /// `virt_x/y` are subtracted from each cursor's `core.pos` when rendering + /// so the pixmap is laid out in window-local coordinates. + virt_x: i32, + virt_y: i32, + virt_w: i32, + virt_h: i32, + /// Last WM_TIMER wall-clock stamp; used to compute real `dt` (Windows + /// timer resolution defaults to 15ms so a hardcoded 8ms would run the + /// animation at half speed). + last_tick: Instant, + /// Frozen launch-time config used as the template for lazily-created + /// cursors (its palette is overridden per-key via `Palette::for_instance`). + template: CursorConfig, + /// Render-side tombstone of permanently-ended session cursor keys. A `Cmd` + /// for a key in here is dropped WITHOUT get-or-create, so an in-flight + /// click/move from another task that lands AFTER the owning session's + /// `Remove` can never resurrect the just-removed cursor. "default" is never + /// tombstoned (it backs the anonymous / one-shot path). + ended: HashSet, + /// Cursor key whose target the overlay should currently sit above. A single + /// layered window can occupy only one z-band, so the most-recently-touched + /// cursor wins (mirrors macOS). `None` until the first PinAbove/Cmd. + last_active: Option, +} + +/// Build the `RenderState` for a lazily-created cursor key: derive from the +/// launch template but give each non-default key its own palette so distinct +/// sessions get distinct colours automatically. +fn render_state_for_key(template: &CursorConfig, key: &str) -> RenderState { + let mut rs = RenderState::new(template.clone()); + rs.core.palette = Palette::for_instance(key); + rs +} + +/// Apply one inbound [`OverlayMsg`] to the render map (drain step). Factored +/// out as a pure function so the per-session ownership + removal lifecycle is +/// unit-testable without any Win32 window. +/// +/// Returns the resolved cursor key for a `Cmd` (so the caller can track the +/// last-active key for z-order pinning); `None` for a `Remove`. +fn apply_msg(map: &mut RenderMap, msg: OverlayMsg) -> Option { + match msg { + OverlayMsg::Remove(key) => { + // The "default" cursor backs the anonymous / one-shot path and must + // survive every session_end + the daemon lifetime. + if key != "default" { + map.cursors.shift_remove(&key); + if let Ok(mut guard) = ARRIVAL_TX.lock() { + if let Some(m) = guard.as_mut() { + m.remove(&key); + } + } + if map.last_active.as_deref() == Some(key.as_str()) { + map.last_active = None; + } + // Tombstone the key so a late in-flight Cmd from another task + // cannot re-create the just-removed cursor. + map.ended.insert(key); + } + None + } + OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }) => { + // Drop a command for an already-ended session WITHOUT get-or-create + // — this is the resurrection guard. Without it, a ClickPulse/MoveTo + // landing after Remove would re-insert (and re-leak) the cursor. + if map.ended.contains(&key) { + return None; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key) + .or_insert_with(|| render_state_for_key(&template, &k)); + rs.apply_command(cmd); + Some(k) + } + } +} pub fn init(cfg: CursorConfig) { let (tx, rx) = std::sync::mpsc::sync_channel(4096); let _ = CMD_TX.set(tx); *CMD_RX_CELL.lock().unwrap() = Some(rx); - *RENDER.lock().unwrap() = Some(RenderState::new(cfg)); + *ARRIVAL_TX.lock().unwrap() = Some(HashMap::new()); + let mut cursors = IndexMap::new(); + cursors.insert("default".to_owned(), RenderState::new(cfg.clone())); + *RENDER.lock().unwrap() = Some(RenderMap { + cursors, + virt_x: 0, + virt_y: 0, + virt_w: 1920, + virt_h: 1080, + last_tick: Instant::now(), + template: cfg, + ended: HashSet::new(), + last_active: None, + }); } -pub fn send_command(cmd: OverlayCommand) { +/// Send a keyed command from any thread (MCP tool, etc.). Non-blocking; drops +/// if the channel is full (old commands are less important than new ones). +/// +/// Empty key = anonymous (no session declared) → no cursor; the command is +/// dropped so a cursor-less run never paints. See `tools::resolve_cursor_key`. +pub fn send_command(key: CursorKey, cmd: OverlayCommand) { + if key.is_empty() { + return; + } if let Some(tx) = CMD_TX.get() { - let _ = tx.try_send(cmd); + let _ = tx.try_send(OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd })); } } -/// Returns the current glide duration in milliseconds (default 750). -/// Used by the click path to wait for the animation before firing ClickPulse. -pub fn glide_duration_ms() -> f64 { - RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.motion.glide_duration_ms)) - .unwrap_or(750.0) +/// Convenience for callsites not yet threaded with a session key: drives the +/// seeded `"default"` cursor (the anonymous / one-shot identity). +pub fn send_command_default(cmd: OverlayCommand) { + send_command("default".to_owned(), cmd); } -/// Returns true if the cursor overlay is currently enabled/visible. -pub fn is_enabled() -> bool { - RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.visible)) +/// Remove a session's owned cursor from the render collection (fired from the +/// `session_end` hook). The `"default"` key is guarded against removal on the +/// render side, so this is a no-op for it; removing an absent key (anonymous +/// session that never created a cursor) is a harmless no-op. +pub fn remove_cursor(key: CursorKey) { + if key.is_empty() { + return; + } + if let Some(tx) = CMD_TX.get() { + let _ = tx.try_send(OverlayMsg::Remove(key)); + } +} + +/// Returns true if the cursor for `key` is currently enabled/visible. A session +/// with no own cursor yet falls back to the seeded `"default"` cursor. +pub fn is_enabled(key: &str) -> bool { + RENDER + .lock() + .ok() + .and_then(|g| { + g.as_ref().and_then(|m| { + m.cursors + .get(key) + .or_else(|| m.cursors.get("default")) + .map(|rs| rs.core.visible) + }) + }) .unwrap_or(false) } -/// Snapshot the current motion config (start_handle / end_handle / arc_size / -/// arc_flow / spring / glide_duration_ms / dwell_after_click_ms / -/// idle_hide_ms). Mirrors macOS `current_motion()` so -/// `get_agent_cursor_state` can report the live values. -pub fn current_motion() -> MotionConfig { - RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.motion.clone())) +/// Snapshot the current motion config for `key`, falling back to the +/// `"default"` cursor's motion when that key has no own entry yet. +pub fn current_motion(key: &str) -> MotionConfig { + RENDER + .lock() + .ok() + .and_then(|g| { + g.as_ref().and_then(|m| { + m.cursors + .get(key) + .or_else(|| m.cursors.get("default")) + .map(|rs| rs.core.motion.clone()) + }) + }) .unwrap_or_default() } -/// Returns the current cursor position in screen coordinates. -pub fn current_position() -> (f64, f64) { - RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.pos)) +/// Current screen position of the cursor for `key` (the off-screen sentinel +/// `(-200, -200)` if it has never been placed). A session with no own cursor +/// yet reports the sentinel so the click path treats it as first-placement. +pub fn current_position(key: &str) -> (f64, f64) { + RENDER + .lock() + .ok() + .and_then(|g| g.as_ref().and_then(|m| m.cursors.get(key)).map(|rs| rs.core.pos)) .unwrap_or((-200.0, -200.0)) } -/// Returns true if the cursor is still at the off-screen initial position -/// (-200, -200), meaning it has never been positioned on screen yet. -pub fn is_at_initial_position() -> bool { - RENDER.lock().ok() - .and_then(|g| g.as_ref().map(|rs| rs.core.pos.0 < 0.0 && rs.core.pos.1 < 0.0)) - .unwrap_or(true) +/// Seed a brand-new (sentinel-positioned) cursor at an on-screen start point +/// offset up-left of `(target_x, target_y)` so the immediately-following +/// `MoveTo` glides INTO the target instead of silently snapping. No-op when the +/// cursor is already on-screen or its session already ended. Returns true if a +/// seed was applied. Mirrors `platform_macos::cursor::overlay::seed_start_*`. +fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool { + let mut guard = RENDER.lock().unwrap(); + let Some(map) = guard.as_mut() else { return false }; + seed_start_in_map(map, key, target_x, target_y) } -/// Animate the overlay cursor to `(x, y)` and suspend until the planned -/// path completes (the spring-settle phase that follows is allowed to keep -/// running — we only wait for the visible glide to land). +/// Pure seed step operating on a borrowed [`RenderMap`] — factored out so the +/// get-or-create + clamp logic is unit-testable without the global `RENDER` +/// static or a Win32 window. +fn seed_start_in_map(map: &mut RenderMap, key: &CursorKey, target_x: f64, target_y: f64) -> bool { + const SEED_OFFSET: f64 = 140.0; + let (virt_x, virt_y) = (map.virt_x as f64, map.virt_y as f64); + let (virt_w, virt_h) = (map.virt_w as f64, map.virt_h as f64); + // Respect the resurrection guard: never seed (and thus re-create) a cursor + // whose session already ended. + if map.ended.contains(key) { + return false; + } + let template = map.template.clone(); + let k = key.clone(); + let rs = map + .cursors + .entry(key.clone()) + .or_insert_with(|| render_state_for_key(&template, &k)); + if !(rs.core.cfg.enabled && rs.core.pos.0 < -50.0) { + return false; + } + let mut sx = target_x - SEED_OFFSET; + let mut sy = target_y - SEED_OFFSET; + // Clamp into the virtual-screen frame so the seed never starts off-display. + if virt_w > 0.0 && virt_h > 0.0 { + sx = sx.clamp(virt_x + 2.0, virt_x + virt_w - 2.0); + sy = sy.clamp(virt_y + 2.0, virt_y + virt_h - 2.0); + // If clamping collapsed the seed onto the target (target in a corner), + // nudge the other way so there is still a visible glide distance. + if (sx - target_x).abs() < 8.0 && (sy - target_y).abs() < 8.0 { + sx = (target_x + SEED_OFFSET).min(virt_x + virt_w - 2.0); + sy = (target_y + SEED_OFFSET).min(virt_y + virt_h - 2.0); + } + } + rs.core.pos = (sx, sy); + true +} + +/// Animate the overlay cursor for `key` to `(x, y)` and suspend until the +/// planned path completes (the spring-settle phase that follows is allowed to +/// keep running — we only wait for the visible glide to land). +/// +/// Returns immediately (no animation, no wait) when: +/// - the key is empty (anonymous run → no cursor), or +/// - the cursor for `key` is disabled. /// -/// Mirrors `platform_macos::cursor::overlay::animate_cursor_to`. Returns -/// immediately (no animation, no wait) when: -/// - the overlay is disabled, or -/// - the cursor is still at the off-screen sentinel `(-200, -200)` — in -/// that case the caller should rely on `ClickPulse` to snap the cursor. -pub async fn animate_cursor_to(x: f64, y: f64) { +/// A brand-new cursor still at the off-screen sentinel is first seeded +/// on-screen via [`seed_start_if_sentinel`] so its FIRST action glides in. +/// Mirrors `platform_macos::cursor::overlay::animate_cursor_to`. +pub async fn animate_cursor_to(key: CursorKey, x: f64, y: f64) { + if key.is_empty() { + return; + } + // Seed a sentinel cursor on-screen so the MoveTo below glides instead of + // being short-circuited. + seed_start_if_sentinel(&key, x, y); + let should_animate = { let guard = RENDER.lock().unwrap(); - match guard.as_ref() { - Some(rs) if rs.core.cfg.enabled && rs.core.visible && rs.core.pos.0 > -50.0 => true, + match guard.as_ref().and_then(|m| m.cursors.get(&key)) { + Some(rs) if rs.core.cfg.enabled && rs.core.pos.0 > -50.0 => true, _ => false, } }; @@ -113,27 +343,22 @@ pub async fn animate_cursor_to(x: f64, y: f64) { return; } - // Install the oneshot sender BEFORE issuing MoveTo, so the render + // Install the keyed oneshot sender BEFORE issuing MoveTo, so the render // thread's arrival-fire can never lose a race against an immediate // path-end (e.g. zero-length glide). let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - { - let mut guard = ARRIVAL_TX.lock().unwrap(); - // A previous in-flight animation gets superseded by this one — - // unblock its waiter so it doesn't hang forever. - if let Some(old_tx) = guard.take() { - let _ = old_tx.send(()); - } - *guard = Some(tx); - } - - send_command(OverlayCommand::MoveTo { - x, - y, - // Arrive pointing upper-left (45°) — same convention as macOS / - // Swift reference (`endAngleDegrees: 45`). - end_heading_radians: std::f64::consts::FRAC_PI_4, - }); + arrival_register(key.clone(), tx); + + send_command( + key, + OverlayCommand::MoveTo { + x, + y, + // Arrive pointing upper-left (45°) — same convention as macOS / + // Swift reference (`endAngleDegrees: 45`). + end_heading_radians: std::f64::consts::FRAC_PI_4, + }, + ); let _ = rx.await; } @@ -149,7 +374,7 @@ pub fn run_on_thread() { let cfg = { let guard = RENDER.lock().unwrap(); match &*guard { - Some(rs) => rs.core.cfg.clone(), + Some(m) => m.template.clone(), None => return, } }; @@ -167,40 +392,26 @@ pub fn run_on_thread() { .expect("spawn overlay thread"); } -// ── Animation state ─────────────────────────────────────────────────────── +// ── Animation / render state ────────────────────────────────────────────── // // The platform-agnostic fields + tick + apply_command + render pipeline live -// in `cursor_overlay::render_state` (2026-05 dedup audit). What stays here -// is the Windows-specific virtual-screen geometry + last_tick stamp for the -// WM_TIMER dt calculation. +// in `cursor_overlay::render_state`. What stays here is just the per-cursor +// wrapper; the virtual-screen geometry + dt stamp moved up to `RenderMap`. struct RenderState { core: RenderStateCore, - /// Virtual screen dimensions set after window creation (Win32 DIPs). - /// `virt_x/y` are subtracted from `core.pos` when rendering so the - /// pixmap is laid out in window-local coordinates. - virt_x: i32, - virt_y: i32, - virt_w: i32, - virt_h: i32, - /// Last WM_TIMER wall-clock stamp; used to compute real `dt` (Windows - /// timer resolution defaults to 15ms so a hardcoded 8ms would run the - /// animation at half speed). - last_tick: Instant, } impl RenderState { fn new(cfg: CursorConfig) -> Self { RenderState { core: RenderStateCore::new(cfg), - last_tick: Instant::now(), - virt_x: 0, virt_y: 0, virt_w: 1920, virt_h: 1080, } } - /// Advance the motion state by `dt`. Returns `true` the tick the - /// planned path completes, so the WM_TIMER handler can fire the - /// arrival oneshot that unblocks `animate_cursor_to`. + /// Advance the motion state by `dt`. Returns `true` the tick the planned + /// path completes, so the WM_TIMER handler can fire the arrival oneshot + /// that unblocks `animate_cursor_to`. fn tick(&mut self, dt: f64) -> bool { self.core.tick_motion(dt) } @@ -217,17 +428,17 @@ impl RenderState { // ── Win32 message-loop thread ───────────────────────────────────────────── #[cfg(target_os = "windows")] -fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { - use windows::Win32::UI::WindowsAndMessaging::*; +fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { use windows::Win32::Media::timeBeginPeriod; use windows::Win32::System::LibraryLoader::GetModuleHandleW; + use windows::Win32::UI::WindowsAndMessaging::*; use windows::core::PCWSTR; // Raise multimedia timer resolution to 1ms so SetTimer can deliver // WM_TIMER messages at ~8ms intervals (default is ~15ms). - // Mirrors `_timerResolutionRaised = timeBeginPeriod(1) == 0` in the - // .NET reference (AgentCursorOverlay.cs). - unsafe { let _ = timeBeginPeriod(1); } + unsafe { + let _ = timeBeginPeriod(1); + } // Collect virtual screen bounds (all monitors). let virt_x = unsafe { GetSystemMetrics(SM_XVIRTUALSCREEN) }; @@ -235,26 +446,23 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver = "Cua.AgentCursorOverlay\0".encode_utf16().collect(); let title_w: Vec = format!("Cua.AgentCursorOverlay.{}\0", cfg.cursor_id) - .encode_utf16().collect(); + .encode_utf16() + .collect(); let hinstance = unsafe { GetModuleHandleW(PCWSTR::null()).unwrap_or_default() }; @@ -266,11 +474,13 @@ fn run_overlay_thread(cfg: CursorConfig, rx: std::sync::mpsc::Receiver) { +fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) { // No-op on non-Windows targets (cross-compile guard). } // ── Win32 globals (only used on Windows) ───────────────────────────────── -static OVERLAY_HWND: std::sync::atomic::AtomicIsize = - std::sync::atomic::AtomicIsize::new(0); -static CMD_RX_WIN: Mutex>> = Mutex::new(None); -static LAST_ZTICK: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); +static OVERLAY_HWND: std::sync::atomic::AtomicIsize = std::sync::atomic::AtomicIsize::new(0); +static CMD_RX_WIN: Mutex>> = Mutex::new(None); +static LAST_ZTICK: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); static Z_ORDER: OnceLock = OnceLock::new(); // ── Window procedure ────────────────────────────────────────────────────── @@ -346,34 +564,67 @@ unsafe extern "system" fn wnd_proc( .unwrap_or_default() .as_millis() as u64; - // Drain commands and tick animation. Measure real dt from last - // tick — Windows timer resolution defaults to 15ms so the - // hardcoded 8ms was running the animation at half speed. - let (pixmap, fire_arrival) = { + // ── Drain commands, tick all cursors, composite one pixmap ─────── + // Measure real dt from last tick — Windows timer resolution defaults + // to 15ms so the hardcoded 8ms ran the animation at half speed. + let (pixmap, arrived, pinned_wid) = { let mut guard = RENDER.lock().unwrap(); - if let Some(rs) = guard.as_mut() { - // Drain the channel. + if let Some(map) = guard.as_mut() { + // Drain the channel via get-or-create; track the last-touched + // key so the z-order pin follows the most-recent cursor. if let Ok(rx_guard) = CMD_RX_WIN.try_lock() { if let Some(ref rx) = *rx_guard { - while let Ok(cmd) = rx.try_recv() { - rs.apply_command(cmd); + while let Ok(m) = rx.try_recv() { + if let Some(k) = apply_msg(map, m) { + map.last_active = Some(k); + } } } } - let now = std::time::Instant::now(); - let dt = now.duration_since(rs.last_tick).as_secs_f64().clamp(0.0, 0.05); - rs.last_tick = now; - let arrived = rs.tick(dt); - (Some(cursor_overlay::render_frame( - &rs.core, - rs.virt_w.max(1) as u32, - rs.virt_h.max(1) as u32, - rs.virt_x as f64, - rs.virt_y as f64, - None, // focus-rect is macOS-only - )), arrived) + + let now = Instant::now(); + let dt = now + .duration_since(map.last_tick) + .as_secs_f64() + .clamp(0.0, 0.05); + map.last_tick = now; + + // Tick every cursor; record the ones that just arrived. + let mut arrived: Vec = Vec::new(); + for (k, rs) in map.cursors.iter_mut() { + if rs.tick(dt) { + arrived.push(k.clone()); + } + } + + // Composite every cursor into ONE virtual-screen pixmap. + // tiny-skia fills are alpha-over, so insertion order = + // paint/z-order; idle/hidden cursors early-return inside + // paint_cursor so an idle session costs ~nothing. + let w = map.virt_w.max(1) as u32; + let h = map.virt_h.max(1) as u32; + let mut pm = tiny_skia::Pixmap::new(w.max(1), h.max(1)) + .unwrap_or_else(|| tiny_skia::Pixmap::new(1, 1).unwrap()); + for (_k, rs) in &map.cursors { + cursor_overlay::paint_cursor( + &mut pm, + &rs.core, + map.virt_x as f64, + map.virt_y as f64, + None, // focus-rect is macOS-only + ); + } + + // Pin above the most-recently-touched cursor's target. + let pinned = map + .last_active + .as_ref() + .and_then(|k| map.cursors.get(k)) + .and_then(|rs| rs.core.pinned_wid); + + (Some(pm), arrived, pinned) } else { - (None, false) + (None, Vec::new(), None) } }; @@ -381,15 +632,11 @@ unsafe extern "system" fn wnd_proc( update_layered_window(hwnd, &pm); } - // Fire arrival oneshot the tick the path just ended — unblocks - // any `animate_cursor_to(...).await` so the click action only - // dispatches once the cursor has visually landed. - if fire_arrival { - if let Ok(mut guard) = ARRIVAL_TX.lock() { - if let Some(tx) = guard.take() { - let _ = tx.send(()); - } - } + // Fire arrival oneshots for cursors whose path just ended — unblocks + // each session's `animate_cursor_to(...).await` so the click action + // only dispatches once that cursor has visually landed. + for k in &arrived { + arrival_fire(k); } // Z-order maintenance every 80ms — delegate to the cross-platform @@ -398,8 +645,6 @@ unsafe extern "system" fn wnd_proc( let last = LAST_ZTICK.load(std::sync::atomic::Ordering::Relaxed); if now_ms.wrapping_sub(last) >= 80 { LAST_ZTICK.store(now_ms, std::sync::atomic::Ordering::Relaxed); - let pinned_wid = RENDER.lock().ok() - .and_then(|g| g.as_ref().and_then(|rs| rs.core.pinned_wid)); if let Some(enforcer) = Z_ORDER.get() { enforcer.reassert(pinned_wid); } @@ -428,7 +673,9 @@ unsafe fn update_layered_window( let w = pixmap.width() as i32; let h = pixmap.height() as i32; - if w <= 0 || h <= 0 { return; } + if w <= 0 || h <= 0 { + return; + } let hdc_screen = GetDC(None); let hdc_mem = CreateCompatibleDC(hdc_screen); @@ -448,14 +695,7 @@ unsafe fn update_layered_window( }; let mut bits_ptr = std::ptr::null_mut::(); - let hbmp = CreateDIBSection( - hdc_mem, - &bmi, - DIB_RGB_COLORS, - &mut bits_ptr, - None, - 0, - ); + let hbmp = CreateDIBSection(hdc_mem, &bmi, DIB_RGB_COLORS, &mut bits_ptr, None, 0); if hbmp.is_err() || bits_ptr.is_null() { let _ = DeleteDC(hdc_mem); ReleaseDC(None, hdc_screen); @@ -473,7 +713,7 @@ unsafe fn update_layered_window( let b = src[i * 4 + 2]; let a = src[i * 4 + 3]; // Swap R <-> B for BGRA. - dst[i * 4] = b; + dst[i * 4] = b; dst[i * 4 + 1] = g; dst[i * 4 + 2] = r; dst[i * 4 + 3] = a; @@ -484,9 +724,9 @@ unsafe fn update_layered_window( let virt_y; { let guard = RENDER.lock().unwrap(); - if let Some(rs) = &*guard { - virt_x = rs.virt_x; - virt_y = rs.virt_y; + if let Some(map) = &*guard { + virt_x = map.virt_x; + virt_y = map.virt_y; } else { virt_x = 0; virt_y = 0; @@ -495,15 +735,24 @@ unsafe fn update_layered_window( let pt_src = POINT { x: 0, y: 0 }; let pt_dst = POINT { x: virt_x, y: virt_y }; - let sz = SIZE { cx: w, cy: h }; - let blend = BLENDFUNCTION { - BlendOp: 0, // AC_SRC_OVER - BlendFlags: 0, + let sz = SIZE { cx: w, cy: h }; + let blend = BLENDFUNCTION { + BlendOp: 0, // AC_SRC_OVER + BlendFlags: 0, SourceConstantAlpha: 255, - AlphaFormat: 1, // AC_SRC_ALPHA + AlphaFormat: 1, // AC_SRC_ALPHA }; - let _ = UpdateLayeredWindow(hwnd, hdc_screen, Some(&pt_dst), Some(&sz), - hdc_mem, Some(&pt_src), COLORREF(0), Some(&blend), ULW_ALPHA); + let _ = UpdateLayeredWindow( + hwnd, + hdc_screen, + Some(&pt_dst), + Some(&sz), + hdc_mem, + Some(&pt_src), + COLORREF(0), + Some(&blend), + ULW_ALPHA, + ); let _ = DeleteObject(hbmp); let _ = DeleteDC(hdc_mem); @@ -566,7 +815,10 @@ impl ZOrderEnforcer for WinZOrderEnforcer { let _ = SetWindowPos( hwnd, HWND_NOTOPMOST, - 0, 0, 0, 0, + 0, + 0, + 0, + 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_NOOWNERZORDER, ); @@ -580,7 +832,10 @@ impl ZOrderEnforcer for WinZOrderEnforcer { let _ = SetWindowPos( hwnd, insert_after, - 0, 0, 0, 0, + 0, + 0, + 0, + 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOOWNERZORDER, ); @@ -588,3 +843,172 @@ impl ZOrderEnforcer for WinZOrderEnforcer { } } } + +// ── Headless unit tests for the keyed render collection ─────────────────── +// +// These prove the per-session ownership data model, the session_end removal +// lifecycle, the "default" guard, the resurrection tombstone, and the +// sentinel seed WITHOUT any Win32 window. The on-screen rendering +// (UpdateLayeredWindow) still needs a real display and is verified separately. + +#[cfg(test)] +mod tests { + use super::*; + + fn empty_map() -> RenderMap { + let mut cursors = IndexMap::new(); + cursors.insert("default".to_owned(), RenderState::new(CursorConfig::default())); + RenderMap { + cursors, + virt_x: 0, + virt_y: 0, + virt_w: 100, + virt_h: 100, + last_tick: Instant::now(), + template: CursorConfig::default(), + ended: HashSet::new(), + last_active: None, + } + } + + fn move_msg(key: &str, x: f64, y: f64) -> OverlayMsg { + OverlayMsg::Cmd(KeyedOverlayCommand { + key: key.to_owned(), + cmd: OverlayCommand::MoveTo { x, y, end_heading_radians: 0.0 }, + }) + } + + #[test] + fn two_sessions_produce_two_distinct_render_entries() { + let mut map = empty_map(); + apply_msg(&mut map, move_msg("sessA", 10.0, 10.0)); + apply_msg(&mut map, move_msg("sessB", 42.0, 24.0)); + // default + sessA + sessB = 3 distinct owned cursors. The pre-port + // regression: a single RenderState would clobber these to one cursor. + assert_eq!(map.cursors.len(), 3); + assert!(map.cursors.contains_key("sessA")); + assert!(map.cursors.contains_key("sessB")); + assert!(map.cursors.contains_key("default")); + } + + #[test] + fn session_end_removes_only_that_session() { + let mut map = empty_map(); + apply_msg(&mut map, move_msg("sessA", 10.0, 10.0)); + apply_msg(&mut map, move_msg("sessB", 20.0, 20.0)); + assert_eq!(map.cursors.len(), 3); + + // session_end(A): A gone, B + default retained. + apply_msg(&mut map, OverlayMsg::Remove("sessA".to_owned())); + assert!(!map.cursors.contains_key("sessA")); + assert!(map.cursors.contains_key("sessB")); + assert!(map.cursors.contains_key("default")); + assert_eq!(map.cursors.len(), 2); + + // Remove("default") is guarded — default survives. + apply_msg(&mut map, OverlayMsg::Remove("default".to_owned())); + assert!(map.cursors.contains_key("default")); + + // Remove of an absent key is a harmless no-op. + let before = map.cursors.len(); + apply_msg(&mut map, OverlayMsg::Remove("never-existed".to_owned())); + assert_eq!(map.cursors.len(), before); + } + + #[test] + fn lazily_created_cursors_get_distinct_palettes() { + let mut map = empty_map(); + apply_msg(&mut map, move_msg("sessA", 10.0, 10.0)); + apply_msg(&mut map, move_msg("sessB", 20.0, 20.0)); + let a = &map.cursors["sessA"].core.palette; + let b = &map.cursors["sessB"].core.palette; + let def = &map.cursors["default"].core.palette; + assert_ne!(a.name, def.name); + assert_ne!(b.name, def.name); + } + + #[test] + fn insertion_order_is_stable_z_order() { + let mut map = empty_map(); + apply_msg(&mut map, move_msg("first", 1.0, 1.0)); + apply_msg(&mut map, move_msg("second", 2.0, 2.0)); + // Re-touching "first" must NOT move it to the back (IndexMap keeps the + // original slot), so z-order is stable frame to frame. + apply_msg(&mut map, move_msg("first", 3.0, 3.0)); + let keys: Vec<&String> = map.cursors.keys().collect(); + assert_eq!(keys, vec!["default", "first", "second"]); + } + + #[test] + fn tombstone_blocks_resurrection_after_remove() { + let mut map = empty_map(); + apply_msg(&mut map, move_msg("sessA", 10.0, 10.0)); + assert_eq!(map.cursors.len(), 2); // default + sessA + + apply_msg(&mut map, OverlayMsg::Remove("sessA".to_owned())); + assert!(!map.cursors.contains_key("sessA")); + assert_eq!(map.cursors.len(), 1); + + // A late in-flight Cmd for the ended session must be dropped WITHOUT + // re-inserting (no get-or-create resurrection). + let resolved = apply_msg(&mut map, move_msg("sessA", 99.0, 99.0)); + assert!(resolved.is_none(), "ended-session Cmd must be dropped, not resolved"); + assert!(!map.cursors.contains_key("sessA"), "tombstone must block resurrection"); + assert_eq!(map.cursors.len(), 1); + } + + #[test] + fn default_is_never_tombstoned() { + let mut map = empty_map(); + apply_msg(&mut map, OverlayMsg::Remove("default".to_owned())); + assert!(map.cursors.contains_key("default")); + assert!(!map.ended.contains("default")); + + let resolved = apply_msg(&mut map, move_msg("default", 5.0, 5.0)); + assert_eq!(resolved.as_deref(), Some("default")); + assert!(map.cursors.contains_key("default")); + } + + #[test] + fn seed_moves_sentinel_cursor_on_screen_for_first_action() { + let mut map = empty_map(); // 100x100 frame at origin + // No "sessA" cursor exists yet — the seed must get-or-create it. + let seeded = seed_start_in_map(&mut map, &"sessA".to_owned(), 60.0, 60.0); + assert!(seeded, "sentinel cursor must be seeded"); + let pos = map.cursors["sessA"].core.pos; + assert!(pos.0 > -50.0 && pos.1 > -50.0, "seed must be on-screen, got {pos:?}"); + assert!( + (pos.0 - 60.0).abs() > 4.0 || (pos.1 - 60.0).abs() > 4.0, + "seed must differ from target to produce a visible glide, got {pos:?}" + ); + } + + #[test] + fn seed_is_noop_when_cursor_already_on_screen() { + let mut map = empty_map(); + seed_start_in_map(&mut map, &"sessA".to_owned(), 60.0, 60.0); + map.cursors.get_mut("sessA").unwrap().core.pos = (30.0, 30.0); + let seeded_again = seed_start_in_map(&mut map, &"sessA".to_owned(), 80.0, 80.0); + assert!(!seeded_again, "on-screen cursor must not be re-seeded"); + assert_eq!(map.cursors["sessA"].core.pos, (30.0, 30.0), "pos must be untouched"); + } + + #[test] + fn seed_does_not_resurrect_ended_session() { + let mut map = empty_map(); + map.ended.insert("sessA".to_owned()); + let seeded = seed_start_in_map(&mut map, &"sessA".to_owned(), 60.0, 60.0); + assert!(!seeded, "ended session must not be seeded"); + assert!(!map.cursors.contains_key("sessA"), "ended session must not be resurrected"); + } + + #[test] + fn remove_clears_last_active_for_that_key() { + let mut map = empty_map(); + let k = apply_msg(&mut map, move_msg("sessA", 10.0, 10.0)); + map.last_active = k; + assert_eq!(map.last_active.as_deref(), Some("sessA")); + apply_msg(&mut map, OverlayMsg::Remove("sessA".to_owned())); + assert_eq!(map.last_active, None, "removing the active cursor must clear last_active"); + } +} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index e274baffe0..0542127abc 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -11,14 +11,14 @@ use async_trait::async_trait; /// the host appears in the z-order. `GA_ROOT` normalises both inputs to the /// host so the overlay sits at z+1 of whatever is actually painted on screen. /// -/// No-op when the overlay is disabled; the command is just dropped by the -/// render thread in that case. -fn pin_overlay_above(hwnd: u64) { +/// No-op when the overlay is disabled or `key` is empty (anonymous, cursor-less +/// run); the command is just dropped by the render thread in that case. +fn pin_overlay_above(key: &str, hwnd: u64) { use windows::Win32::Foundation::HWND; use windows::Win32::UI::WindowsAndMessaging::{GetAncestor, GA_ROOT}; let root = unsafe { GetAncestor(HWND(hwnd as *mut _), GA_ROOT) }; let wid = if !root.0.is_null() { root.0 as u64 } else { hwnd }; - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(wid)); + crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::PinAbove(wid)); } /// Convert (px, py) — "window-local screenshot pixels, top-left origin @@ -83,15 +83,20 @@ fn bitmap_to_screen(hwnd: u64, px: i32, py: i32) -> (i32, i32) { /// render thread's arrival oneshot — that's how we keep the click action /// from firing before the cursor has visually landed (the old heuristic /// `tokio::sleep(80..600 ms)` was racing the spring-physics glide). -async fn overlay_glide_to(sx: f64, sy: f64) { - if !crate::overlay::is_enabled() { return; } - let pos = crate::overlay::current_position(); +/// +/// `key` is the session's cursor key (see [`resolve_cursor_key`]). An empty key +/// (anonymous, no declared session) is cursor-less: every overlay op +/// short-circuits, so the action runs with no visible cursor. +async fn overlay_glide_to(key: &str, sx: f64, sy: f64) { + if key.is_empty() { return; } + if !crate::overlay::is_enabled(key) { return; } + let pos = crate::overlay::current_position(key); if pos.0 < 0.0 && pos.1 < 0.0 { // Snap to target on first use; no animation to wait for. - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); return; } - crate::overlay::animate_cursor_to(sx, sy).await; + crate::overlay::animate_cursor_to(key.to_owned(), sx, sy).await; } use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}}; use serde_json::{json, Value}; @@ -101,6 +106,32 @@ use crate::uia::ElementCache; use cursor_overlay::CursorRegistry; use windows::core::Interface as _; +/// The cursor key for an anonymous (cursor-less) call. A run opts into a cursor +/// by declaring a `session`; without one, every cursor op short-circuits on +/// this empty key (see `overlay::send_command` / `overlay_glide_to`). +pub(crate) const NO_CURSOR: &str = ""; + +/// Resolve the cursor key for a tool invocation, or [`NO_CURSOR`] (`""`) for an +/// anonymous call. +/// +/// A cursor is tied to a **caller-declared session**, never to the MCP +/// connection. Precedence: an explicit `session` arg, then its legacy alias +/// `cursor_id`. We deliberately do NOT fall back to the connection-injected +/// `_session_id` or to a seeded `"default"` cursor — `""` means "no session +/// declared → no cursor", while the underlying action (click/type/…) still +/// executes. Mirrors `platform_macos::tools::cursor_tools::resolve_cursor_key` +/// so the two platforms key cursors identically. +pub(crate) fn resolve_cursor_key(args: &Value) -> String { + for key in ["session", "cursor_id"] { + if let Some(v) = args.get(key).and_then(|v| v.as_str()) { + if !v.is_empty() { + return v.to_owned(); + } + } + } + NO_CURSOR.to_owned() +} + // ── DriverConfig + ResizeRegistry + ZoomRegistry ───────────────────────────── #[derive(Clone)] @@ -1785,6 +1816,7 @@ impl Tool for ClickTool { use cua_driver_core::tool_args::ArgsExt; use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error}; use crate::uia::cache::SnapshotKind; + let cursor_key = resolve_cursor_key(&args); let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; let hwnd_opt = args.opt_u64("window_id"); let elem_idx = args.opt_u64("element_index").map(|v| v as usize); @@ -1869,9 +1901,9 @@ impl Tool for ClickTool { )), } }; - pin_overlay_above(hwnd); - overlay_glide_to(tx as f64, ty as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, tx as f64, ty as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: tx as f64, y: ty as f64, }); let btn_fg = button.clone(); @@ -1898,10 +1930,10 @@ impl Tool for ClickTool { None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")), }; // Step 2: pin overlay to target window, then animate to screen coords. - pin_overlay_above(hwnd); - overlay_glide_to(cx as f64, cy as f64).await; + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; // Step 3: click pulse + actual click. - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64, }); let btn = button.clone(); @@ -2067,9 +2099,9 @@ impl Tool for ClickTool { // mapping (DWM-frame top-left + 1-px inset, NOT ClientToScreen). let (sx_i, sy_i) = bitmap_to_screen(hwnd, px as i32, py as i32); let (sx, sy) = (sx_i as f64, sy_i as f64); - pin_overlay_above(hwnd); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, sx, sy).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); let btn = button.clone(); // Vision-mode (x, y) dispatch is **layered**, mirroring the // trope-cua reference impl @@ -2307,6 +2339,7 @@ impl Tool for TypeTextTool { let raw_pid = match args.require_i64("pid") { Ok(v) => v, Err(e) => return e }; let pid = raw_pid as u32; let text_raw = match args.require_str("text") { Ok(v) => v, Err(e) => return e }; + let cursor_key = resolve_cursor_key(&args); // Strip trailing agent-protocol closing tags before delivery — // catches the case where an LLM hallucinated its own tool- // invocation tags into the text param (see text_sanitize docs). @@ -2364,7 +2397,7 @@ impl Tool for TypeTextTool { // Pin the agent-cursor overlay above the target window so the synthetic // cursor stays sandwiched at z+1 of the type target for the full // duration of the keystrokes (both XAML/UIA and PostMessage paths). - pin_overlay_above(hwnd); + pin_overlay_above(&cursor_key, hwnd); // Glide the agent cursor onto the field being typed into, so the viewer // can see *where* the agent is typing — same visual feedback as a click. @@ -2372,8 +2405,8 @@ impl Tool for TypeTextTool { // the focused-element path has no resolvable position to point at. if let Some(idx) = elem_idx { if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx as usize) { - overlay_glide_to(cx as f64, cy as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64, }); } @@ -2825,6 +2858,7 @@ impl Tool for SetValueTool { async fn invoke(&self, args: Value) -> ToolResult { // Swift's "Missing required integer fields pid, window_id, and element_index." let mut missing_ints: Vec<&str> = Vec::new(); + let cursor_key = resolve_cursor_key(&args); let raw_pid = args.get("pid").and_then(|v| v.as_i64()); if raw_pid.is_none() { missing_ints.push("pid"); } if args.get("window_id").and_then(|v| v.as_u64()).is_none() { missing_ints.push("window_id"); } @@ -2847,9 +2881,9 @@ impl Tool for SetValueTool { // the viewer can see *where* the agent is acting. No-op when the // overlay is disabled or the element has no cached center. if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) { - pin_overlay_above(hwnd); - overlay_glide_to(cx as f64, cy as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64, }); } @@ -3154,6 +3188,7 @@ impl Tool for DoubleClickTool { let x = args.opt_f64("x"); let y = args.opt_f64("y"); let dispatch = DispatchMode::from_args(&args); + let cursor_key = resolve_cursor_key(&args); // Swift validates "both x and y or neither" and "no element_index without window_id". let has_xy = x.is_some() && y.is_some(); let partial_xy = x.is_some() != y.is_some(); @@ -3190,9 +3225,9 @@ impl Tool for DoubleClickTool { Some(v) => v, None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")), }; - pin_overlay_above(hwnd); - overlay_glide_to(cx as f64, cy as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); // dispatch:"background" — reject if PostMessage would be silently dropped. if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) @@ -3244,9 +3279,9 @@ impl Tool for DoubleClickTool { // `bitmap_to_screen` doc for why ClientToScreen is wrong). let (sx_i, sy_i) = bitmap_to_screen(hwnd, px as i32, py as i32); let (sx, sy) = (sx_i as f64, sy_i as f64); - pin_overlay_above(hwnd); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, sx, sy).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); // dispatch:"background" — reject if PostMessage would be silently dropped. if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) @@ -3345,6 +3380,7 @@ impl Tool for RightClickTool { let x = args.opt_f64("x"); let y = args.opt_f64("y"); let dispatch = DispatchMode::from_args(&args); + let cursor_key = resolve_cursor_key(&args); // Port Swift's full validation set. let has_xy = x.is_some() && y.is_some(); let partial_xy = x.is_some() != y.is_some(); @@ -3381,9 +3417,9 @@ impl Tool for RightClickTool { Some(v) => v, None => return ToolResult::error(format!("Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first.")), }; - pin_overlay_above(hwnd); - overlay_glide_to(cx as f64, cy as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { @@ -3434,9 +3470,9 @@ impl Tool for RightClickTool { // `bitmap_to_screen` doc). let (sx_i, sy_i) = bitmap_to_screen(hwnd, px as i32, py as i32); let (sx, sy) = (sx_i as f64, sy_i as f64); - pin_overlay_above(hwnd); - overlay_glide_to(sx, sy).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + pin_overlay_above(&cursor_key, hwnd); + overlay_glide_to(&cursor_key, sx, sy).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { @@ -3520,6 +3556,7 @@ impl Tool for DragTool { let dispatch = DispatchMode::from_args(&args); use cua_driver_core::tool_args::ArgsExt; + let cursor_key = resolve_cursor_key(&args); // Accepts numeric JSON as either float or integer — coerce both to f64. let coerce = |key: &str| -> Option { args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64)) @@ -3614,7 +3651,7 @@ impl Tool for DragTool { // cursor stays sandwiched at z+1 of the dragged window for the full // path. Drag stays within a single HWND, so one pin at the start is // sufficient — the 80 ms z-order tick keeps it asserted thereafter. - pin_overlay_above(hwnd); + pin_overlay_above(&cursor_key, hwnd); // Animate the agent cursor to the drag-start, fire a press pulse, // run the actual drag synthesis, then glide to the drag-end and @@ -3623,8 +3660,8 @@ impl Tool for DragTool { // the drag itself (the timing coordination would be invasive); // pre- and post-glides plus the press/release pulses are enough // signal for a user watching the agent operate. - overlay_glide_to(sx_from as f64, sy_from as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + overlay_glide_to(&cursor_key, sx_from as f64, sy_from as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx_from as f64, y: sy_from as f64, }); @@ -3642,8 +3679,8 @@ impl Tool for DragTool { // pulse the release. Skipped on error so the cursor doesn't lie // about a successful endpoint. if matches!(&result, Ok(Ok(()))) { - overlay_glide_to(sx_to as f64, sy_to as f64).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + overlay_glide_to(&cursor_key, sx_to as f64, sy_to as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx_to as f64, y: sy_to as f64, }); } @@ -3748,16 +3785,20 @@ impl Tool for MoveCursorTool { use cua_driver_core::tool_args::ArgsExt; let x = args.f64_or("x", 0.0); let y = args.f64_or("y", 0.0); - let cursor_id_owned = args.str_or("cursor_id", "default"); - let cursor_id = cursor_id_owned.as_str(); - self.state.cursor_registry.update_position(cursor_id, x, y); + // Cursor key precedence: caller-declared `session` > legacy `cursor_id` + // > NO_CURSOR. An anonymous run (no session) has no cursor to move. + let cursor_key = resolve_cursor_key(&args); + if !cursor_key.is_empty() { + self.state.cursor_registry.update_position(&cursor_key, x, y); + } // End pointing upper-left (45°) — matches Swift's // `AgentCursor.animateAndWait(endAngleDegrees: 45)` convention so // the cursor settles to the natural macOS-style pose. - crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: std::f64::consts::FRAC_PI_4, }); - ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).")) + let shown = if cursor_key.is_empty() { "default" } else { cursor_key.as_str() }; + ToolResult::text(format!("Agent cursor '{shown}' moved to ({x:.1}, {y:.1}).")) } } @@ -3797,11 +3838,11 @@ impl Tool for SetAgentCursorEnabledTool { Some(v) => v, None => return ToolResult::error("Missing required boolean field `enabled`."), }; - use cua_driver_core::tool_args::ArgsExt; - let cursor_id_owned = args.str_or("cursor_id", "default"); - let cursor_id = cursor_id_owned.as_str(); - self.state.cursor_registry.set_enabled(cursor_id, enabled); - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); + let cursor_key = resolve_cursor_key(&args); + if !cursor_key.is_empty() { + self.state.cursor_registry.set_enabled(&cursor_key, enabled); + } + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::SetEnabled(enabled)); // Match Swift text format 1:1: `"✅ Agent cursor enabled."` // (or `"✅ Agent cursor disabled."`). ToolResult::text(if enabled { @@ -3867,7 +3908,8 @@ impl Tool for SetAgentCursorMotionTool { fn num(v: Option<&Value>) -> Option { v.and_then(|x| x.as_f64().or_else(|| x.as_i64().map(|i| i as f64))) } - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + // Cursor key: caller-declared `session` > legacy `cursor_id` > NO_CURSOR. + let cursor_id = resolve_cursor_key(&args); // 1. Per-instance appearance fields (Rust-only). self.state.cursor_registry.update_config(&cursor_id, |cfg| { if let Some(v) = args.get("cursor_icon").and_then(|v| v.as_str()) { cfg.cursor_icon = Some(v.to_owned()); } @@ -3878,7 +3920,7 @@ impl Tool for SetAgentCursorMotionTool { }); // 2. Apply motion knobs to the live render state — was silently // dropped before; this is the Swift parity behavior. - let current = crate::overlay::current_motion(); + let current = crate::overlay::current_motion(&cursor_id); let updated = current.with_overrides( num(args.get("start_handle")), num(args.get("end_handle")), @@ -3890,7 +3932,7 @@ impl Tool for SetAgentCursorMotionTool { num(args.get("idle_hide_ms")), None, // press_duration_ms — not in Swift tool surface ); - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetMotion(updated.clone())); + crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetMotion(updated.clone())); // Match Swift text format 1:1. let summary = format!( "cursor motion: startHandle={sh} endHandle={eh} arcSize={asz} arcFlow={af} \ @@ -3938,9 +3980,12 @@ impl Tool for GetAgentCursorStateTool { read_only: true, destructive: false, idempotent: true, open_world: false, }) } - async fn invoke(&self, _args: Value) -> ToolResult { - let enabled = crate::overlay::is_enabled(); - let motion = crate::overlay::current_motion(); + async fn invoke(&self, args: Value) -> ToolResult { + // Report THIS session's cursor (caller-declared `session` > `cursor_id` + // > "default"), mirroring macOS get_agent_cursor_state scoping. + let cursor_key = resolve_cursor_key(&args); + let enabled = crate::overlay::is_enabled(&cursor_key); + let motion = crate::overlay::current_motion(&cursor_key); // Swift text format 1:1: single-line camelCase key=value pairs. let summary = format!( "cursor: enabled={enabled} startHandle={sh} endHandle={eh} arcSize={asz} \ @@ -4025,7 +4070,8 @@ impl Tool for SetAgentCursorStyleTool { } async fn invoke(&self, args: Value) -> ToolResult { - let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default").to_owned(); + // Cursor key: caller-declared `session` > legacy `cursor_id` > NO_CURSOR. + let cursor_id = resolve_cursor_key(&args); // image_path let image_path = args.get("image_path").and_then(|v| v.as_str()); @@ -4084,12 +4130,12 @@ impl Tool for SetAgentCursorStyleTool { // Dispatch to overlay if let Some(cmd) = shape_cmd { - crate::overlay::send_command(cmd); + crate::overlay::send_command(cursor_id.clone(), cmd); } let gradient_provided = args.get("gradient_colors").is_some(); let bloom_provided = args.get("bloom_color").is_some(); if gradient_provided || bloom_provided { - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetGradient { + crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetGradient { gradient_colors, bloom_color: bloom_color.flatten(), }); @@ -4322,15 +4368,15 @@ impl Tool for GetConfigTool { read_only: true, destructive: false, idempotent: true, open_world: false, }) } - async fn invoke(&self, _args: Value) -> ToolResult { + async fn invoke(&self, args: Value) -> ToolResult { let cfg = self.state.config.read().unwrap(); // Mirror the macOS agent's parity addition (commit adb9ecca): // nested `agent_cursor.enabled` block so Swift-shaped get_config // consumers can read the cursor's enabled state from one place. - let cursor_enabled = self.state.cursor_registry.all_states() - .first() - .map(|s| s.config.enabled) - .unwrap_or(true); + // Scope to the CALLING session's cursor (session > cursor_id > default) + // and read it from the overlay deterministically — `all_states().first()` + // was a nondeterministic HashMap read across sessions (macOS BUG 3). + let cursor_enabled = crate::overlay::is_enabled(&resolve_cursor_key(&args)); let (pip_enabled, pip_geometry) = pip_preview::read_pip_keys_from_file(); let payload = json!({ "schema_version": 1, @@ -5195,6 +5241,26 @@ pub fn build_registry(compat: bool) -> ToolRegistry { // Share the element cache with the recording-hook layer so it can // resolve element_index → window-local screenshot coords for click.png. crate::recording_hooks::set_element_cache(state.element_cache.clone()); + + // Drop a session's owned cursor on `session_end` (explicit end_session, the + // CLI `session end` verb, or the daemon idle-TTL sweep). The session id IS + // the cursor key (caller-declared `session`), so this prunes the metadata + // registry AND stops the overlay painting that session's cursor. Both paths + // guard "default" so the anonymous / one-shot cursor survives. Registering + // once per process (build_registry runs once in the daemon) is guarded so a + // repeated build in tests can't accumulate duplicate hooks. Mirrors the + // macOS `register_all` session_end hook (platform-macos/src/tools/mod.rs). + { + static HOOK_ONCE: std::sync::OnceLock<()> = std::sync::OnceLock::new(); + if HOOK_ONCE.set(()).is_ok() { + let cursor_registry = state.cursor_registry.clone(); + cua_driver_core::session::register_session_end_hook(move |session_id| { + cursor_registry.remove(session_id); + crate::overlay::remove_cursor(session_id.to_owned()); + }); + } + } + let mut r = ToolRegistry::new(); r.register(Box::new(ListAppsTool)); r.register(Box::new(ListWindowsTool)); @@ -5258,6 +5324,52 @@ pub fn build_registry(compat: bool) -> ToolRegistry { r } +#[cfg(test)] +mod cursor_key_resolution_tests { + use super::{resolve_cursor_key, NO_CURSOR}; + use serde_json::json; + + #[test] + fn anonymous_resolves_to_no_cursor() { + // No session/cursor_id → NO_CURSOR (""): the action still runs but no + // cursor is shown. The connection-injected `_session_id` is NOT a cursor + // source — it stays the recording/config lifecycle key. + assert_eq!(resolve_cursor_key(&json!({})), NO_CURSOR); + assert_eq!(resolve_cursor_key(&json!({ "pid": 1 })), NO_CURSOR); + assert_eq!(resolve_cursor_key(&json!({ "_session_id": "mcp-1-2" })), NO_CURSOR); + } + + #[test] + fn explicit_session_owns_a_cursor() { + assert_eq!(resolve_cursor_key(&json!({ "session": "research-run" })), "research-run"); + } + + #[test] + fn cursor_id_is_a_legacy_alias_and_session_wins() { + assert_eq!(resolve_cursor_key(&json!({ "cursor_id": "user-handle" })), "user-handle"); + assert_eq!(resolve_cursor_key(&json!({ "session": "s1", "cursor_id": "c1" })), "s1"); + } + + #[test] + fn empty_strings_fall_through_to_no_cursor() { + // An empty `session` falls through to `cursor_id`; both empty → NO_CURSOR. + assert_eq!(resolve_cursor_key(&json!({ "session": "", "cursor_id": "c1" })), "c1"); + assert_eq!(resolve_cursor_key(&json!({ "session": "", "cursor_id": "" })), NO_CURSOR); + } + + #[test] + fn two_parallel_sessions_resolve_distinct_keys() { + // The regression this whole port fixes: two concurrent runs each declare + // their own `session`, so they resolve DISTINCT cursor keys and own + // separate overlay cursors instead of clobbering one shared cursor. + let a = resolve_cursor_key(&json!({ "pid": 10, "element_index": 1, "session": "calc-2plus1" })); + let b = resolve_cursor_key(&json!({ "pid": 20, "element_index": 1, "session": "calc-5plus6" })); + assert_eq!(a, "calc-2plus1"); + assert_eq!(b, "calc-5plus6"); + assert_ne!(a, b); + } +} + #[cfg(test)] mod launch_focus_restore_decision_tests { use super::{should_restore_foreground_after_launch, LaunchTargetShape}; diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs index 5557ceb53d..8d25a7c514 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs @@ -228,10 +228,14 @@ impl PageBackend for WindowsPageBackend { use windows::Win32::UI::WindowsAndMessaging::{GetAncestor, GA_ROOT}; let root = unsafe { GetAncestor(HWND(hwnd as *mut _), GA_ROOT) }; let pin_wid = if !root.0.is_null() { root.0 as u64 } else { hwnd }; - crate::overlay::send_command(cursor_overlay::OverlayCommand::PinAbove(pin_wid)); + crate::overlay::send_command_default(cursor_overlay::OverlayCommand::PinAbove(pin_wid)); } - crate::overlay::animate_cursor_to(screen_x, screen_y).await; - crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + // The cross-platform `PageBackend::click_element` trait carries no + // caller `session`, so this drives the seeded `"default"` cursor rather + // than a per-session one. Threading session through the trait is a + // separate cross-platform change (tracked as a follow-up). + crate::overlay::animate_cursor_to("default".to_owned(), screen_x, screen_y).await; + crate::overlay::send_command_default(cursor_overlay::OverlayCommand::ClickPulse { x: screen_x, y: screen_y, }); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/stubs.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/stubs.rs index d8f486c30a..a7eb705411 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/stubs.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/stubs.rs @@ -164,7 +164,7 @@ mod move_cursor_m { let x = args.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0); let y = args.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0); let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); - crate::overlay::send_command(cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: 0.0 }); + crate::overlay::send_command(cursor_id.to_owned(), cursor_overlay::OverlayCommand::MoveTo { x, y, end_heading_radians: 0.0 }); ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).")) } } @@ -191,7 +191,7 @@ mod set_enabled_m { async fn invoke(&self, args: Value) -> ToolResult { let enabled = args.get("enabled").and_then(|v| v.as_bool()).unwrap_or(true); let cursor_id = args.get("cursor_id").and_then(|v| v.as_str()).unwrap_or("default"); - crate::overlay::send_command(cursor_overlay::OverlayCommand::SetEnabled(enabled)); + crate::overlay::send_command(cursor_id.to_owned(), cursor_overlay::OverlayCommand::SetEnabled(enabled)); ToolResult::text(format!("Agent cursor '{}' {}.", cursor_id, if enabled { "enabled" } else { "disabled" })) } }