From 1f347c643f9514c3dd6cafbdc255431077b669ec Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Wed, 27 May 2026 08:49:37 +0000 Subject: [PATCH] feat(cua-driver-rs)(platform-windows,platform-linux): glide agent cursor to the field on set_value/type_text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent-cursor overlay only animated to click targets, so `set_value` and `type_text` filled fields with no visual cue of where the agent was acting. Move/pulse the cursor onto the target element first — reusing the same overlay helpers the click path already uses — when an `element_index` is supplied. - Windows (platform-windows/src/tools/impl_.rs): `SetValueTool` and `TypeTextTool` now pin_overlay_above + overlay_glide_to(center) + ClickPulse before writing. - Linux (platform-linux/src/tools/impl_.rs): both tools resolve `atspi::get_element_bounds` and ClickPulse at the element center (mirrors the existing click path). - macOS already animated the cursor (AgentCursor.animateAndWait) in both tools; no change. The glide only fires when an element_index is provided (where a position is resolvable); the focused-element typing path is unchanged. Co-Authored-By: Claude Opus 4.7 --- .../crates/platform-linux/src/tools/impl_.rs | 25 +++++++++++++++++++ .../platform-windows/src/tools/impl_.rs | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 7866f617c9..08c78b16de 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -681,6 +681,19 @@ impl Tool for TypeTextTool { } } }; + // Pulse the agent cursor onto the field being typed into (when an + // element_index is supplied) so the viewer sees *where* typing happens. + if let Some(idx) = args.opt_u64("element_index") { + let idx = idx as usize; + if let Ok(Ok((bx, by, bw, bh))) = + tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await + { + crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + x: bx as f64 + bw as f64 / 2.0, + y: by as f64 + bh as f64 / 2.0, + }); + } + } let text_len = text.chars().count(); let result = tokio::task::spawn_blocking(move || { crate::input::send_type_text(xid, &text) @@ -849,6 +862,18 @@ impl Tool for SetValueTool { let idx = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e }; let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e }; let value_for_task = value.clone(); + // Pulse the agent cursor onto the target element before writing, so a + // value write gets the same visual feedback as a click — the viewer can + // see *where* the agent is acting. No-op when the element bounds can't + // be resolved or the overlay is disabled. + if let Ok(Ok((bx, by, bw, bh))) = + tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await + { + crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + x: bx as f64 + bw as f64 / 2.0, + y: by as f64 + bh as f64 / 2.0, + }); + } let result = tokio::task::spawn_blocking(move || { crate::atspi::set_value(pid, idx, &value_for_task) }).await; diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 0350bae0b8..9b521160ef 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -2366,6 +2366,19 @@ impl Tool for TypeTextTool { // duration of the keystrokes (both XAML/UIA and PostMessage paths). pin_overlay_above(hwnd); + // Glide the agent cursor onto the field being typed into, so the viewer + // can see *where* the agent is typing — same visual feedback as a click. + // Only when an element_index is supplied (we have its cached center); + // the focused-element path has no resolvable position to point at. + if let Some(idx) = elem_idx { + if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx as usize) { + overlay_glide_to(cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + x: cx as f64, y: cy as f64, + }); + } + } + // CUA-543 routing: PostMessage WM_CHAR doesn't reach modern // XAML / WinUI3 hosts. When the target is one of those AND the // caller has supplied an element_index, route through UIA @@ -2829,6 +2842,18 @@ impl Tool for SetValueTool { None => return ToolResult::error("Missing required string field value."), }; + // Glide the agent cursor onto the target element before writing its + // value, so a value write gets the same visual feedback as a click — + // the viewer can see *where* the agent is acting. No-op when the + // overlay is disabled or the element has no cached center. + if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) { + pin_overlay_above(hwnd); + overlay_glide_to(cx as f64, cy as f64).await; + crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse { + x: cx as f64, y: cy as f64, + }); + } + let state = self.state.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result { let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)