Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,19 @@ impl Tool for TypeTextTool {
}
}
};
// Pulse the agent cursor onto the field being typed into (when an
// element_index is supplied) so the viewer sees *where* typing happens.
if let Some(idx) = args.opt_u64("element_index") {
let idx = idx as usize;
if let Ok(Ok((bx, by, bw, bh))) =
tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await
{
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
x: bx as f64 + bw as f64 / 2.0,
y: by as f64 + bh as f64 / 2.0,
});
}
}
let text_len = text.chars().count();
let result = tokio::task::spawn_blocking(move || {
crate::input::send_type_text(xid, &text)
Expand Down Expand Up @@ -849,6 +862,18 @@ impl Tool for SetValueTool {
let idx = match args.require_u64("element_index") { Ok(v) => v as usize, Err(e) => return e };
let value = match args.require_str("value") { Ok(v) => v, Err(e) => return e };
let value_for_task = value.clone();
// Pulse the agent cursor onto the target element before writing, so a
// value write gets the same visual feedback as a click — the viewer can
// see *where* the agent is acting. No-op when the element bounds can't
// be resolved or the overlay is disabled.
if let Ok(Ok((bx, by, bw, bh))) =
tokio::task::spawn_blocking(move || crate::atspi::get_element_bounds(pid, idx)).await
{
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
x: bx as f64 + bw as f64 / 2.0,
y: by as f64 + bh as f64 / 2.0,
});
}
let result = tokio::task::spawn_blocking(move || {
crate::atspi::set_value(pid, idx, &value_for_task)
}).await;
Expand Down
25 changes: 25 additions & 0 deletions libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2366,6 +2366,19 @@ impl Tool for TypeTextTool {
// duration of the keystrokes (both XAML/UIA and PostMessage paths).
pin_overlay_above(hwnd);

// Glide the agent cursor onto the field being typed into, so the viewer
// can see *where* the agent is typing — same visual feedback as a click.
// Only when an element_index is supplied (we have its cached center);
// the focused-element path has no resolvable position to point at.
if let Some(idx) = elem_idx {
if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx as usize) {
overlay_glide_to(cx as f64, cy as f64).await;
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
x: cx as f64, y: cy as f64,
});
}
}

// CUA-543 routing: PostMessage WM_CHAR doesn't reach modern
// XAML / WinUI3 hosts. When the target is one of those AND the
// caller has supplied an element_index, route through UIA
Expand Down Expand Up @@ -2829,6 +2842,18 @@ impl Tool for SetValueTool {
None => return ToolResult::error("Missing required string field value."),
};

// Glide the agent cursor onto the target element before writing its
// value, so a value write gets the same visual feedback as a click —
// the viewer can see *where* the agent is acting. No-op when the
// overlay is disabled or the element has no cached center.
if let Some((cx, cy)) = self.state.element_cache.get_element_center(pid, hwnd, idx) {
pin_overlay_above(hwnd);
overlay_glide_to(cx as f64, cy as f64).await;
crate::overlay::send_command(cursor_overlay::OverlayCommand::ClickPulse {
x: cx as f64, y: cy as f64,
});
}

let state = self.state.clone();
let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> {
let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)
Expand Down
Loading