From 9325054b981bfad4ffcfea52c2557b120c4ca9ae Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Thu, 21 May 2026 12:29:11 +0200 Subject: [PATCH] fix(cua-driver-rs)(windows)(#1621): x,y click skips UIA Invoke on canvas-like surfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `click(x, y)` previously routed through `try_invoke_in_window_at_point` for any UIA element advertising InvokePattern at the click coordinates. For container surfaces (Pane / Image / Custom / Document / Group) `Invoke()` fires the element's default action at its centre and ignores the requested (x, y) — silently breaking pixel precision on canvases, paint surfaces, image maps, and 3D viewports. ## Repro (from #1621, verified 2026-05-21 on the Windows VM against `libs/cua-driver-fixtures/test_page.html` loaded in Edge) ``` cua-driver call click '{"pid":,"x":110,"y":677}' # → "✅ Performed UIA Invoke at (110,677)" # But #canvas-status shows: "canvas: clicked at (152,77)" # ─ canvas center, not the requested (110, 677) ``` The canvas's `mousedown` handler fired at synthesised centre coords because UIA `Invoke()` has no notion of "where inside the element". ## Fix Add `is_coord_independent_action()` — a control-type whitelist for elements whose primary action is coord-independent (Button, MenuItem, Hyperlink, TabItem, ListItem, CheckBox, RadioButton, SplitButton, TreeItem). For these, UIA Invoke is the semantically correct path — the element identity *is* the action target, and the click coords don't matter past hit-testing. For everything else (Pane / Image / Custom / Document / Group / etc.), even if the element advertises InvokePattern, fall through to PostMessage with the literal coords. This preserves UWP / WebView2 coverage for buttons + menu items (the original motivation for the UIA-first path) without silently rerouting canvas clicks to "click at centre". ## What this does NOT change - Element-indexed click (`click(element_index=N)`) is unchanged — it takes the UIA Invoke path explicitly via a different code site (`impl_.rs:1168`). Callers asking for "invoke this specific element" by index keep the previous behaviour. - Right-click and multi-click already skipped the UIA path (`use_uia = (btn == "left" || btn == "middle") && count == 1`); they remain on PostMessage. - The ExpandCollapsePattern preference for Qt menu-bar items (added in #1566) is unchanged — that path runs after the new control-type filter and only fires when a whitelisted-type element happens to also have ExpandCollapse. ## Test plan - [x] `cargo check -p platform-windows` clean on the VM (41.82s, 0 new warnings — all 28 warnings are pre-existing) - [x] `cargo build --release -p cua-driver` clean (24.43s release build) - [ ] **Runtime verification deferred to next interactive RDP session**: load `libs/cua-driver-fixtures/test_page.html` in Edge with the four anti-occlusion + a11y flags (see #1620), call `click(pid, x, y)` at a non-centre point inside the canvas, expect tool response to say `"✅ Posted click to pid "` (not `"Performed UIA Invoke"`) and `#canvas-status` to report the requested coords ±2px. The SSH-only session in tonight's autonomous run can't reach an interactive desktop (daemon ends up in Session 0; `list_windows` returns empty) so the click test couldn't run end-to-end — user needs to RDP in to verify. ## Related - #1620 — Chromium anti-throttling flags (separate fix; needed for any Edge/Chrome DOM verification on hidden launches, including this test) Closes #1621. Co-Authored-By: Claude Opus 4.7 --- .../platform-windows/src/uia/windows_enum.rs | 52 ++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs index 94c8b9e47f..fb316a279c 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs @@ -23,8 +23,11 @@ use windows::Win32::System::Com::{ use windows::Win32::UI::Accessibility::{ CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationInvokePattern, IUIAutomationTogglePattern, TreeScope_Children, TreeScope_Subtree, - UIA_AcceleratorKeyPropertyId, UIA_InvokePatternId, UIA_PROPERTY_ID, - UIA_TogglePatternId, + UIA_AcceleratorKeyPropertyId, UIA_ButtonControlTypeId, UIA_CheckBoxControlTypeId, + UIA_CONTROLTYPE_ID, UIA_HyperlinkControlTypeId, UIA_InvokePatternId, + UIA_ListItemControlTypeId, UIA_MenuItemControlTypeId, UIA_PROPERTY_ID, + UIA_RadioButtonControlTypeId, UIA_SplitButtonControlTypeId, UIA_TabItemControlTypeId, + UIA_TogglePatternId, UIA_TreeItemControlTypeId, }; use windows::core::{BSTR, Interface}; use windows::Win32::UI::WindowsAndMessaging::{ @@ -176,6 +179,35 @@ pub fn enumerate_top_level_windows() -> Vec { /// `CurrentBoundingRectangle` contains the point AND which exposes /// `InvokePattern`. Smallest-area approximates "deepest" without /// having to track tree depth explicitly. +/// Returns `true` when the element's control type has a *coord-independent* +/// primary action — i.e. a UIA `Invoke()` on it does something semantically +/// equivalent to "click the element" regardless of where inside its bounding +/// rectangle the click was requested. +/// +/// Used by the `x, y` click path to decide whether to take the UIA Invoke +/// route or fall through to PostMessage with the literal coords. The split +/// matters for canvases, panes, and custom-drawn surfaces where Invoke would +/// fire `mousedown` at the element centre — losing the caller's pixel +/// precision (see #1621). +fn is_coord_independent_action(elem: &IUIAutomationElement) -> bool { + let ct: UIA_CONTROLTYPE_ID = match unsafe { elem.CurrentControlType() } { + Ok(t) => t, + Err(_) => return false, + }; + matches!( + ct, + UIA_ButtonControlTypeId + | UIA_MenuItemControlTypeId + | UIA_HyperlinkControlTypeId + | UIA_TabItemControlTypeId + | UIA_ListItemControlTypeId + | UIA_CheckBoxControlTypeId + | UIA_RadioButtonControlTypeId + | UIA_SplitButtonControlTypeId + | UIA_TreeItemControlTypeId + ) +} + pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool { if hwnd == 0 { return false; @@ -234,6 +266,22 @@ pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool { if !has_invoke && !has_expand { continue; } + // For coordinate-addressed clicks, only accept elements whose + // control type has a *coord-independent* primary action. UIA + // `Invoke()` fires the element's default action at its centre, + // ignoring the requested (sx, sy). For container surfaces + // (Pane, Image, Custom, Document, Group, etc.) that means the + // caller's pixel precision is silently lost — see #1621, where + // `click(canvas, x=110, y=677)` reported success but actually + // fired the canvas's `mousedown` at its centre (152, 77). + // Buttons / MenuItems / Hyperlinks / TabItems / ListItems / + // CheckBoxes / RadioButtons / SplitButtons / TreeItems all + // have a single primary action whose location is the element + // itself — Invoke is the right path for those. Everything + // else falls through to PostMessage with the literal coords. + if !is_coord_independent_action(&elem) { + continue; + } let w = (rect.right - rect.left).max(0) as i64; let h = (rect.bottom - rect.top).max(0) as i64; let area = w.saturating_mul(h);