From c3089908b42170d6c622b2518033a5b4ee8f3605 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Tue, 23 Jun 2026 02:38:51 -0700 Subject: [PATCH 1/3] fix(cua-driver/windows): route DoubleClick/RightClick on Chromium targets via SendInput (#1984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClickTool auto-detects Chromium/Electron target HWNDs and routes clicks through SendInput because Chromium's input thread silently drops PostMessage mouse events (#1623). DoubleClickTool and RightClickTool lacked this short-circuit, so element_index/pixel double- and right-clicks on Electron apps (Obsidian, VS Code, Slack, …) fell through to post_click_screen (PostMessage) and no-op'd — matching the #1984 candidate B audit hypothesis. - Add chromium_click_short_circuit() helper (mirrors the ClickTool branch: detect Chromium HWND, deliver via send_click_synthesized with async foreground restore, else return None to keep the PostMessage path). - Wire it into both dispatch paths (element_index + x/y) of DoubleClickTool and RightClickTool, before the default PostMessage call. DragTool has the same gap on its default path; left as a fast-follow since its press-move-release SendInput path (send_drag_synthesized) needs separate handling. Candidate A (stale element-cache center) is a deeper audit, tracked separately. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KMXCW4M5uK1HRGjjH4wueZ --- .../platform-windows/src/tools/impl_.rs | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 9938a156bd..37fefd5d6d 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -3511,6 +3511,54 @@ impl Tool for ScrollTool { // `GetWindowStateTool` — that's where the actual screenshot machinery // lives now. +/// Chromium/Electron windows silently drop `PostMessage` mouse events — their +/// input thread only honors SendInput-origin events (#1623). `ClickTool` +/// auto-routes Chromium targets through SendInput, but `DoubleClickTool` / +/// `RightClickTool` did not, so element/pixel gestures on Electron apps +/// (Obsidian, VS Code, Slack, …) reached `post_click_screen` and no-op'd +/// silently. This mirrors that short-circuit for those tools (#1984): when +/// `hwnd` is a Chromium window, deliver `count` clicks of `button` at screen +/// `(sx, sy)` via SendInput with async foreground restore and return +/// `Some(result)`. Returns `None` for non-Chromium targets so the caller +/// proceeds to its normal PostMessage path. +async fn chromium_click_short_circuit( + hwnd: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, + pid: u32, + gesture: &str, +) -> Option { + let is_chromium = tokio::task::spawn_blocking(move || { + crate::input::is_chromium_target_window(hwnd) + }) + .await + .unwrap_or(false); + if !is_chromium { + return None; + } + // Capture the pre-click foreground so the poller can restore it even if + // Chromium re-activates itself from a renderer-side handler (same pattern + // and rationale as the ClickTool Chromium branch). + let prev_fg_addr = unsafe { + windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize + }; + let button_owned = button.to_string(); + let send_result = tokio::task::spawn_blocking(move || { + crate::input::send_click_synthesized(hwnd, sx, sy, count, &button_owned) + }) + .await; + tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid)); + Some(match send_result { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Sent {gesture} via SendInput to pid {pid} at screen ({sx},{sy}) (Chromium target)." + )), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }) +} + // ── double_click ────────────────────────────────────────────────────────────── pub struct DoubleClickTool { @@ -3651,6 +3699,10 @@ impl Tool for DoubleClickTool { Err(e) => ToolResult::error(format!("Task error: {e}")), }; } + // Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput. + if let Some(r) = chromium_click_short_circuit(hwnd, cx, cy, 2, "left", pid, "double-click").await { + return r; + } let result = tokio::task::spawn_blocking(move || -> anyhow::Result { crate::input::post_click_screen(hwnd, cx, cy, 2, "left")?; // Swift text format 1:1: `"✅ Posted double-click to [N] role \"title\" at screen-point (X, Y)."`. @@ -3706,6 +3758,10 @@ impl Tool for DoubleClickTool { }; } let (xi, yi) = (px as i32, py as i32); + // Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput. + if let Some(r) = chromium_click_short_circuit(hwnd, sx_i, sy_i, 2, "left", pid, "double-click").await { + return r; + } let result = tokio::task::spawn_blocking(move || crate::input::post_click_screen(hwnd, sx_i, sy_i, 2, "left")).await; match result { Ok(Ok(())) => { @@ -3861,6 +3917,10 @@ impl Tool for RightClickTool { Err(e) => ToolResult::error(format!("Task error: {e}")), }; } + // Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput. + if let Some(r) = chromium_click_short_circuit(hwnd, cx, cy, 1, "right", pid, "right-click").await { + return r; + } let result = tokio::task::spawn_blocking(move || -> anyhow::Result { crate::input::post_click_screen(hwnd, cx, cy, 1, "right")?; // Match Swift's element-path text 1:1 @@ -3915,6 +3975,10 @@ impl Tool for RightClickTool { }; } let (xi, yi) = (px as i32, py as i32); + // Chromium/Electron silently drops PostMessage clicks (#1984) — route via SendInput. + if let Some(r) = chromium_click_short_circuit(hwnd, sx_i, sy_i, 1, "right", pid, "right-click").await { + return r; + } let result = tokio::task::spawn_blocking(move || crate::input::post_click_screen(hwnd, sx_i, sy_i, 1, "right")).await; match result { Ok(Ok(())) => { From 66e0091370b70735527b0d38aaa7259b31847143 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Tue, 23 Jun 2026 18:40:04 -0700 Subject: [PATCH 2/3] fix(cua-driver/windows): inject (no FG swap) for Chromium in background dispatch, not just auto (#1984) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runtime testing on a real desktop (Edge) revealed the first cut was ineffective in the DEFAULT path: double_click/right_click default to dispatch:background, where the pre-existing 'would_be_silently_dropped' guard returned background_unavailable_error for Chromium BEFORE the auto-only SendInput short-circuit could run — so a Chromium double-click just errored. Mirror ClickTool exactly: in the background branch, route Chromium/GTK targets through inject_click_screen (coordinate injection into the system input queue, NO foreground swap) and only fall back to background_unavailable_error if the actuator can't express the click (e.g. right/middle). The auto-path SendInput short-circuit is retained for dispatch:auto. Covers both addressing modes (element_index + x/y) of DoubleClickTool and RightClickTool. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KMXCW4M5uK1HRGjjH4wueZ --- .../platform-windows/src/tools/impl_.rs | 58 +++++++++++++++++-- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 37fefd5d6d..9fb49e2059 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -3676,11 +3676,24 @@ impl Tool for DoubleClickTool { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); - // dispatch:"background" — reject if PostMessage would be silently dropped. + // dispatch:"background" (default): Chromium/Electron & GTK targets + // silently drop posted clicks (#1984) — inject via the coordinate + // actuator (system input queue, NO foreground swap), exactly like + // ClickTool, instead of refusing. Only error if injection can't + // express this click (e.g. right/middle on such a target). if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { - return background_unavailable_error(hwnd, EventKind::MouseClick); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, cx, cy, 2, "left") + }).await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected double-click to [{idx}] at screen ({cx},{cy}) (background, no foreground swap)." + )), + Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; } // dispatch:"foreground" — route through SendInput at the cached coords. if dispatch == DispatchMode::Foreground { @@ -3734,11 +3747,21 @@ impl Tool for DoubleClickTool { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, sx, sy).await; crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); - // dispatch:"background" — reject if PostMessage would be silently dropped. + // dispatch:"background" (default): inject via the coordinate actuator + // (no foreground swap) for targets that drop posted clicks (#1984). if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { - return background_unavailable_error(hwnd, EventKind::MouseClick); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, sx_i, sy_i, 2, "left") + }).await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected double-click to pid {pid} at screen ({sx_i},{sy_i}) (background, no foreground swap)." + )), + Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; } // dispatch:"foreground" — SendInput at screen coords with FG swap. if dispatch == DispatchMode::Foreground { @@ -3896,10 +3919,22 @@ impl Tool for RightClickTool { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, cx as f64, cy as f64).await; crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: cx as f64, y: cy as f64 }); + // dispatch:"background" (default): try coordinate injection (no + // foreground swap) for drop-prone targets (#1984); a right-click + // injection that the actuator can't express falls back to the error. if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { - return background_unavailable_error(hwnd, EventKind::MouseClick); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, cx, cy, 1, "right") + }).await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected right-click to [{idx}] at screen ({cx},{cy}) (background, no foreground swap)." + )), + Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; } if dispatch == DispatchMode::Foreground { let prev_fg_addr = unsafe { @@ -3953,10 +3988,21 @@ impl Tool for RightClickTool { pin_overlay_above(&cursor_key, hwnd); overlay_glide_to(&cursor_key, sx, sy).await; crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }); + // dispatch:"background" (default): try coordinate injection (no + // foreground swap) for drop-prone targets (#1984). if dispatch == DispatchMode::Background && crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick) { - return background_unavailable_error(hwnd, EventKind::MouseClick); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, sx_i, sy_i, 1, "right") + }).await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected right-click to pid {pid} at screen ({sx_i},{sy_i}) (background, no foreground swap)." + )), + Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; } if dispatch == DispatchMode::Foreground { let prev_fg_addr = unsafe { From 7ef67c1bfe5f73933789db2c17e444294a22b5a9 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Tue, 23 Jun 2026 18:48:24 -0700 Subject: [PATCH 3/3] fix(cua-driver/windows): background double/triple-click via one synthetic pen device (#1984) Runtime testing showed single click injected fine in background but double_click errored: inject_click_screen looped pen_tap, which creates AND destroys a synthetic pen device per tap. The second CreateSyntheticPointerDevice in quick succession fails, so the second tap (and thus every double/triple click) returned Err -> background_unavailable_error. Rename pen_tap -> pen_taps(count): create ONE device and emit count down/up cycles (70ms apart) on it, then destroy once. A real double-click is two taps from one digitizer, so this is also more correct. inject_click_screen now calls it once. Fixes background double_click / right-double scenarios for Chromium/Electron/GTK targets. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01KMXCW4M5uK1HRGjjH4wueZ --- .../platform-windows/src/input/inject.rs | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs index fe562a236e..96522002cc 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs @@ -339,7 +339,7 @@ impl Drop for ZorderGuard { /// (right) click — both for `WM_POINTER`-aware apps (Chromium/WPF/UWP) and via /// pen→mouse promotion for legacy Win32. A fresh synthetic pen device is /// created and destroyed per tap (right/middle clicks are rare). -fn pen_tap(sx: i32, sy: i32, barrel: bool) -> Result<()> { +fn pen_taps(sx: i32, sy: i32, barrel: bool, count: usize) -> Result<()> { unsafe { let dev = CreateSyntheticPointerDevice(PT_PEN, 1, POINTER_FEEDBACK_DEFAULT) .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(PEN): {e}"))?; @@ -367,12 +367,28 @@ fn pen_tap(sx: i32, sy: i32, barrel: bool) -> Result<()> { }, }; let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT); - let r1 = InjectSyntheticPointerInput(dev, &[down]); - sleep(Duration::from_millis(25)); let up = mk(POINTER_FLAG_UP); - let r2 = InjectSyntheticPointerInput(dev, &[up]); + // Reuse the SAME synthetic device for every tap. A double/triple click + // is two/three down-up cycles from one digitizer; creating a fresh + // device per tap (the old loop) fails the next + // CreateSyntheticPointerDevice in quick succession — which is exactly + // why background double_click on Chromium errored. See #1984. + let mut result: Result<()> = Ok(()); + let n = count.max(1); + for i in 0..n { + let r1 = InjectSyntheticPointerInput(dev, &[down]); + sleep(Duration::from_millis(25)); + let r2 = InjectSyntheticPointerInput(dev, &[up]); + if let Err(e) = r1.and(r2) { + result = Err(anyhow::anyhow!("InjectSyntheticPointerInput(pen): {e}")); + break; + } + if i + 1 < n { + sleep(Duration::from_millis(70)); + } + } let _ = DestroySyntheticPointerDevice(dev); - r1.and(r2).map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(pen): {e}"))?; + result?; } Ok(()) } @@ -419,13 +435,8 @@ pub fn inject_click_screen(target: u64, sx: i32, sy: i32, count: usize, button: // and hide/restore any residual z-order via the cloak/SWP guard. let _noact = NoActivateGuard::arm(target_h); let _guard = unsafe { ZorderGuard::arm(target_h) }; - let count = count.max(1); - for i in 0..count { - pen_tap(sx, sy, barrel)?; - if i + 1 < count { - sleep(Duration::from_millis(70)); - } - } + // One synthetic device does all `count` taps (single/double/triple click). + pen_taps(sx, sy, barrel, count)?; // _guard drops here: restore the user's foreground + uncloak target. Ok(()) }