diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs index cf91ff22ae..114ac9bd5f 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs @@ -12,7 +12,7 @@ pub mod mouse; pub mod keyboard; -pub use mouse::{post_click, post_click_screen}; +pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized}; pub use keyboard::{ is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay, send_key_synthesized, diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs index d0eb66469f..317d6a415b 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mouse.rs @@ -4,14 +4,21 @@ //! (via ChildWindowFromPointEx), so the message never reaches the top-level //! chrome that would call SetForegroundWindow in response to WM_LBUTTONDOWN. -use anyhow::Result; +use anyhow::{Result, bail}; use std::thread::sleep; use std::time::Duration; use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM}; use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient}; +use windows::Win32::UI::Input::KeyboardAndMouse::{ + INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, + MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, + MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput, +}; use windows::Win32::UI::WindowsAndMessaging::{ ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, - PostMessageW, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, + GetCursorPos, GetForegroundWindow, GetSystemMetrics, PostMessageW, SetCursorPos, + SetForegroundWindow, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, + SM_YVIRTUALSCREEN, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEMOVE, WM_RBUTTONDOWN, WM_RBUTTONUP, }; @@ -163,3 +170,191 @@ pub fn post_drag( fn make_lparam(x: i32, y: i32) -> LPARAM { LPARAM((((y as u16 as u32) << 16) | (x as u16 as u32)) as isize) } + +/// Returns `true` when `hwnd` is a top-level frame of a Chromium-based browser +/// — Edge, Chrome, Brave, Vivaldi, Opera, Chromium, Arc, Thorium, Iridium, +/// Yandex, or any other Chromium-derivative. Matches by window class name, +/// which is stable across versions and consistent across Chromium forks. +/// +/// Chromium uses the window class `Chrome_WidgetWin_1` (or `Chrome_WidgetWin_0` +/// for in-process child frames; both should be treated the same way). Electron +/// apps that embed Chromium also use this class, so Electron app coord clicks +/// will route through the SendInput path too — that's intentional, same root +/// cause (#1623). +/// +/// Cheap call: one `GetClassNameW` to a 32-char buffer + a `matches!` against +/// the known prefixes. Suitable to call inline in the click dispatch hot path. +pub fn is_chromium_target_window(hwnd: u64) -> bool { + use windows::Win32::UI::WindowsAndMessaging::GetClassNameW; + if hwnd == 0 { + tracing::debug!(target: "click", "is_chromium_target_window: hwnd=0 short-circuit"); + return false; + } + let mut buf = [0u16; 64]; + let n = unsafe { GetClassNameW(HWND(hwnd as *mut _), &mut buf) }; + if n <= 0 { + tracing::debug!( + target: "click", + "is_chromium_target_window: GetClassNameW returned {n} for hwnd=0x{hwnd:x}" + ); + return false; + } + let class_name = String::from_utf16_lossy(&buf[..n as usize]); + // Chromium-family classes. Match by prefix because Chromium suffixes a + // 0/1 digit; future Chromium forks may use other suffixes. + let is_chromium = class_name.starts_with("Chrome_WidgetWin_") + // Electron sometimes uses CefBrowserWindow or similar — be permissive. + || class_name.starts_with("CefBrowser"); + tracing::debug!( + target: "click", + "is_chromium_target_window: hwnd=0x{hwnd:x} class={class_name:?} → {is_chromium}" + ); + is_chromium +} + +/// Click at **screen** coordinates `(sx, sy)` via `SendInput` against the +/// system input queue, briefly focusing `target` so the click lands there. +/// +/// Why this exists alongside `post_click_screen`: PostMessage(WM_LBUTTONDOWN) +/// to Chromium-based browsers' top-level frame HWND (or Chrome_RenderWidgetHostHWND +/// descendant) doesn't fire DOM `onclick` / `mousedown` handlers. Chromium's +/// input thread architecture requires events with `SendInput`-queue origin — +/// the same constraint that broke modifier-state hotkey delivery (#1614/#1618) +/// applies to coord clicks on Chromium content (#1623). +/// +/// `SendInput` puts the synthetic mouse events on the **system input queue**, +/// where Chromium's input filter accepts them. The trade-off is a brief +/// foreground swap + visible cursor jump (mitigated by saving/restoring the +/// previous foreground HWND and previous cursor position after the click). +/// +/// UIAccess constraint: `SetForegroundWindow` is restricted from non-UIAccess +/// processes when not driven by user input. The `cua-driver-uia` worker runs +/// at UIAccess integrity precisely so this restriction is lifted; outside the +/// worker, the foreground swap may silently fail and SendInput land on the +/// wrong window. Callers should funnel Chromium coord clicks through the +/// uia worker (the MCP proxy already prefers the uia pipe over the regular +/// pipe when both are running). +pub fn send_click_synthesized( + target: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, +) -> Result<()> { + let target = HWND(target as *mut _); + if target.0.is_null() { + bail!("invalid target hwnd"); + } + if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) { + // Same UIPI defense as PostMessage path — SendInput from non-UIAccess + // would fail just as silently as PostMessage when target is at higher + // integrity. Surface the diagnostic early. + bail!(msg); + } + + let (down_flag, up_flag) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + + // Convert screen pixel coords to normalized absolute coords spanning the + // virtual desktop (0..65535 across the union of all monitors). This is + // what `MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK` expects. + // + // Without VIRTUALDESK the coords are relative to the primary monitor only; + // multi-monitor setups would misroute. Better to always use VIRTUALDESK. + let (vd_x, vd_y) = unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + ) + }; + let (vd_w, vd_h) = unsafe { + ( + GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), + GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), + ) + }; + let norm_x = ((sx - vd_x) as i64 * 65535 / vd_w as i64).clamp(0, 65535) as i32; + let norm_y = ((sy - vd_y) as i64 * 65535 / vd_h as i64).clamp(0, 65535) as i32; + + let move_input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: norm_x, + dy: norm_y, + mouseData: 0, + dwFlags: MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + time: 0, + dwExtraInfo: 0, + }, + }, + }; + let down_input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: norm_x, dy: norm_y, mouseData: 0, + dwFlags: down_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + time: 0, dwExtraInfo: 0, + }, + }, + }; + let up_input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: norm_x, dy: norm_y, mouseData: 0, + dwFlags: up_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + time: 0, dwExtraInfo: 0, + }, + }, + }; + + unsafe { + // Save previous foreground + cursor position so we can restore. + let prev_fg = GetForegroundWindow(); + let mut prev_cursor = POINT::default(); + let _ = GetCursorPos(&mut prev_cursor); + + // Focus the target so the click lands there (mirrors send_key_synthesized). + let _ = SetForegroundWindow(target); + sleep(Duration::from_millis(8)); + + // Move the cursor first so the OS hover state matches before the click. + // `SetCursorPos` is the visible cursor move; the MOUSEEVENTF_MOVE input + // ensures Chromium's input filter sees a coordinated move event. + let _ = SetCursorPos(sx, sy); + + let count = count.max(1); + for i in 0..count { + let events = [move_input, down_input, up_input]; + let sent = SendInput(&events, std::mem::size_of::() as i32); + if sent as usize != events.len() { + // Partial insertion — restore foreground+cursor and bail with + // the standard "needs UIAccess worker" diagnostic. + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + let _ = SetForegroundWindow(prev_fg); + bail!( + "SendInput inserted only {sent} of {} mouse events. Likely cause: \ + the daemon is not at UIAccess integrity, so SetForegroundWindow was \ + rejected and the events landed on the wrong window. Route Chromium \ + coord clicks through the cua-driver-uia worker.", + events.len() + ); + } + if i + 1 < count { + sleep(Duration::from_millis(80)); + } + } + + // Brief settle so the target processes the click before we restore. + sleep(Duration::from_millis(40)); + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + let _ = SetForegroundWindow(prev_fg); + } + + Ok(()) +} diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs index 4971dfe2fc..ac1affba17 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs @@ -1292,6 +1292,55 @@ impl Tool for ClickTool { )); } } + + // #1623: PostMessage(WM_LBUTTONDOWN) to Chromium frame HWNDs doesn't + // reach the DOM input pipeline — Chromium's input thread only accepts + // events with SendInput-queue origin. For Chromium targets, take the + // SendInput path; for everything else, take the existing PostMessage + // path (the path was added as a no-focus-steal click delivery, which + // PostMessage uniquely provides). The Chromium path moves the cursor + // visibly and briefly steals foreground — there's no Chromium-native + // alternative that gets DOM events to fire without these tradeoffs. + // + // SendInput on Chromium requires the daemon to have UIAccess integrity + // (otherwise SetForegroundWindow is rejected and the events land on the + // wrong window). The MCP proxy auto-prefers the cua-driver-uia worker + // when both pipes are up, so this path runs with UIAccess in the + // common case. When UIAccess is missing, send_click_synthesized + // surfaces an actionable error and we fall through to PostMessage — + // PostMessage won't fire DOM events on Chromium either, but the user + // gets a meaningful diagnostic instead of silent no-op. + let chromium = tokio::task::spawn_blocking(move || { + crate::input::is_chromium_target_window(hwnd) + }) + .await + .unwrap_or(false); + if chromium { + let send_result = tokio::task::spawn_blocking(move || { + crate::input::send_click_synthesized(hwnd, sx as i32, sy as i32, count, &btn) + }) + .await; + match send_result { + Ok(Ok(())) => { + let click_word = match count { + 2 => "double-click", + 3 => "triple-click", + _ => "click", + }; + return ToolResult::text(format!( + "✅ Sent {click_word} via SendInput to pid {pid} (Chromium target)." + )); + } + Ok(Err(e)) => { + // Bubble the actionable diagnostic ("Run through uia worker") up + // to the caller rather than silently falling to PostMessage, + // which we know doesn't work for Chromium. + return ToolResult::error(e.to_string()); + } + Err(e) => return ToolResult::error(format!("Task error: {e}")), + } + } + let result = tokio::task::spawn_blocking(move || { crate::input::post_click(hwnd, px as i32, py as i32, count, &btn) }).await;