diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs index 28af93dd1c..416dfd409e 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/input/keyboard.rs @@ -20,7 +20,12 @@ use std::thread::sleep; use std::time::Duration; use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; use windows::Win32::UI::Input::KeyboardAndMouse::{ - MapVirtualKeyW, MAPVK_VK_TO_VSC, VIRTUAL_KEY, + MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, + KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, + KEYEVENTF_SCANCODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + GetForegroundWindow, SetForegroundWindow, }; use windows::Win32::UI::WindowsAndMessaging::{ GetClassNameW, GetWindowThreadProcessId, IsChild, @@ -252,6 +257,118 @@ pub fn post_key(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> { Ok(()) } +/// Press `key` (with optional `modifiers`) via `SendInput` against the system +/// input queue, briefly focusing `hwnd` so the keystrokes land there. +/// +/// Why this exists alongside `post_key`: `PostMessage(WM_KEYDOWN, VK_CONTROL)` +/// puts a message in the target's queue but does NOT update the system-wide +/// modifier state that apps poll via `GetKeyState` / `GetAsyncKeyState`. For +/// any Win32 app whose accelerator dispatcher uses `TranslateAccelerator` (which +/// is most native Win32 apps — LibreOffice, FAR, classic Notepad, etc.), the +/// shortcut never fires; the `s` arrives as plain text input. +/// +/// `SendInput` puts the synthesized events on the **system input queue** — +/// the same queue `GetKeyState` reads from — so `Ctrl+S` is properly detected +/// as an accelerator. The trade-off is a brief foreground swap (focus theft), +/// which we mitigate by saving the previous foreground HWND and restoring it +/// after the keystrokes are flushed. +/// +/// UIAccess constraint: `SetForegroundWindow` is restricted from non-UIAccess +/// processes when not driven by user input. The `cua-driver-uia` worker runs +/// at UIAccess integrity precisely so this restriction is lifted; outside the +/// worker, the foreground swap may silently fail and SendInput land on the +/// wrong window. Callers should funnel hotkey calls through the uia worker. +pub fn send_key_synthesized(hwnd: u64, key: &str, modifiers: &[&str]) -> Result<()> { + let target = HWND(hwnd as *mut _); + if target.0.is_null() { + bail!("invalid target hwnd"); + } + if let Some(msg) = crate::input::post_message_blocked_by_uipi(hwnd) { + // Same UIPI defense as the PostMessage path. SendInput from UIAccess + // _is_ allowed cross-integrity, but if our daemon is somehow at a + // lower integrity than target, SendInput would land in the wrong + // window (we couldn't set foreground). Better to surface the + // diagnostic early than silently no-op. + bail!(msg); + } + let key_vk = key_name_to_vk(key)?; + let mod_vks: Vec = modifiers + .iter() + .filter_map(|m| modifier_vk(m)) + .collect(); + + // Build the INPUT sequence: modifiers down, key down, key up, modifiers up + // (reverse order). Each event sends the scancode + EXTENDEDKEY flag where + // appropriate so apps that read scancodes (not virtual keys) work too. + let mut events: Vec = Vec::with_capacity(mod_vks.len() * 2 + 2); + for mvk in &mod_vks { + events.push(key_input(*mvk, false)); + } + events.push(key_input(key_vk, false)); + events.push(key_input(key_vk, true)); + for mvk in mod_vks.iter().rev() { + events.push(key_input(*mvk, true)); + } + + unsafe { + // Save & set foreground so SendInput lands on `target`. + let prev_fg = GetForegroundWindow(); + let _ = SetForegroundWindow(target); + // Brief settle so the foreground swap is processed before we send. + sleep(Duration::from_millis(8)); + + let sent = SendInput(&events, std::mem::size_of::() as i32); + if sent as usize != events.len() { + // SendInput returns the number of events successfully inserted. + // Anything less is a partial insertion (blocked by another input + // injector, foreground UIPI denial, etc.). + let restored = SetForegroundWindow(prev_fg); + let _ = restored; + bail!( + "SendInput inserted only {sent} of {} events. Likely cause: \ + the daemon is not at UIAccess integrity, so SetForegroundWindow \ + was rejected and the events landed on the wrong window. Run \ + hotkey through the cua-driver-uia worker.", + events.len() + ); + } + + // Brief settle to let the target process the keystrokes before we + // restore the previous foreground (otherwise the target might not + // get a chance to handle the accelerator before losing focus). + sleep(Duration::from_millis(40)); + if !prev_fg.0.is_null() && prev_fg != target { + let _ = SetForegroundWindow(prev_fg); + } + } + Ok(()) +} + +/// Build a single keyboard INPUT struct for `vk`, either down (`up = false`) +/// or up (`up = true`). Uses scancode + EXTENDEDKEY where applicable so the +/// target sees a hardware-like keystroke. +fn key_input(vk: VIRTUAL_KEY, up: bool) -> INPUT { + let scan = unsafe { MapVirtualKeyW(vk.0 as u32, MAPVK_VK_TO_VSC) } as u16; + let mut flags: KEYBD_EVENT_FLAGS = KEYBD_EVENT_FLAGS(0); + // Scancode is more reliable than VK for some apps. EXTENDEDKEY flag + // makes arrow / nav / right-side modifier keys work correctly. + if scan != 0 { flags |= KEYEVENTF_SCANCODE; } + if is_extended(vk) { flags |= KEYEVENTF_EXTENDEDKEY; } + if up { flags |= KEYEVENTF_KEYUP; } + INPUT { + r#type: INPUT_KEYBOARD, + Anonymous: INPUT_0 { + ki: KEYBDINPUT { + wVk: if scan != 0 { VIRTUAL_KEY(0) } else { vk }, + wScan: scan, + dwFlags: flags, + time: 0, + dwExtraInfo: 0, + }, + }, + } +} + fn modifier_vk(name: &str) -> Option { use windows::Win32::UI::Input::KeyboardAndMouse::*; match name.to_lowercase().as_str() { diff --git a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs index 56124149ec..cf91ff22ae 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/input/mod.rs @@ -15,6 +15,7 @@ pub mod keyboard; pub use mouse::{post_click, post_click_screen}; pub use keyboard::{ is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay, + send_key_synthesized, }; use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND}; diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs index 4c3dcb5eff..4971dfe2fc 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs @@ -929,9 +929,19 @@ impl Tool for LaunchAppTool { } // Resolve the pid's windows so the caller can skip a list_windows - // round-trip — same approach as Swift's `resolveWindows`. Retry + // round-trip — same approach as Swift's `resolveWindows`. Retry // 5×200ms; Win32 window registration can lag the launch. + // + // Launcher-stub fallback: when the launched binary is a wrapper that + // re-execs and exits (GIMP's `gimp-3.exe` → `gimp-3.2.exe`; LO's + // `swriter.exe` → `soffice.bin`), the launched pid never gets a + // window — list_windows(Some(pid)) stays empty forever. After the + // primary retry budget we fall back to scanning the launched pid's + // descendant + name-related processes and pick the first one with a + // window. The resolved pid is reflected in the response's `pid` field + // so callers can target it with subsequent calls. See #1615. let mut windows_json: Vec = Vec::new(); + let mut resolved_pid: u32 = pid; for _ in 0..5 { let wins = tokio::task::spawn_blocking(move || crate::win32::list_windows(Some(pid))) .await.unwrap_or_default(); @@ -947,6 +957,90 @@ impl Tool for LaunchAppTool { } tokio::time::sleep(std::time::Duration::from_millis(200)).await; } + if windows_json.is_empty() { + // Launcher-stub fallback. Compute the exe basename from the + // launchable target so name-based matching has something to work + // with (e.g. "gimp-3.exe" → prefix "gimp" matches "gimp-3.2.exe"). + let basename_for_match = target_file_opt + .as_deref() + .and_then(|t| t.rsplit(|c: char| c == '\\' || c == '/').next()) + .unwrap_or("") + .to_owned(); + // Known-slow launchers get an extended retry budget. GIMP 3.x in + // particular spends 10-20s on its first launch (font cache rebuild, + // plugin scan, etc.) before the main window registers. We don't + // want to wait 20s for every launcher — gate on basename prefix + // matching known-slow apps. Add to this list as encountered. + let bn_lower = basename_for_match.to_ascii_lowercase(); + let is_slow_launcher = bn_lower.starts_with("gimp") + || bn_lower.starts_with("blender") // OpenGL init can stall + || bn_lower.starts_with("inkscape") // similar GTK pattern + || bn_lower.starts_with("krita") + || bn_lower.starts_with("freecad"); + let max_candidate_attempts: usize = if is_slow_launcher { 30 } else { 3 }; + + let basename_clone = basename_for_match.clone(); + let candidates_initial = tokio::task::spawn_blocking(move || { + crate::win32::related_processes(pid, &basename_clone) + }) + .await + .unwrap_or_default(); + + // For slow launchers we may also need to RE-SCAN candidates over + // time, because the wrapper may not have spawned its child yet + // when we first scanned. Cap total wait at ~12s (slow) / 0.6s (fast). + let mut tried: std::collections::HashSet = std::collections::HashSet::new(); + tried.insert(pid); // already tried in the primary loop + let mut candidate_queue: Vec = candidates_initial + .into_iter() + .filter(|p| tried.insert(*p)) + .collect(); + let mut total_attempts: usize = 0; + 'outer: loop { + while let Some(candidate_pid) = candidate_queue.pop() { + for _ in 0..max_candidate_attempts { + total_attempts += 1; + let wins = tokio::task::spawn_blocking(move || { + crate::win32::list_windows(Some(candidate_pid)) + }) + .await + .unwrap_or_default(); + if !wins.is_empty() { + windows_json = wins.iter().map(|w| json!({ + "window_id": w.hwnd, "title": w.title, + "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height }, + "layer": 0, + "z_index": 0, + "is_on_screen": true, + })).collect(); + resolved_pid = candidate_pid; + break 'outer; + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + } + // For slow launchers, keep re-scanning descendants — the + // wrapper may not have spawned its child yet. Cap total + // wait at ~12s (60 × 200ms) for the slow path. + if !is_slow_launcher || total_attempts > 60 { break; } + // Give the wrapper a moment to spawn before re-scanning. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + total_attempts += 3; // count the 500ms wait as 3 attempts + let basename_rescan = basename_for_match.clone(); + let fresh = tokio::task::spawn_blocking(move || { + crate::win32::related_processes(pid, &basename_rescan) + }) + .await + .unwrap_or_default(); + let new_ones: Vec = fresh.into_iter().filter(|p| tried.insert(*p)).collect(); + candidate_queue = new_ones; + // Continue the outer loop regardless — even with empty + // new_ones we want another iteration that hits the + // total_attempts cap. The loop body handles the empty queue + // by falling through to the re-scan again. + } + } + let pid = resolved_pid; // Match Swift text format 1:1. let mut summary = format!("✅ Launched {display} (pid {pid}) in background."); @@ -1603,12 +1697,33 @@ impl Tool for HotkeyTool { }; } + // Non-XAML Win32 path. Two routes available: + // 1. SendInput synthesized hotkey — pushes the events onto the + // *system input queue*. Updates GetKeyState's modifier state, so + // TranslateAccelerator-based apps (LibreOffice, FAR, classic + // Notepad, etc.) see Ctrl+S as a real accelerator. Trade-off: + // brief focus theft to ensure SendInput lands on the right HWND. + // 2. PostMessage WM_KEYDOWN/UP — no focus theft, but the + // synthesized Ctrl/Shift/Alt modifier never updates + // GetKeyState, so accelerators don't fire. Only useful for + // non-accelerator key sequences. + // We pick route 1 when modifiers are present (the accelerator case), + // route 2 otherwise (plain non-modifier keys still post fine). + let has_modifiers = !mods.is_empty(); let result = tokio::task::spawn_blocking(move || { let m: Vec<&str> = mods.iter().map(String::as_str).collect(); - crate::input::post_key(hwnd, &key, &m) + if has_modifiers { + crate::input::send_key_synthesized(hwnd, &key, &m) + } else { + crate::input::post_key(hwnd, &key, &m) + } }).await; + let path = if has_modifiers { "SendInput" } else { "PostMessage" }; match result { - Ok(Ok(())) => ToolResult::text(format!("✅ Pressed {key_display} on pid {raw_pid}.")), + Ok(Ok(())) => ToolResult::text(format!( + "✅ Pressed {key_display} on pid {raw_pid} via {path} \ + (Win32 target)." + )), Ok(Err(e)) => ToolResult::error(e.to_string()), Err(e) => ToolResult::error(format!("Task error: {e}")), } diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs index 9e81d35568..98dba9e432 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/apps.rs @@ -52,3 +52,82 @@ fn decode_wstr(buf: &[u16]) -> String { let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len()); String::from_utf16_lossy(&buf[..len]) } + +/// Return all transitive descendants of `root_pid` (BFS through the process +/// tree). Includes processes that may have been spawned *after* `root_pid` +/// itself exited — useful for tracking launcher-stub chains where the +/// originally-launched binary re-execs into another process and exits (GIMP's +/// `gimp-3.exe` → `gimp-3.2.exe`; LibreOffice's `swriter.exe` → `soffice.bin`). +/// +/// The result is in arrival order, which on Windows tends to correlate with +/// process-creation order — useful when picking the "main" descendant to +/// query for windows. Always includes `root_pid` itself first (even if it's +/// no longer alive, callers handle the empty-windows case the same way). +pub fn list_descendants(root_pid: u32) -> Vec { + let all = list_processes(); + let mut result = vec![root_pid]; + let mut frontier = vec![root_pid]; + while let Some(parent) = frontier.pop() { + for p in &all { + if p.parent_pid == parent && !result.contains(&p.pid) { + result.push(p.pid); + frontier.push(p.pid); + } + } + } + result +} + +/// Like `list_descendants` but ALSO returns processes whose executable name +/// matches a prefix derived from `exe_basename`. This catches the LibreOffice +/// pattern (swriter.exe spawns soffice.bin via a parent-pid relationship we +/// might miss if the spawn happened before our pre-launch snapshot, OR via +/// CreateProcess flags that detach the child from the launcher's tree) as +/// well as the GIMP pattern (gimp-3.exe spawns gimp-3.2.exe whose name starts +/// with the same prefix). +/// +/// Heuristic: strip extension and trailing version digits/dots/dashes from +/// the basename to derive a stable prefix. E.g.: +/// `gimp-3.exe` → prefix `gimp` +/// `gimp-3.2.exe` → prefix `gimp` +/// `swriter.exe` → prefix `swriter` (won't match `soffice.bin` — that's +/// LibreOffice's parent-pid path; usually still reachable +/// via `list_descendants`) +/// `notepad++.exe` → prefix `notepad++` (no version stripping needed) +/// +/// Returns deduplicated pids; ordering favors descendants over name-matches. +pub fn related_processes(root_pid: u32, exe_basename: &str) -> Vec { + let mut out = list_descendants(root_pid); + let prefix = strip_version_suffix(exe_basename); + if !prefix.is_empty() { + let all = list_processes(); + for p in &all { + let p_prefix = strip_version_suffix(&p.name); + if p_prefix.eq_ignore_ascii_case(&prefix) && !out.contains(&p.pid) { + out.push(p.pid); + } + } + } + out +} + +/// Strip `.exe` (case-insensitive) and any trailing `-....` +/// or `....` version suffix. Used by `related_processes` to +/// match `gimp-3.exe` and `gimp-3.2.exe` under the common prefix `gimp`. +fn strip_version_suffix(basename: &str) -> String { + let mut s = basename.to_ascii_lowercase(); + if let Some(stripped) = s.strip_suffix(".exe") { + s = stripped.to_owned(); + } + // Strip trailing version-like tail: `-3`, `-3.2`, `3`, `3.2`, etc. + let bytes = s.as_bytes(); + let mut cut = bytes.len(); + while cut > 0 { + let c = bytes[cut - 1] as char; + if c.is_ascii_digit() || c == '.' || c == '-' { cut -= 1; } else { break; } + } + // Avoid stripping an entire name (e.g. "7z" → "" would lose information). + // If everything past cut is purely digits/dots/dashes AND cut > 0, accept. + if cut == 0 { return s; } + s[..cut].to_string() +} diff --git a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs index 8bfec092bc..16836e10f0 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/win32/mod.rs @@ -4,6 +4,6 @@ pub mod apps; pub mod installed_apps; pub mod windows; -pub use apps::{list_processes, ProcessInfo}; +pub use apps::{list_descendants, list_processes, related_processes, ProcessInfo}; pub use installed_apps::{list_installed_apps, InstalledApp}; pub use windows::{list_windows, WindowInfo};