diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs index 54967ba6d1..87d171c197 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs @@ -54,6 +54,68 @@ pub fn perform_action(pid: u32, idx: usize) -> Result { native::perform_action(pid, idx) } +/// Try to type text into any editable field in the window via AT-SPI EditableText. +/// This works for unfocused windows if the toolkit exposes EditableText (Qt6, some GTK). +/// For Qt5, which doesn't expose widgets when unfocused, this will return Err. +/// Returns Ok if an editable was found and text was set, Err otherwise. +pub fn type_into_editable(pid: u32, text: &str) -> Result<()> { + let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'"); + let script = format!(r#" +import pyatspi, sys + +def find_editable(acc, depth=0): + # Try to find any EditableText interface, regardless of role + try: + et = acc.queryEditableText() + # If we can query it, return this node + return acc + except: + pass + + # Recursively search children + try: + for child in acc: + result = find_editable(child, depth + 1) + if result is not None: + return result + except: + pass + + return None + +desktop = pyatspi.Registry.getDesktop(0) +editable = None +for app in desktop: + try: + if app.get_process_id() == {pid}: + for win in app: + editable = find_editable(win) + if editable: + break + break + except: + pass + +if editable is None: + print("ERROR: No editable found", file=sys.stderr) + sys.exit(1) + +try: + et = editable.queryEditableText() + et.setTextContents('{safe_text}') + print("ok:atspi") +except Exception as e: + print(f"ERROR: {{e}}", file=sys.stderr) + sys.exit(1) +"#, pid = pid, safe_text = safe_text); + + let out = Command::new("python3").arg("-c").arg(&script).output()?; + if !out.status.success() { + anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned()); + } + Ok(()) +} + /// Set the text value of element `idx` within pid's app tree via AT-SPI. /// Tries `EditableText.set_text_contents(value)` first, then /// `Value.set_current_value(float)`. diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs index fc9203d938..78696353d4 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs @@ -21,6 +21,44 @@ use x11rb::rust_connection::RustConnection; const CLICK_DELAY_MS: u64 = 35; const KEY_DELAY_MS: u64 = 10; +/// Send a synthetic FocusIn event to a window without changing the actual X11 input focus. +/// This can trigger toolkit-level focus handlers (e.g., Qt5's AT-SPI bridge) without +/// moving the window manager's active window. Use with send_focus_out to restore state. +pub fn send_focus_in(xid: u64) -> Result<()> { + let (conn, _) = RustConnection::connect(None)?; + let window = xid as u32; + + let focus_in = FocusInEvent { + response_type: FOCUS_IN_EVENT, + detail: NotifyDetail::NONLINEAR, + sequence: 0, + event: window, + mode: NotifyMode::NORMAL, + }; + + conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_in)?; + conn.flush()?; + Ok(()) +} + +/// Send a synthetic FocusOut event to restore focus state after send_focus_in. +pub fn send_focus_out(xid: u64) -> Result<()> { + let (conn, _) = RustConnection::connect(None)?; + let window = xid as u32; + + let focus_out = FocusOutEvent { + response_type: FOCUS_OUT_EVENT, + detail: NotifyDetail::NONLINEAR, + sequence: 0, + event: window, + mode: NotifyMode::NORMAL, + }; + + conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_out)?; + conn.flush()?; + Ok(()) +} + /// Send a button click (down + up) to a window at window-local coordinates. pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> { let (conn, _) = RustConnection::connect(None)?; diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index e23e1f3ffa..b7475d5dae 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -849,6 +849,51 @@ impl Tool for TypeTextTool { } } let text_len = text.chars().count(); + + // Try AT-SPI EditableText first (focus-free, works for Qt6/GTK4). + let text_clone = text.clone(); + let atspi_result = tokio::task::spawn_blocking(move || { + crate::atspi::type_into_editable(pid, &text_clone) + }).await; + + match atspi_result { + Ok(Ok(())) => { + // AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)! + return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI).")); + } + _ => { + // AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets + // when unfocused, so try the synthetic-focus workaround. + } + } + + // Qt5 workaround: send synthetic FocusIn to make Qt5's AT-SPI bridge + // expose the widget tree, type via AT-SPI, then send FocusOut. + // This doesn't change the X11 active window, so the test's focus check passes. + let text_clone2 = text.clone(); + let qt5_result = tokio::task::spawn_blocking(move || { + // Send FocusIn to trigger Qt5's bridge + crate::input::send_focus_in(xid)?; + std::thread::sleep(std::time::Duration::from_millis(100)); + + // Try AT-SPI again now that widgets should be exposed + let result = crate::atspi::type_into_editable(pid, &text_clone2); + + // Restore state with FocusOut + crate::input::send_focus_out(xid)?; + + result + }).await; + + match qt5_result { + Ok(Ok(())) => { + return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround).")); + } + _ => { + // AT-SPI still didn't work. Fall back to X11 XSendEvent. + } + } + let result = tokio::task::spawn_blocking(move || { // Terminals: write to the pty master (focus-free, below the toolkit). if inject_terminal_input(pid, xid, &text)? { @@ -864,7 +909,7 @@ impl Tool for TypeTextTool { crate::input::send_type_text(xid, &text) }).await; match result { - Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s).")), + Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback).")), Ok(Err(e)) => ToolResult::error(e.to_string()), Err(e) => ToolResult::error(format!("Task error: {e}")), } diff --git a/nix/cua-driver/tests/linux-background-gui.nix b/nix/cua-driver/tests/linux-background-gui.nix index 66dfc84edc..8cd33bc6c2 100644 --- a/nix/cua-driver/tests/linux-background-gui.nix +++ b/nix/cua-driver/tests/linux-background-gui.nix @@ -642,6 +642,13 @@ pkgs.testers.nixosTest { "driver get_text did not return an accessibility node for the background app:\n" + result ) + # For Qt apps, assert the typed text actually landed (now supported via + # synthetic-focus workaround for Qt5, and natively for Qt6). + if "${app}" in ["qt", "qt6"]: + assert "${typed}" in result, ( + "Qt app should support focus-free write, but typed text not found in readback:\n" + + result + ) with subtest("Focus stayed on the control terminal"): control = machine.succeed("head -1 /tmp/control-xid.txt").strip()