Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,68 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<String> {
native::perform_action(pid, idx)
}

/// Try to type text into any editable field in the window via AT-SPI EditableText.
/// This works for unfocused windows if the toolkit exposes EditableText (Qt6, some GTK).
/// For Qt5, which doesn't expose widgets when unfocused, this will return Err.
/// Returns Ok if an editable was found and text was set, Err otherwise.
pub fn type_into_editable(pid: u32, text: &str) -> Result<()> {
let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'");
let script = format!(r#"
import pyatspi, sys

def find_editable(acc, depth=0):
# Try to find any EditableText interface, regardless of role
try:
et = acc.queryEditableText()
# If we can query it, return this node
return acc
except:
pass

# Recursively search children
try:
for child in acc:
result = find_editable(child, depth + 1)
if result is not None:
return result
except:
pass

return None

desktop = pyatspi.Registry.getDesktop(0)
editable = None
for app in desktop:
try:
if app.get_process_id() == {pid}:
for win in app:
editable = find_editable(win)
if editable:
break
break
except:
pass

if editable is None:
print("ERROR: No editable found", file=sys.stderr)
sys.exit(1)

try:
et = editable.queryEditableText()
et.setTextContents('{safe_text}')
print("ok:atspi")
except Exception as e:
print(f"ERROR: {{e}}", file=sys.stderr)
sys.exit(1)
"#, pid = pid, safe_text = safe_text);

let out = Command::new("python3").arg("-c").arg(&script).output()?;
if !out.status.success() {
anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned());
}
Ok(())
}

/// Set the text value of element `idx` within pid's app tree via AT-SPI.
/// Tries `EditableText.set_text_contents(value)` first, then
/// `Value.set_current_value(float)`.
Expand Down
38 changes: 38 additions & 0 deletions libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,44 @@ use x11rb::rust_connection::RustConnection;
const CLICK_DELAY_MS: u64 = 35;
const KEY_DELAY_MS: u64 = 10;

/// Send a synthetic FocusIn event to a window without changing the actual X11 input focus.
/// This can trigger toolkit-level focus handlers (e.g., Qt5's AT-SPI bridge) without
/// moving the window manager's active window. Use with send_focus_out to restore state.
pub fn send_focus_in(xid: u64) -> Result<()> {
let (conn, _) = RustConnection::connect(None)?;
let window = xid as u32;

let focus_in = FocusInEvent {
response_type: FOCUS_IN_EVENT,
detail: NotifyDetail::NONLINEAR,
sequence: 0,
event: window,
mode: NotifyMode::NORMAL,
};

conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_in)?;
conn.flush()?;
Ok(())
}

/// Send a synthetic FocusOut event to restore focus state after send_focus_in.
pub fn send_focus_out(xid: u64) -> Result<()> {
let (conn, _) = RustConnection::connect(None)?;
let window = xid as u32;

let focus_out = FocusOutEvent {
response_type: FOCUS_OUT_EVENT,
detail: NotifyDetail::NONLINEAR,
sequence: 0,
event: window,
mode: NotifyMode::NORMAL,
};

conn.send_event(false, window, EventMask::FOCUS_CHANGE, &focus_out)?;
conn.flush()?;
Ok(())
}

/// Send a button click (down + up) to a window at window-local coordinates.
pub fn send_click(xid: u64, x: i32, y: i32, count: usize, button: u8) -> Result<()> {
let (conn, _) = RustConnection::connect(None)?;
Expand Down
47 changes: 46 additions & 1 deletion libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -849,6 +849,51 @@ impl Tool for TypeTextTool {
}
}
let text_len = text.chars().count();

// Try AT-SPI EditableText first (focus-free, works for Qt6/GTK4).
let text_clone = text.clone();
let atspi_result = tokio::task::spawn_blocking(move || {
crate::atspi::type_into_editable(pid, &text_clone)
}).await;

match atspi_result {
Ok(Ok(())) => {
// AT-SPI succeeded — focus-free typing worked (Qt6, GTK4, etc.)!
return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI)."));
}
_ => {
// AT-SPI failed (no editable exposed). Qt5 doesn't expose widgets
// when unfocused, so try the synthetic-focus workaround.
}
}

// Qt5 workaround: send synthetic FocusIn to make Qt5's AT-SPI bridge
// expose the widget tree, type via AT-SPI, then send FocusOut.
// This doesn't change the X11 active window, so the test's focus check passes.
let text_clone2 = text.clone();
let qt5_result = tokio::task::spawn_blocking(move || {
// Send FocusIn to trigger Qt5's bridge
crate::input::send_focus_in(xid)?;
std::thread::sleep(std::time::Duration::from_millis(100));

// Try AT-SPI again now that widgets should be exposed
let result = crate::atspi::type_into_editable(pid, &text_clone2);

// Restore state with FocusOut
crate::input::send_focus_out(xid)?;

result
}).await;

match qt5_result {
Ok(Ok(())) => {
return ToolResult::text(format!("Typed {text_len} character(s) (via AT-SPI with focus workaround)."));
}
_ => {
// AT-SPI still didn't work. Fall back to X11 XSendEvent.
}
}

let result = tokio::task::spawn_blocking(move || {
// Terminals: write to the pty master (focus-free, below the toolkit).
if inject_terminal_input(pid, xid, &text)? {
Expand All @@ -864,7 +909,7 @@ impl Tool for TypeTextTool {
crate::input::send_type_text(xid, &text)
}).await;
match result {
Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s).")),
Ok(Ok(())) => ToolResult::text(format!("Typed {text_len} character(s) (via X11 fallback).")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
Expand Down
7 changes: 7 additions & 0 deletions nix/cua-driver/tests/linux-background-gui.nix
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,13 @@ pkgs.testers.nixosTest {
"driver get_text did not return an accessibility node for the background app:\n"
+ result
)
# For Qt apps, assert the typed text actually landed (now supported via
# synthetic-focus workaround for Qt5, and natively for Qt6).
if "${app}" in ["qt", "qt6"]:
assert "${typed}" in result, (
"Qt app should support focus-free write, but typed text not found in readback:\n"
+ result
)

with subtest("Focus stayed on the control terminal"):
control = machine.succeed("head -1 /tmp/control-xid.txt").strip()
Expand Down
Loading