Skip to content

feat(cua-driver-rs)(macos): expose get_window_state element bounds #1820

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cyq1017 wants to merge 2 commits into from
Closed

feat(cua-driver-rs)(macos): expose get_window_state element bounds #1820

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d1e98cc
feat(cua-driver-rs): expose macOS element bounds
cyq1017 Jun 3, 2026
b6d5e5e
fix(cua-driver-rs): clip element bounds to window
cyq1017 Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,9 @@ pub struct ElementCache {

impl ElementCache {
pub fn new() -> Self {
Self { core: ElementCacheCore::new() }
Self {
core: ElementCacheCore::new(),
}
}

/// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk.
Expand All @@ -93,7 +95,8 @@ impl ElementCache {
.filter(|n| n.element_index.is_some())
.map(|n| n.element_ptr)
.collect();
self.core.insert(CacheKey { pid, window_id }, CachedSnapshot { elements });
self.core
.insert(CacheKey { pid, window_id }, CachedSnapshot { elements });
}

/// Look up + `CFRetain` the element for `element_index` in (pid, window_id),
Expand Down Expand Up @@ -155,6 +158,7 @@ mod tests {
identifier: None,
help: None,
actions: Vec::new(),
screen_rect: None,
element_ptr: ptr,
}
}
Expand All @@ -176,11 +180,21 @@ mod tests {
unsafe { CFRetain(ptr as CFTypeRef) };
let cache = ElementCache::new();
cache.update(1, 2, &[node_with_ptr(ptr)]);
assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1, "cache owns one retain");
assert_eq!(
unsafe { CFGetRetainCount(ptr as CFTypeRef) },
base + 1,
"cache owns one retain"
);

// Borrow the element out for an action.
let guard = cache.get_element_retained(1, 2, 0).expect("element is cached");
assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 2, "guard adds a retain");
let guard = cache
.get_element_retained(1, 2, 0)
.expect("element is cached");
assert_eq!(
unsafe { CFGetRetainCount(ptr as CFTypeRef) },
base + 2,
"guard adds a retain"
);

// Concurrent get_window_state replaces the snapshot → old one dropped →
// CFRelease of the cache's retain. The guard's retain must remain.
Expand All @@ -193,7 +207,11 @@ mod tests {
);

drop(guard);
assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base, "guard drop releases its retain");
assert_eq!(
unsafe { CFGetRetainCount(ptr as CFTypeRef) },
base,
"guard drop releases its retain"
);
}

/// A missing index returns None without retaining anything.
Expand Down
151 changes: 119 additions & 32 deletions libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ pub struct AXNode {
pub identifier: Option<String>,
pub help: Option<String>,
pub actions: Vec<String>,
/// Element frame in screen points: [x, y, width, height].
pub screen_rect: Option<[f64; 4]>,
/// The raw AXUIElementRef pointer value, for caching.
pub element_ptr: usize,
}
Expand Down Expand Up @@ -98,7 +100,11 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
unsafe {
let app_elem = AXUIElementCreateApplication(pid);
if app_elem.is_null() {
return TreeWalkResult { tree_markdown: String::new(), nodes, truncated: false };
return TreeWalkResult {
tree_markdown: String::new(),
nodes,
truncated: false,
};
}

// Chromium/Electron apps (Arc, VS Code, Electron shells) ship their
Expand All @@ -110,7 +116,10 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
// read it. Native Cocoa apps reject the attribute, so they pay no
// settle cost. This relies on the MAX_ELEMENTS node cap to keep the
// now-materialized (potentially large) tree bounded.
let already_enabled = enabled_pids().lock().map(|s| s.contains(&pid)).unwrap_or(false);
let already_enabled = enabled_pids()
.lock()
.map(|s| s.contains(&pid))
.unwrap_or(false);
if !already_enabled && enable_chromium_accessibility(app_elem) {
crate::permissions::panel::pump_run_loop_briefly(CHROMIUM_SETTLE_SECONDS);
if let Ok(mut set) = enabled_pids().lock() {
Expand All @@ -137,21 +146,33 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW

// Filter: keep non-window children (menu bar) + the target window.
let walk_these: Vec<AXUIElementRef> = if let Some(wid) = window_id {
top_level.iter().copied().filter(|&child| {
let role = copy_string_attr(child, "AXRole").unwrap_or_default();
if role != "AXWindow" {
return true; // always keep menu bar and other non-window items
}
// Match AX window element → CGWindowID via private SPI.
ax_get_window_id(child) == Some(wid)
}).collect()
top_level
.iter()
.copied()
.filter(|&child| {
let role = copy_string_attr(child, "AXRole").unwrap_or_default();
if role != "AXWindow" {
return true; // always keep menu bar and other non-window items
}
// Match AX window element → CGWindowID via private SPI.
ax_get_window_id(child) == Some(wid)
})
.collect()
} else {
top_level.iter().copied().collect()
};

// Walk each top-level child at depth 0.
for child in walk_these {
walk_element(child, 0, &mut nodes, &mut lines, &mut index_counter, &mut visited_count, &mut truncated);
walk_element(
child,
0,
&mut nodes,
&mut lines,
&mut index_counter,
&mut visited_count,
&mut truncated,
);
}

// Release all top-level elements (copy_children / copy_ax_windows both retain).
Expand Down Expand Up @@ -179,7 +200,11 @@ pub fn walk_tree(pid: i32, window_id: Option<u32>, query: Option<&str>) -> TreeW
));
}

TreeWalkResult { tree_markdown, nodes, truncated: truncated_flag }
TreeWalkResult {
tree_markdown,
nodes,
truncated: truncated_flag,
}
}

unsafe fn walk_element(
Expand All @@ -191,7 +216,9 @@ unsafe fn walk_element(
visited_count: &mut usize,
truncated: &mut bool,
) {
if depth > MAX_DEPTH { return; }
if depth > MAX_DEPTH {
return;
}
// Enforce total-node cap — mirrors Swift's maxElements guard.
// Set the truncated flag only when we actually stop early.
if *visited_count >= MAX_ELEMENTS {
Expand All @@ -200,15 +227,22 @@ unsafe fn walk_element(
}
*visited_count += 1;

let role = copy_string_attr(element, "AXRole")
.unwrap_or_else(|| "AXUnknown".into());
let role = copy_string_attr(element, "AXRole").unwrap_or_else(|| "AXUnknown".into());

// Skip pure layout containers that have no interesting content.
if role == "AXScrollArea" || role == "AXGroup" {
// Still recurse — children may be interesting.
let children = copy_children(element);
for child in children {
walk_element(child, depth, nodes, lines, counter, visited_count, truncated);
walk_element(
child,
depth,
nodes,
lines,
counter,
visited_count,
truncated,
);
CFRelease(child as CFTypeRef);
}
return;
Expand All @@ -222,7 +256,8 @@ unsafe fn walk_element(
let title = copy_string_attr(element, "AXTitle");
let value = copy_string_attr(element, "AXValue");
// AXPlaceholderValue as fallback for empty text fields.
let value = value.filter(|v| !v.trim().is_empty())
let value = value
.filter(|v| !v.trim().is_empty())
.or_else(|| copy_string_attr(element, "AXPlaceholderValue"));
let description = copy_string_attr(element, "AXDescription");
let identifier = copy_string_attr(element, "AXIdentifier");
Expand All @@ -233,15 +268,22 @@ unsafe fn walk_element(
let visible_description = description.as_deref().unwrap_or("").trim().to_owned();
let visible_value = value.as_deref().unwrap_or("").trim().to_owned();

let has_content = !visible_title.is_empty()
|| !visible_description.is_empty()
|| !visible_value.is_empty();
let has_content =
!visible_title.is_empty() || !visible_description.is_empty() || !visible_value.is_empty();
let is_actionable = !actions.is_empty();

if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" {
let children = copy_children(element);
for child in children {
walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated);
walk_element(
child,
depth + 1,
nodes,
lines,
counter,
visited_count,
truncated,
);
CFRelease(child as CFTypeRef);
}
return;
Expand All @@ -251,30 +293,57 @@ unsafe fn walk_element(
let node = if is_actionable {
let idx = *counter;
*counter += 1;
let screen_rect = element_screen_rect(element);
// Retain so the element stays alive in the cache after `copy_children`
// releases the per-child ref at the end of the caller's loop.
CFRetain(element as CFTypeRef);
AXNode {
element_index: Some(idx),
role: role.clone(),
title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) },
value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) },
description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) },
title: if visible_title.is_empty() {
None
} else {
Some(visible_title.clone())
},
value: if visible_value.is_empty() {
None
} else {
Some(visible_value.clone())
},
description: if visible_description.is_empty() {
None
} else {
Some(visible_description.clone())
},
identifier: identifier.clone(),
help: help.clone(),
actions: actions.clone(),
screen_rect,
element_ptr,
}
} else {
AXNode {
element_index: None,
role: role.clone(),
title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) },
value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) },
description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) },
title: if visible_title.is_empty() {
None
} else {
Some(visible_title.clone())
},
value: if visible_value.is_empty() {
None
} else {
Some(visible_value.clone())
},
description: if visible_description.is_empty() {
None
} else {
Some(visible_description.clone())
},
identifier: identifier.clone(),
help: help.clone(),
actions: vec![],
screen_rect: None,
element_ptr,
}
};
Expand All @@ -285,7 +354,15 @@ unsafe fn walk_element(

let children = copy_children(element);
for child in children {
walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated);
walk_element(
child,
depth + 1,
nodes,
lines,
counter,
visited_count,
truncated,
);
CFRelease(child as CFTypeRef);
}
}
Expand Down Expand Up @@ -324,7 +401,9 @@ fn format_node_line(node: &AXNode) -> String {
attrs.push(format!("help=\"{}\"", h));
}
if !node.actions.is_empty() {
let action_str = node.actions.iter()
let action_str = node
.actions
.iter()
.map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase())
.collect::<Vec<_>>()
.join(",");
Expand Down Expand Up @@ -376,8 +455,12 @@ fn filter_tree(markdown: &str, query: &str) -> String {
if line.to_lowercase().contains(&needle) {
for ancestor_depth in 0..depth {
let ancestor = current_ancestor[ancestor_depth];
if ancestor.is_empty() { continue; }
if last_emitted_at[ancestor_depth] == Some(ancestor) { continue; }
if ancestor.is_empty() {
continue;
}
if last_emitted_at[ancestor_depth] == Some(ancestor) {
continue;
}
last_emitted_at[ancestor_depth] = Some(ancestor);
output.push(ancestor);
}
Expand All @@ -397,7 +480,11 @@ fn filter_tree(markdown: &str, query: &str) -> String {
fn leading_indent_depth(line: &str) -> usize {
let mut count = 0;
for ch in line.chars() {
if ch == ' ' { count += 1; } else { break; }
if ch == ' ' {
count += 1;
} else {
break;
}
}
count / 2
}
Loading