Skip to content

fix(cua-driver): honor disabled UIA ancestors #3263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ethanblake4 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(cua-driver): honor disabled UIA ancestors #3263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions libs/cua-driver/rust/crates/platform-windows/src/browser_consent_ui.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefus
BrowserRefusal::new(code, message)
}

fn release_nodes(nodes: &[UiaNode]) {
for node in nodes.iter().filter(|node| node.element_ptr != 0) {
unsafe { drop(IUIAutomationElement::from_raw(node.element_ptr as *mut _)) };
}
}

fn is_in_web_content(nodes: &[UiaNode], node: &UiaNode) -> bool {
let mut parent = node.parent_element_index;
for _ in 0..nodes.len() {
Expand Down Expand Up @@ -390,7 +384,6 @@ pub fn dismiss(pid: u32, hwnd: u64) -> Result<bool, BrowserRefusal> {
let tree = crate::uia::walk_tree(hwnd, None);
let prompt_present = native_prompt_surface_present(&tree.nodes);
if !prompt_present {
release_nodes(&tree.nodes);
return Ok(dismissed);
}
let cancel = exact_cancel_button(&tree.nodes);
Expand All @@ -402,7 +395,6 @@ pub fn dismiss(pid: u32, hwnd: u64) -> Result<bool, BrowserRefusal> {
)),
Err(error) => Err(error),
};
release_nodes(&tree.nodes);
invoked?;
dismissed = true;
if Instant::now() >= deadline {
Expand Down Expand Up @@ -443,15 +435,11 @@ pub async fn handle(
match exact_allow_button(&tree.nodes) {
Ok(Some(element)) => {
let invoked = unsafe { invoke(element) };
release_nodes(&tree.nodes);
invoked?;
return Ok(BrowserConsentOutcome::Accepted);
}
Ok(None) => release_nodes(&tree.nodes),
Err(error) => {
release_nodes(&tree.nodes);
return Err(error);
}
Ok(None) => {}
Err(error) => return Err(error),
}
if saw_prompt && !prompt_present {
return Err(refusal(
Expand Down
35 changes: 2 additions & 33 deletions libs/cua-driver/rust/crates/platform-windows/src/browser_setup_ui.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,6 @@ fn refusal(code: BrowserRefusalCode, message: impl Into<String>) -> BrowserRefus
BrowserRefusal::new(code, message)
}

fn release_nodes(nodes: &[UiaNode]) {
for node in nodes.iter().filter(|node| node.element_ptr != 0) {
unsafe { drop(IUIAutomationElement::from_raw(node.element_ptr as *mut _)) };
}
}

fn unique_web_actionable(
nodes: &[UiaNode],
control_type: &str,
Expand Down Expand Up @@ -178,7 +172,6 @@ fn stable_native_tab_count(hwnd: u64, initial_count: usize) -> Result<usize, Bro
std::thread::sleep(Duration::from_millis(100));
let tree = crate::uia::walk_tree(hwnd, None);
let current = native_tab_count(&tree.nodes);
release_nodes(&tree.nodes);
if current > 0 && current == previous {
return Ok(current);
}
Expand Down Expand Up @@ -393,7 +386,6 @@ impl SetupUiHandle {
},
_ => false,
};
release_nodes(&tree.nodes);
if restored {
self.enabled_remote_debugging = false;
}
Expand Down Expand Up @@ -430,7 +422,6 @@ impl SetupUiHandle {
}
let tree = crate::uia::walk_tree(self.hwnd, None);
let proven = setup_page_proven(&tree.nodes, self.descriptor);
release_nodes(&tree.nodes);
if !proven {
let error = refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
Expand All @@ -456,7 +447,6 @@ impl SetupUiHandle {
}
let tree = crate::uia::walk_tree(self.hwnd, None);
let proven = setup_page_proven(&tree.nodes, self.descriptor);
release_nodes(&tree.nodes);
Some(
proven
&& crate::input::keyboard::send_key_synthesized(self.hwnd, "w", &["ctrl"]).is_ok(),
Expand Down Expand Up @@ -525,7 +515,6 @@ fn set_remote_debugging(
},
Ok(None) => {
let initial_tab_count = native_tab_count(&initial.nodes);
release_nodes(&initial.nodes);
let tab_count_before = stable_native_tab_count(hwnd, initial_tab_count)?;

let mut handle = SetupUiHandle {
Expand All @@ -542,7 +531,6 @@ fn set_remote_debugging(
let new_tab_button = match exact_native_new_tab_button(&tab_tree.nodes) {
Ok(Some(element)) => element,
Ok(None) => {
release_nodes(&tab_tree.nodes);
return Err(handle.abort(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
format!(
Expand All @@ -552,15 +540,12 @@ fn set_remote_debugging(
)));
}
Err(error) => {
release_nodes(&tab_tree.nodes);
return Err(handle.abort(error));
}
};
if let Err(error) = unsafe { invoke(new_tab_button, "native new-tab button") } {
release_nodes(&tab_tree.nodes);
return Err(handle.abort(error));
}
release_nodes(&tab_tree.nodes);
handle.opened_setup_page = true;

let deadline = Instant::now() + Duration::from_secs(3);
Expand All @@ -570,7 +555,6 @@ fn set_remote_debugging(
if tab_count_after == tab_count_before + 1 {
break tree;
}
release_nodes(&tree.nodes);
if Instant::now() >= deadline {
return Err(handle.abort(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
Expand All @@ -585,7 +569,6 @@ fn set_remote_debugging(
let omnibox = match unique_native_actionable(&created.nodes, "Edit", "set_value") {
Ok(Some(element)) => element,
Ok(None) => {
release_nodes(&created.nodes);
return Err(handle.abort(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
format!(
Expand All @@ -595,15 +578,12 @@ fn set_remote_debugging(
)));
}
Err(error) => {
release_nodes(&created.nodes);
return Err(handle.abort(error));
}
};
if let Err(error) = unsafe { set_value(omnibox, descriptor.setup_url) } {
release_nodes(&created.nodes);
return Err(handle.abort(error));
}
release_nodes(&created.nodes);
created = crate::uia::walk_tree(hwnd, None);
let refreshed_omnibox =
match unique_native_actionable(&created.nodes, "Edit", "set_value") {
Expand All @@ -618,14 +598,12 @@ fn set_remote_debugging(
element
}
Ok(_) => {
release_nodes(&created.nodes);
return Err(handle.abort(refusal(
BrowserRefusalCode::BrowserWrongTargetRefused,
"the unique native address field did not retain the exact setup URL",
)));
}
Err(error) => {
release_nodes(&created.nodes);
return Err(handle.abort(error));
}
};
Expand All @@ -636,21 +614,14 @@ fn set_remote_debugging(
&mut handle.injected_global_input,
&mut handle.focused_setup_address_field,
) {
release_nodes(&created.nodes);
return Err(handle.abort(error));
}
release_nodes(&created.nodes);
handle
}
Err(error) => {
release_nodes(&initial.nodes);
return Err(error);
}
};
if !handle.opened_setup_page {
release_nodes(&initial.nodes);
}

let deadline = Instant::now() + EXISTING_PROFILE_SETUP_READY_TIMEOUT;
loop {
let tree = crate::uia::walk_tree(hwnd, None);
Expand All @@ -671,16 +642,14 @@ fn set_remote_debugging(
Ok(_) => Ok(false),
Err(error) => Err(error),
};
release_nodes(&tree.nodes);
match outcome {
Ok(true) => return Ok(handle),
Ok(false) => handle.enable_attempted = true,
Err(error) => return Err(handle.abort(error)),
}
}
Ok(None) => release_nodes(&tree.nodes),
Ok(None) => {}
Err(error) => {
release_nodes(&tree.nodes);
return Err(handle.abort(error));
}
}
Expand Down Expand Up @@ -732,7 +701,7 @@ mod tests {
actions: actions.iter().map(|value| (*value).to_owned()).collect(),
enabled: None,
selected: None,
element_ptr: 7,
element_ptr: if actions.is_empty() { 0 } else { 7 },
center_x: 0,
center_y: 0,
rect: None,
Expand Down
85 changes: 35 additions & 50 deletions libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,60 +170,45 @@ unsafe fn walk(
let has_content = name.is_some();

if is_actionable || has_content {
// Retain the IAccessible pointer for the cache. Mirror what the UIA
// walker does: clone, get raw, forget the local — the cache's Drop
// releases via IUnknown.
let retained: IAccessible = acc.clone();
let ptr = retained.as_raw() as usize;
std::mem::forget(retained);

let (center_x, center_y) = rect
.map(|(l, t, r, b)| ((l + r) / 2, (t + b) / 2))
.unwrap_or((0, 0));

let node = if is_actionable {
let element_index = if is_actionable {
let idx = *counter;
*counter += 1;
UiaNode {
element_index: Some(idx),
control_type: control_type.clone(),
name: name.clone(),
value: None,
automation_id: None,
help_text: None,
actions: actions.clone(),
enabled: None,
selected: None,
element_ptr: ptr,
center_x,
center_y,
rect,
msaa_role: role_int,
depth,
parent_element_index: parent_index,
in_web_content: false,
}
Some(idx)
} else {
UiaNode {
element_index: None,
control_type: control_type.clone(),
name: name.clone(),
value: None,
automation_id: None,
help_text: None,
actions: Vec::new(),
enabled: None,
selected: None,
element_ptr: ptr,
center_x: 0,
center_y: 0,
rect,
msaa_role: role_int,
depth,
parent_element_index: parent_index,
in_web_content: false,
}
None
};
let (element_ptr, center_x, center_y) = if element_index.is_some() {
// The cache adopts this reference when the walk becomes a snapshot.
let retained: IAccessible = acc.clone();
let ptr = retained.as_raw() as usize;
std::mem::forget(retained);
let (center_x, center_y) = rect
.map(|(l, t, r, b)| ((l + r) / 2, (t + b) / 2))
.unwrap_or((0, 0));
(ptr, center_x, center_y)
} else {
(0, 0, 0)
};
let node = UiaNode {
element_index,
control_type: control_type.clone(),
name: name.clone(),
value: None,
automation_id: None,
help_text: None,
actions: actions.clone(),
enabled: None,
selected: None,
element_ptr,
center_x,
center_y,
rect,
msaa_role: role_int,
depth,
parent_element_index: parent_index,
in_web_content: false,
};
debug_assert_eq!(node.element_ptr != 0, node.element_index.is_some());
// Track this node as the parent_index for its descendants only when
// it received an element_index (mirrors what the markdown shows:
// only indexed rows are addressable).
Expand Down
2 changes: 1 addition & 1 deletion libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Ve
"pid": pid,
"window_id": hwnd,
"element_count": element_count,
"tree_markdown": result.tree_markdown,
"tree_markdown": &result.tree_markdown,
});
serde_json::to_vec_pretty(&payload).ok()
}
Expand Down
10 changes: 3 additions & 7 deletions libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1299,7 +1299,7 @@ impl Tool for GetWindowStateTool {
let mut content = Vec::new();
let mut structured = json!({ "window_id": hwnd, "pid": pid });

if let Some(tr) = tree_opt {
if let Some(mut tr) = tree_opt {
let is_msaa = tr.nodes.iter().any(|n| n.msaa_role.is_some());
let count = tr
.nodes
Expand All @@ -1315,18 +1315,14 @@ impl Tool for GetWindowStateTool {
// walker, so the entire snapshot must Drop via
// IAccessible and click must dispatch through MSAA.
if !observation_only {
if is_msaa {
state.element_cache.update_msaa(pid, hwnd, &tr.nodes);
} else {
state.element_cache.update(pid, hwnd, &tr.nodes);
}
state.element_cache.adopt_walk(pid, hwnd, &mut tr);
}
structured["element_count"] = json!(count);
// UIA currently does not expose whether a bounded walk
// exhausted every subtree. Keep negative existence
// conservative until that proof is available.
structured["elements_complete"] = json!(false);
structured["tree_markdown"] = json!(tr.tree_markdown);
structured["tree_markdown"] = json!(&tr.tree_markdown);

// Surface 6: register a snapshot in the global token
// registry. Windows uses u64 HWND but the registry
Expand Down
Loading
Loading