Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 119 additions & 2 deletions libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ use windows::Win32::UI::Accessibility::{
UIA_AutomationIdPropertyId, UIA_BoundingRectanglePropertyId,
UIA_ControlTypePropertyId, UIA_HelpTextPropertyId,
UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_NamePropertyId,
UIA_ValueValuePropertyId, UIA_InvokePatternId, UIA_SelectionItemPatternId,
UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId,
UIA_InvokePatternId, UIA_SelectionItemPatternId,
UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId,
UIA_ValuePatternId, UIA_ScrollPatternId,
TreeScope_Subtree,
TreeScope_Children, TreeScope_Subtree,
};

pub mod cache;
Expand Down Expand Up @@ -128,6 +129,53 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {

walk_cached(&root_elem, 0, &mut nodes, &mut lines, &mut counter, &mut total);

// Fallback for CoreWindow-class apps (Calculator, Settings, older UWPs).
// `ElementFromHandle(hwnd)` on a `Windows.UI.Core.CoreWindow` HWND returns
// an empty wrapper — the actual XAML tree is registered at the desktop
// root as a separate UIA element with the same ProcessId, not as a child
// of the hwnd's UIA element. inspect.exe walks from root for these apps;
// we mirror that here when the primary path yields nothing actionable.
//
// Trigger: walk produced zero actionable nodes (the primary path may have
// still pushed a wrapper-only node — that's why we filter on
// `element_index.is_some()`).
//
// Stage the fallback walk into fresh accumulators and only swap them in
// if the fallback actually finds actionable elements. Otherwise the
// wrapper-only node from the primary walk stays the result — better than
// erasing it AND leaving the consumed `MAX_TOTAL_ELEMENTS` budget intact
// for the fallback (which would then truncate large trees prematurely).
if nodes.iter().filter(|n| n.element_index.is_some()).count() == 0 {
if let Some(target_pid) = pid_from_hwnd(hwnd_win) {
let mut fallback_nodes: Vec<UiaNode> = Vec::new();
let mut fallback_lines: Vec<(usize, String)> = Vec::new();
let mut fallback_counter = 0usize;
let mut fallback_total = 0usize;

tracing::debug!(
target: "uia",
"ElementFromHandle returned empty tree for hwnd 0x{hwnd:x}; \
falling back to GetRootElement + filter ProcessId={target_pid}"
);
walk_root_by_pid(
&automation,
&cache_req,
target_pid,
&mut fallback_nodes,
&mut fallback_lines,
&mut fallback_counter,
&mut fallback_total,
);
Comment on lines +148 to +168

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Stage the PID fallback into fresh buffers before swapping it in.

Line 142 reuses nodes, lines, counter, and total from the failed HWND walk. If the primary pass emitted a wrapper-only node, the final tree now contains both that stale root and the PID-based subtree, and any consumed MAX_TOTAL_ELEMENTS budget also carries into the fallback. Run walk_root_by_pid into temporary accumulators and replace the primary result only if the fallback actually finds actionable nodes.

💡 Suggested shape
     if nodes.iter().filter(|n| n.element_index.is_some()).count() == 0 {
         if let Some(target_pid) = pid_from_hwnd(hwnd_win) {
+            let mut fallback_nodes = Vec::new();
+            let mut fallback_lines = Vec::new();
+            let mut fallback_counter = 0usize;
+            let mut fallback_total = 0usize;
+
             tracing::debug!(
                 target: "uia",
                 "ElementFromHandle returned empty tree for hwnd 0x{hwnd:x}; \
                  falling back to GetRootElement + filter ProcessId={target_pid}"
             );
             walk_root_by_pid(
                 &automation,
                 &cache_req,
                 target_pid,
-                &mut nodes,
-                &mut lines,
-                &mut counter,
-                &mut total,
+                &mut fallback_nodes,
+                &mut fallback_lines,
+                &mut fallback_counter,
+                &mut fallback_total,
             );
+
+            if fallback_nodes.iter().any(|n| n.element_index.is_some()) {
+                nodes = fallback_nodes;
+                lines = fallback_lines;
+                counter = fallback_counter;
+                total = fallback_total;
+            }
         }
     }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
if nodes.iter().filter(|n| n.element_index.is_some()).count() == 0 {
if let Some(target_pid) = pid_from_hwnd(hwnd_win) {
tracing::debug!(
target: "uia",
"ElementFromHandle returned empty tree for hwnd 0x{hwnd:x}; \
falling back to GetRootElement + filter ProcessId={target_pid}"
);
walk_root_by_pid(
&automation,
&cache_req,
target_pid,
&mut nodes,
&mut lines,
&mut counter,
&mut total,
);
if nodes.iter().filter(|n| n.element_index.is_some()).count() == 0 {
if let Some(target_pid) = pid_from_hwnd(hwnd_win) {
let mut fallback_nodes = Vec::new();
let mut fallback_lines = Vec::new();
let mut fallback_counter = 0usize;
let mut fallback_total = 0usize;
tracing::debug!(
target: "uia",
"ElementFromHandle returned empty tree for hwnd 0x{hwnd:x}; \
falling back to GetRootElement + filter ProcessId={target_pid}"
);
walk_root_by_pid(
&automation,
&cache_req,
target_pid,
&mut fallback_nodes,
&mut fallback_lines,
&mut fallback_counter,
&mut fallback_total,
);
if fallback_nodes.iter().any(|n| n.element_index.is_some()) {
nodes = fallback_nodes;
lines = fallback_lines;
counter = fallback_counter;
total = fallback_total;
}
}
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@libs/cua-driver-rs/crates/platform-windows/src/uia/mod.rs` around lines 142 -
157, The HWND fallback currently calls walk_root_by_pid reusing nodes, lines,
counter, and total, which can merge stale wrapper-only results and preserve
consumed budget; instead, create temporary accumulators (e.g., temp_nodes,
temp_lines, temp_counter, temp_total), call walk_root_by_pid(&automation,
&cache_req, target_pid, &mut temp_nodes, &mut temp_lines, &mut temp_counter,
&mut temp_total), then check the temp result for any actionable nodes (e.g., any
node.element_index.is_some()) and only swap temp_* into the original nodes,
lines, counter, total if the fallback produced actionable entries; otherwise
leave the original buffers and budget untouched.


if fallback_nodes.iter().any(|n| n.element_index.is_some()) {
nodes = fallback_nodes;
lines = fallback_lines;
counter = fallback_counter;
total = fallback_total;
}
}
}

let raw_md = render_lines(&lines);
let tree_markdown = if let Some(q) = query {
filter_tree(&raw_md, q)
Expand All @@ -138,6 +186,75 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult {
UiaTreeResult { tree_markdown, nodes }
}

/// Resolve the owning process id of `hwnd` via `GetWindowThreadProcessId`.
/// Returns `None` if the API fails or the HWND is invalid.
unsafe fn pid_from_hwnd(hwnd: windows::Win32::Foundation::HWND) -> Option<u32> {
use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId;
let mut pid: u32 = 0;
let tid = GetWindowThreadProcessId(hwnd, Some(&mut pid));
if tid == 0 || pid == 0 { None } else { Some(pid) }
}

/// Root-walk UIA fallback for apps whose top-level window is a
/// `Windows.UI.Core.CoreWindow` (Calculator, Settings, older UWPs).
///
/// `ElementFromHandle(CoreWindow_hwnd)` returns an empty wrapper for these —
/// the actual XAML tree is registered at the desktop root as a sibling
/// element with the same `ProcessId`. We enumerate the root's children
/// (filtered by `ProcessId == target_pid`) and walk descendants from there,
/// reusing the caller's `cache_req` so the same properties + patterns get
/// pre-fetched as the primary path.
unsafe fn walk_root_by_pid(
automation: &IUIAutomation,
cache_req: &IUIAutomationCacheRequest,
target_pid: u32,
nodes: &mut Vec<UiaNode>,
lines: &mut Vec<(usize, String)>,
counter: &mut usize,
total: &mut usize,
) {
let root = match automation.GetRootElement() {
Ok(r) => r,
Err(e) => { tracing::debug!(target: "uia", "GetRootElement failed: {e}"); return; }
};
let true_cond = match automation.CreateTrueCondition() {
Ok(c) => c,
Err(e) => { tracing::debug!(target: "uia", "CreateTrueCondition failed: {e}"); return; }
};
let kids = match root.FindAll(TreeScope_Children, &true_cond) {
Ok(a) => a,
Err(e) => { tracing::debug!(target: "uia", "root.FindAll(Children) failed: {e}"); return; }
};
let count = kids.Length().unwrap_or(0);
for i in 0..count {
let elem = match kids.GetElement(i) { Ok(e) => e, Err(_) => continue };
// Read ProcessId without a cache — root.FindAll didn't use one.
// VARIANT for VT_I4 (UIA's ProcessId type) puts the int at
// Anonymous.Anonymous.Anonymous.lVal; mirrors the read_cached_bool
// pattern below for VT_BOOL.
let pid: u32 = match elem.GetCurrentPropertyValue(UIA_ProcessIdPropertyId) {
Ok(v) => {
let raw = v.as_raw();
if raw.Anonymous.Anonymous.vt != 3 /* VT_I4 */ { continue; }
raw.Anonymous.Anonymous.Anonymous.lVal as u32
}
Err(_) => continue,
};
if pid != target_pid { continue; }
// Match — pull a cached subtree from this element using the same
// cache_req shape as the primary path so walk_cached sees the same
// properties + patterns.
let cached = match elem.BuildUpdatedCache(cache_req) {
Ok(e) => e,
Err(e) => {
tracing::debug!(target: "uia", "BuildUpdatedCache on pid={target_pid} match failed: {e}");
continue;
}
};
walk_cached(&cached, 0, nodes, lines, counter, total);
}
}

unsafe fn walk_cached(
element: &IUIAutomationElement,
depth: usize,
Expand Down
Loading