Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions libs/cua-driver/docs/tool-output-format.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,11 @@ Observation tools retain their typed tool-specific structured payloads.
records in `structuredContent` and can attach a PNG as image content. A
multimodal harness interprets the image; Cua Driver does not OCR it or assign
task meaning.

On Windows, `get_window_state.elements_complete` is true only when the
unprojected UI Automation walk visited every exposed node without a traversal
bound or enumeration failure skipping a node or subtree. Reaching
`max_elements` or `max_depth` exactly at the end of a tree remains complete.
`query` projects the returned Markdown and structured element rows, but does
not make the underlying snapshot incomplete; compare `total_element_count`
with `returned_element_count` to measure that projection.
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,32 @@ fn harness_wpf_query_projects_structured_elements() {
returned < total,
"query did not compact {returned}/{total} elements"
);
assert_eq!(
response.structured()["elements_complete"].as_bool(),
Some(true),
"query projection must preserve the complete underlying UIA snapshot"
);
let _ = element_token_by_id(&response, "btn-increment");

let bounded = driver.call(
"get_window_state",
serde_json::json!({
"pid": pid as i64,
"window_id": wid,
"max_elements": 1,
"include_screenshot": false
}),
);
assert!(
!bounded.is_error(),
"bounded snapshot failed: {}",
bounded.text()
);
assert_eq!(
bounded.structured()["elements_complete"].as_bool(),
Some(false),
"a bound that skips the WPF root's descendants must report an incomplete snapshot"
);
Observation::delivered(vec![OracleKind::AxState], Evidence::default())
},
);
Expand Down
5 changes: 5 additions & 0 deletions libs/cua-driver/rust/crates/platform-windows/src/msaa.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ unsafe fn walk_unsafe(hwnd: u64) -> UiaTreeResult {
"- Window <SAL/VCL — MSAA fallback failed (AccessibleObjectFromWindow hr={hr:?})>\n"
),
nodes: Vec::new(),
elements_complete: false,
};
}
let root: IAccessible = IAccessible::from_raw(raw_root);
Expand All @@ -110,6 +111,10 @@ unsafe fn walk_unsafe(hwnd: u64) -> UiaTreeResult {
UiaTreeResult {
tree_markdown,
nodes,
// This compatibility path cannot prove the complete UIA search
// domain (and intentionally omits UIA-only state), even when its own
// bounded MSAA recursion reaches every exposed child.
elements_complete: false,
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1102,7 +1102,9 @@ impl Tool for GetWindowStateTool {
Set `query` to a case-insensitive substring to project BOTH `tree_markdown` \
and `structuredContent.elements` to matching rows plus their ancestor chain. \
Original element indices are preserved. `total_element_count` reports the \
complete snapshot; `returned_element_count` reports the projection.\n\n\
unprojected walked snapshot; `returned_element_count` reports the projection. \
`elements_complete` describes the underlying walk and is unaffected by query \
projection.\n\n\
Always returns BOTH the element tree AND a screenshot — ground on both \
and cross-check (the tree lies on some surfaces). Choose the modality at \
ACTION time: an element ax action (element_index/element_token → \
Expand All @@ -1114,8 +1116,9 @@ impl Tool for GetWindowStateTool {
Optional `max_elements` / `max_depth` bound the UIA walk to mitigate \
context-window blow-up on Electron / large web apps that produce 10k+ \
element trees. When applied, BOTH the markdown and the structured \
elements are truncated identically. Omit both for current default behaviour \
(≤5 000 elements, depth ≤25).\n\n\
elements are truncated identically. `elements_complete` is false when a bound \
actually skips a node or subtree; ending exactly at a bound remains complete. \
Omit both for current default behaviour (≤5 000 elements, depth ≤25).\n\n\
CHROMIUM COVERAGE: a browser-owned permission bubble can be \
composited outside the requested native window. Chromium-family \
snapshots therefore describe this limit in structuredContent.capture_coverage. \
Expand All @@ -1132,7 +1135,7 @@ impl Tool for GetWindowStateTool {
"capture_mode": cua_driver_core::capture_mode::capture_mode_schema(),
"include_screenshot":{"type":"boolean","description":"Default true — returns a grounding screenshot alongside the tree. Set false to skip the grab and return tree only (the cheap path for re-indexing before an element ax action)."},
"screenshot_out_file":{"type":"string","description":"When set, write the PNG to this file path instead of embedding base64 in the response. The structured output will contain `screenshot_file_path` instead."},
"query":{"type":"string","description":"Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count."},
"query":{"type":"string","description":"Optional case-insensitive substring. Projects both tree_markdown and structured elements to matches plus ancestors while preserving original indices. Compare total_element_count with returned_element_count; elements_complete continues to describe the underlying unprojected walk."},
"max_elements":{"type":"integer","minimum":1,"description":"Cap on the total number of UIA nodes walked. Truncates depth-first; markdown and structured elements truncate together. Omit for the default (5 000). Lower for Electron / large web apps that produce 10k+ element trees."},
"max_depth":{"type":"integer","minimum":1,"description":"Cap on the UIA-tree walk depth. Nodes whose rendered indent would exceed this are omitted. Omit for the default (25). Lower for deep menu / Electron trees."}
},"additionalProperties":false}),
Expand Down Expand Up @@ -1322,10 +1325,7 @@ impl Tool for GetWindowStateTool {
}
}
structured["element_count"] = json!(count);
// UIA currently does not expose whether a bounded walk
// exhausted every subtree. Keep negative existence
// conservative until that proof is available.
structured["elements_complete"] = json!(false);
structured["elements_complete"] = json!(tr.elements_complete);
structured["tree_markdown"] = json!(tr.tree_markdown);

// Surface 6: register a snapshot in the global token
Expand Down
Loading
Loading