fix(cua-driver): clarify Windows window-state coordinates

docs/content/docs/reference/cua-driver/limits.mdx

@@ -268,8 +268,11 @@ When a pixel action is unavoidable, match the scope to the space:
 </Callout>
 
 On macOS, `get_window_state` also reports `window_bounds`, which may be used in
-place of `list_windows` there. Windows and Linux report `screenshot_width` and
-`screenshot_height` only.
+place of `list_windows` there. Windows reports exact physical-pixel
+`window_bounds` as well as `element_frame_coordinate_space`,
+`pixel_action_coordinate_space`, and `pixel_action_to_screen`; use those fields
+instead of inferring a screenshot origin from a separate `list_windows` call.
+Linux reports `screenshot_width` and `screenshot_height` only.
 
 ---
 

libs/cua-driver/docs/tool-output-format.md

@@ -35,3 +35,19 @@ Observation tools retain their typed tool-specific structured payloads.
 records in `structuredContent` and can attach a PNG as image content. A
 multimodal harness interprets the image; Cua Driver does not OCR it or assign
 task meaning.
+
+On Windows, `get_window_state` explicitly distinguishes the two coordinate
+spaces in its structured payload:
+
+- `elements[].frame` is in `screen_physical_px`, as declared by
+  `element_frame_coordinate_space`. Frames wholly outside the exact target
+  `window_bounds` are omitted and carry
+  `frame_reliability:"outside_target_window"` instead.
+- Pixel-action `x,y` values are in `window_screenshot_px`, as declared by
+  `pixel_action_coordinate_space`. When a screenshot is present,
+  `pixel_action_to_screen` gives its screen origin and physical-screen pixels
+  per action pixel. Convert with
+  `screen = screen_origin + action * screen_pixels_per_action_pixel`.
+
+The accessibility elements remain one structured `elements` array; coordinate
+metadata does not add a second rendering of the tree.

libs/cua-driver/rust/crates/platform-windows/src/lib.rs

@@ -43,6 +43,11 @@ mod keycodes;
 // pass that lives in this commit).
 pub mod lparam;
 
+// Pure geometry and structured-output helpers for the Windows window-state
+// coordinate contract. Kept outside the Windows-only modules so unit tests can
+// validate the wire shape without a live HWND or interactive desktop.
+pub(crate) mod window_state_coordinates;
+
 #[cfg(target_os = "windows")]
 pub mod win32;
 

libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs

@@ -228,6 +228,44 @@ fn screen_to_bitmap(hwnd: u64, sx: i32, sy: i32) -> (i32, i32) {
     (sx - origin_x, sy - origin_y)
 }
 
+/// Read physical screen bounds only after revalidating the exact native
+/// `(pid, HWND)` identity. This closes the small race between the public
+/// ownership check and the blocking UIA/capture work, and ensures coordinate
+/// metadata never describes an HWND that was destroyed and reused.
+fn exact_target_window_bounds(
+    pid: u32,
+    hwnd: u64,
+) -> anyhow::Result<crate::window_state_coordinates::ScreenRect> {
+    use windows::Win32::Foundation::{HWND, RECT};
+    use windows::Win32::UI::WindowsAndMessaging::GetWindowRect;
+
+    let owner = crate::win32::window_owner_pid(hwnd)
+        .ok_or_else(|| anyhow::anyhow!("target window 0x{hwnd:x} no longer exists"))?;
+    if owner != pid {
+        anyhow::bail!(
+            "target window 0x{hwnd:x} now belongs to pid {owner}, not requested pid {pid}"
+        );
+    }
+    let mut rect = RECT::default();
+    unsafe { GetWindowRect(HWND(hwnd as *mut _), &mut rect) }
+        .map_err(|error| anyhow::anyhow!("could not read target window bounds: {error}"))?;
+    crate::window_state_coordinates::ScreenRect::from_edges(
+        rect.left,
+        rect.top,
+        rect.right,
+        rect.bottom,
+    )
+    .ok_or_else(|| {
+        anyhow::anyhow!(
+            "target window 0x{hwnd:x} has invalid bounds ({},{})-({},{})",
+            rect.left,
+            rect.top,
+            rect.right,
+            rect.bottom
+        )
+    })
+}
+
 /// Animate the agent cursor to (sx, sy) in screen coordinates and wait for the
 /// glide to finish before returning.  No-op when the overlay is not enabled.
 ///
@@ -1092,7 +1130,13 @@ impl Tool for GetWindowStateTool {
                 `selected`, `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \
                 `tree_markdown` stays available \
                 and unchanged in shape for existing text-parsing callers — but new \
-                fields will only be added to the structured side.\n\n\
+                fields will only be added to the structured side. On Windows, valid \
+                element `frame` values are screen-absolute physical pixels; a provider \
+                frame wholly outside the exact target window is omitted and marked with \
+                `frame_reliability:\"outside_target_window\"`. \
+                `element_frame_coordinate_space`, `pixel_action_coordinate_space`, \
+                `window_bounds`, and `pixel_action_to_screen` make the distinct spaces \
+                and conversion explicit without duplicating the element tree.\n\n\
                 The UIA tree walked is the window's tree (HWND-scoped); the screenshot and \
                 window bounds reported come from the same `window_id`. This is the source of \
                 truth for which window the caller intends to reason about — the driver never \
@@ -1224,6 +1268,7 @@ impl Tool for GetWindowStateTool {
         let q = query.clone();
         let out_file = screenshot_out_file.clone();
         let blocking = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
+            let window_bounds = exact_target_window_bounds(pid, hwnd)?;
             let tree_result = if do_tree {
                 Some(crate::uia::walk_tree_bounded(
                     hwnd,
@@ -1244,29 +1289,49 @@ impl Tool for GetWindowStateTool {
             let (screenshot, screenshot_err) = if do_shot {
                 match crate::capture::screenshot_window_bytes(hwnd) {
                     Ok(raw) => {
-                        let orig_w = crate::capture::png_dimensions_pub(&raw)
-                            .map(|(w, _)| w)
-                            .unwrap_or(0);
+                        let (orig_w, orig_h) = crate::capture::png_dimensions_pub(&raw)?;
                         let png = crate::capture::resize_png_if_needed(&raw, max_dim)?;
                         let (w, h) = crate::capture::png_dimensions_pub(&png)?;
-                        let original_w = if w < orig_w { Some(orig_w) } else { None };
+                        let original_dimensions =
+                            (w < orig_w || h < orig_h).then_some((orig_w, orig_h));
+                        let screenshot_origin = bitmap_to_screen(hwnd, 0, 0);
                         // `screenshot_out_file` set (any mode) → write to disk and
                         // surface the path, never embed bytes. Otherwise (vision,
                         // no out_file) → embed base64.
                         if let Some(ref path) = out_file {
                             std::fs::write(path, &png)?;
-                            (Some((None, Some(path.clone()), w, h, original_w)), None)
+                            (
+                                Some((
+                                    None,
+                                    Some(path.clone()),
+                                    w,
+                                    h,
+                                    original_dimensions,
+                                    screenshot_origin,
+                                )),
+                                None,
+                            )
                         } else {
                             use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
-                            (Some((Some(B64.encode(&png)), None, w, h, original_w)), None)
+                            (
+                                Some((
+                                    Some(B64.encode(&png)),
+                                    None,
+                                    w,
+                                    h,
+                                    original_dimensions,
+                                    screenshot_origin,
+                                )),
+                                None,
+                            )
                         }
                     }
                     Err(e) => (None, Some(format!("{e}"))),
                 }
             } else {
                 (None, None)
             };
-            Ok((tree_result, screenshot, screenshot_err))
+            Ok((tree_result, screenshot, screenshot_err, window_bounds))
         });
         // Timeout: Chrome's UIA provider can block indefinitely on property reads.
         let result: Result<anyhow::Result<_>, _> =
@@ -1295,9 +1360,13 @@ impl Tool for GetWindowStateTool {
         let result = result.and_then(|r| r);
 
         match result {
-            Ok((tree_opt, screenshot_opt, screenshot_err)) => {
+            Ok((tree_opt, screenshot_opt, screenshot_err, window_bounds)) => {
                 let mut content = Vec::new();
                 let mut structured = json!({ "window_id": hwnd, "pid": pid });
+                crate::window_state_coordinates::annotate_coordinate_spaces(
+                    &mut structured,
+                    window_bounds,
+                );
 
                 if let Some(tr) = tree_opt {
                     let is_msaa = tr.nodes.iter().any(|n| n.msaa_role.is_some());
@@ -1392,13 +1461,12 @@ impl Tool for GetWindowStateTool {
                             if let Some(parent) = n.parent_element_index {
                                 entry["parent_index"] = json!(parent);
                             }
-                            if let Some((l, t, r, b)) = n.rect {
-                                entry["frame"] = json!({
-                                    "x": l,
-                                    "y": t,
-                                    "w": (r - l).max(0),
-                                    "h": (b - t).max(0),
-                                });
+                            if let Some(frame) = n.rect {
+                                crate::window_state_coordinates::annotate_element_frame(
+                                    &mut entry,
+                                    frame,
+                                    window_bounds,
+                                );
                             }
                             Some(entry)
                         })
@@ -1458,9 +1526,11 @@ impl Tool for GetWindowStateTool {
                     }
                 }
 
-                if let Some((b64_opt, file_path, w, h, orig_w)) = screenshot_opt {
+                if let Some((b64_opt, file_path, w, h, original_dimensions, screenshot_origin)) =
+                    screenshot_opt
+                {
                     if !observation_only {
-                        if let Some(ow) = orig_w {
+                        if let Some((ow, _)) = original_dimensions {
                             if w > 0 {
                                 state.resize_registry.set_ratio(pid, ow as f64 / w as f64);
                             }
@@ -1480,6 +1550,18 @@ impl Tool for GetWindowStateTool {
                     // the structured payload so consumers don't have to sniff
                     // magic bytes off the base64 to know the format.
                     structured["screenshot_mime_type"] = json!("image/png");
+                    // Pixel actions use ResizeRegistry's width-derived uniform
+                    // ratio for both axes. Report that exact actuator mapping
+                    // (including any one-pixel height rounding in the PNG), not
+                    // an independently inferred visual Y ratio.
+                    let action_scale = original_dimensions
+                        .map(|(ow, _)| ow as f64 / w as f64)
+                        .unwrap_or(1.0);
+                    crate::window_state_coordinates::annotate_pixel_action_transform(
+                        &mut structured,
+                        screenshot_origin,
+                        (action_scale, action_scale),
+                    );
                     if let Some(fp) = file_path {
                         structured["screenshot_file_path"] = json!(fp);
                     }