diff --git a/libs/cua-driver/rust/crates/cua-driver/build.rs b/libs/cua-driver/rust/crates/cua-driver/build.rs index e0de604b1a..0c11c96116 100644 --- a/libs/cua-driver/rust/crates/cua-driver/build.rs +++ b/libs/cua-driver/rust/crates/cua-driver/build.rs @@ -7,8 +7,9 @@ // emitting crate is the final binary crate — for transitive deps Cargo // silently drops them. So we re-emit the same rpaths from here. // -// On Windows, embed the DPI-awareness manifest so the process receives -// logical (not physical) screen coordinates at 125%/150%/200% scaling. +// On Windows, embed the Per-Monitor V2 DPI-awareness manifest so the +// process sees physical pixels (no DWM coordinate virtualization) at +// 125%/150%/200% scaling and clicks land where screenshots say they do. fn main() { #[cfg(target_os = "windows")] diff --git a/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc b/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc index d7c5fc9f80..49037afde0 100644 --- a/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc +++ b/libs/cua-driver/rust/crates/cua-driver/cua-driver.rc @@ -1 +1,6 @@ -1 RT_MANIFEST "cua-driver.manifest" +// 24 = RT_MANIFEST. The numeric type is required: `RT_MANIFEST` is not an +// .rc keyword, so the resource compiler would emit a custom *string-typed* +// resource named "RT_MANIFEST" that the Windows loader never applies as a +// manifest (the process then starts DPI-unaware). ID 1 = +// CREATEPROCESS_MANIFEST_RESOURCE_ID. +1 24 "cua-driver.manifest" diff --git a/libs/cua-driver/rust/crates/platform-windows/examples/get_screen_size_parity.rs b/libs/cua-driver/rust/crates/platform-windows/examples/get_screen_size_parity.rs index 493d72218a..558028a13e 100644 --- a/libs/cua-driver/rust/crates/platform-windows/examples/get_screen_size_parity.rs +++ b/libs/cua-driver/rust/crates/platform-windows/examples/get_screen_size_parity.rs @@ -1,7 +1,7 @@ //! Parity check for `get_screen_size`. //! -//! Asserts text matches Swift exactly: -//! `"✅ Main display: WxH points @ Sx"` +//! Asserts text matches the Windows wording: +//! `"✅ Main display: WxH pixels @ Sx"` //! and `structuredContent` has `width`, `height`, and `scale_factor` //! (snake_case) that agree with the platform's `GetSystemMetrics` and //! `GetDpiForSystem`. @@ -40,11 +40,11 @@ fn main() { .expect("missing text"); println!("Response text: {text:?}"); - // Parse "✅ Main display: WxH points @ Sx" by hand. + // Parse "✅ Main display: WxH pixels @ Sx" by hand. let rest = text.strip_prefix("✅ Main display: ") - .unwrap_or_else(|| panic!("Text {text:?} missing Swift prefix `✅ Main display: `")); - let (wh, scale_part) = rest.split_once(" points @ ") - .unwrap_or_else(|| panic!("Text {text:?} missing ` points @ `")); + .unwrap_or_else(|| panic!("Text {text:?} missing prefix `✅ Main display: `")); + let (wh, scale_part) = rest.split_once(" pixels @ ") + .unwrap_or_else(|| panic!("Text {text:?} missing ` pixels @ `")); let (w_str, h_str) = wh.split_once('x') .unwrap_or_else(|| panic!("Text {text:?} missing `x` separator")); let scale_str = scale_part.strip_suffix('x') @@ -73,7 +73,7 @@ fn main() { assert_eq!((tw, th), (nw, nh)); assert!((ts - ns).abs() < 0.001); - println!("\n✅ PASS: text format matches Swift, structuredContent has scale_factor, native agrees"); + println!("\n✅ PASS: text format matches, structuredContent has scale_factor, native agrees"); } #[cfg(not(target_os = "windows"))] diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs index 2247e39c90..abc3e9e51a 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs @@ -131,23 +131,20 @@ unsafe fn target_is_obscured(target: HWND) -> bool { unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec, i32, i32)> { use windows::Win32::Foundation::RECT; use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; - use windows::Win32::UI::HiDpi::GetDpiForWindow; let mut rect = RECT::default(); GetWindowRect(hwnd, &mut rect)?; - // With permonitorv2 DPI awareness, GetWindowRect returns logical coordinates. - // BitBlt needs physical pixel coordinates, so scale by the window's DPI. - let dpi = GetDpiForWindow(hwnd); - let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }; + // Under Per-Monitor V2 DPI awareness, GetWindowRect already returns + // PHYSICAL pixels (coordinate virtualization only applies to + // DPI-unaware/system-aware processes), and BitBlt operates in physical + // pixels too — use the rect as-is. Scaling by DPI/96 here would shift + // and oversize the captured screen region (issue #1879). + let physical_left = rect.left; + let physical_top = rect.top; - let physical_left = (rect.left as f64 * scale).round() as i32; - let physical_top = (rect.top as f64 * scale).round() as i32; - let physical_right = (rect.right as f64 * scale).round() as i32; - let physical_bottom = (rect.bottom as f64 * scale).round() as i32; - - let w = physical_right - physical_left; - let h = physical_bottom - physical_top; + let w = rect.right - rect.left; + let h = rect.bottom - rect.top; if w <= 0 || h <= 0 { bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}"); } @@ -285,23 +282,21 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve // Window-sized buffer captures title bar + body + non-client trim // correctly. // - // With permonitorv2 DPI awareness, GetWindowRect returns logical dimensions - // but GetWindowDC and PrintWindow/BitBlt operate in physical pixels. - // Scale to physical dimensions for the bitmap. + // Under Per-Monitor V2 DPI awareness, GetWindowRect returns PHYSICAL + // pixels — the same unit GetWindowDC and PrintWindow/BitBlt work in. + // Use the dimensions as-is; scaling by DPI/96 would allocate an + // oversized bitmap with the content in its top-left corner and a + // black margin around it (issue #1879). It also kept the + // DWMWA_EXTENDED_FRAME_BOUNDS crop below (physical pixels) from + // matching the bitmap. let mut win_rect = RECT::default(); GetWindowRect(hwnd, &mut win_rect)?; - let logical_w = (win_rect.right - win_rect.left) as i32; - let logical_h = (win_rect.bottom - win_rect.top) as i32; - if logical_w <= 0 || logical_h <= 0 { - bail!("Window has zero/negative size: {}x{}", logical_w, logical_h); + let w = win_rect.right - win_rect.left; + let h = win_rect.bottom - win_rect.top; + if w <= 0 || h <= 0 { + bail!("Window has zero/negative size: {}x{}", w, h); } - use windows::Win32::UI::HiDpi::GetDpiForWindow; - let dpi = GetDpiForWindow(hwnd); - let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }; - let w = (logical_w as f64 * scale).round() as i32; - let h = (logical_h as f64 * scale).round() as i32; - let screen_dc = GetWindowDC(hwnd); let mem_dc = CreateCompatibleDC(screen_dc); let bitmap = CreateCompatibleBitmap(screen_dc, w, h); @@ -476,17 +471,13 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve pub fn screenshot_display_bytes() -> Result> { unsafe { use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN}; - use windows::Win32::UI::HiDpi::GetDpiForSystem; - // With permonitorv2 DPI awareness, GetSystemMetrics returns logical pixels. - // BitBlt captures physical pixels, so we must scale by DPI factor. - let logical_w = GetSystemMetrics(SM_CXSCREEN); - let logical_h = GetSystemMetrics(SM_CYSCREEN); - if logical_w <= 0 || logical_h <= 0 { bail!("Could not get screen metrics"); } - - let dpi = GetDpiForSystem(); - let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }; - let w = (logical_w as f64 * scale).round() as i32; - let h = (logical_h as f64 * scale).round() as i32; + // Under Per-Monitor V2 DPI awareness, GetSystemMetrics returns + // PHYSICAL pixels — the same unit BitBlt captures in. Scaling by + // DPI/96 would allocate an oversized bitmap with black margins + // (issue #1879). + let w = GetSystemMetrics(SM_CXSCREEN); + let h = GetSystemMetrics(SM_CYSCREEN); + if w <= 0 || h <= 0 { bail!("Could not get screen metrics"); } let screen_dc = GetDC(HWND::default()); let mem_dc = CreateCompatibleDC(screen_dc); let bitmap = CreateCompatibleBitmap(screen_dc, w, h); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index e0c60bbd7b..80e1210d5f 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -3872,9 +3872,9 @@ impl Tool for GetScreenSizeTool { fn def(&self) -> &ToolDef { GSS_DEF.get_or_init(|| ToolDef { name: "get_screen_size".into(), - description: "Return the logical size of the main display in points plus its backing \ - scale factor. Agents click in points; Retina displays have scale_factor 2.0. \ - Requires no TCC permissions.".into(), + description: "Return the size of the main display in physical pixels plus its display \ + scale factor. On Windows, screenshots and pixel clicks use this same physical-pixel \ + coordinate space. Requires no special permissions.".into(), input_schema: json!({"type":"object","properties":{},"additionalProperties":false}), read_only: true, destructive: false, idempotent: true, open_world: false, }) @@ -3883,14 +3883,13 @@ impl Tool for GetScreenSizeTool { use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN}; use windows::Win32::UI::HiDpi::GetDpiForSystem; // With permonitorv2 DPI awareness (set in cua-driver.manifest), - // SM_CXSCREEN/SM_CYSCREEN already return logical points (DPI-scaled). + // SM_CXSCREEN/SM_CYSCREEN return PHYSICAL pixels — the same + // coordinate space screenshots and pixel clicks use on Windows. // Report these as-is, along with the scale factor for reference. - // Matches Swift's NSScreen.frame behavior on macOS. let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) }; let dpi = unsafe { GetDpiForSystem() }; let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }; - // Matches Swift text format 1:1. - ToolResult::text(format!("✅ Main display: {w}x{h} points @ {scale}x")) + ToolResult::text(format!("✅ Main display: {w}x{h} pixels @ {scale}x")) .with_structured(json!({ "width": w, "height": h, "scale_factor": scale })) } }