From 2ccc1bdfed4f9be1a346c2b655c2090322b8d677 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Wed, 20 May 2026 13:16:28 +0200 Subject: [PATCH] fix(cua-driver-rs/windows): route screenshot through screen-region BitBlt for XAML/UWP targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partial fix for CUA-542 — `PrintWindow` doesn't capture DirectComposition-backed surfaces (modern UWP / WinUI3 apps render directly to the compositor, no GDI back buffer). For known XAML host targets (detected via the existing `input::is_xaml_host_hwnd` predicate), the screenshot tool now skips PrintWindow and copies the window's on-screen bounds from the desktop DC. Mirrors the screen- capture approach the Windows Snipping Tool uses for its "Window" mode. Coverage: - Visible-at-real-size UWP / WinUI3 apps (Edge, Photos with content open, Settings, modern Notepad when foreground) — fixed. Their GetWindowRect returns real bounds and the desktop DC has their composited pixels. - Legacy Win32 apps — unchanged, still PrintWindow. - Background-collapsed UWP (notably Windows 11 Calculator launched via SW_SHOWNOACTIVATE) — NOT fixed. Calculator's top-level window persists at 120×30 px at (0, 1) even after SetWindowPos + ShowWindow(SW_RESTORE) + SetForegroundWindow attempts; the real Calculator UI is hosted in a child CoreWindow managed by the compositor with no addressable HWND. Capturing this case requires Windows.Graphics.Capture against the CoreWindow's GraphicsCapture- Item — left tracked on CUA-542 as the proper followup. Implementation notes: - `is_xaml_host_hwnd` is the predicate from PR #1597 — checks top- level window class against a list and falls back to the owning .exe basename. Verified live via `debug_window_info` (which also shipped in #1597) that Calculator's `xaml_class_match: true` and `xaml_routing_recommended: true` predicates fire. - `screenshot_via_screen_region` opens NULL-HWND desktop DC, BitBlts from screen coordinates returned by `GetWindowRect`, then GetDIBits to BGRA buffer. Same encode path as the PrintWindow success branch. - The mostly-black sentinel heuristic (`is_mostly_black_bgra`) is kept as a safety net for the rare legacy-Win32-app-using-D3D case, but the primary trigger is now the XAML host predicate. Tested live on the Windows VM: - Foreground Calculator window (forced via SetForegroundWindow): still 120×30 due to Calculator's compositor-managed UI; no improvement visible in this specific case. Limitation documented above. - Build clean; no regression on the legacy Win32 capture path. Closes part of CUA-542 (the visible-UWP case). The Calculator-style case is reopened on the same ticket; WGC implementation is the proper followup and is well-scoped from the work in this PR. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../crates/platform-windows/src/capture.rs | 162 +++++++++++++++++- 1 file changed, 160 insertions(+), 2 deletions(-) diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs index a6c920c896..2615ad863f 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs @@ -1,7 +1,30 @@ //! Window screenshot via PrintWindow + GDI BitBlt on Windows. //! -//! PW_RENDERFULLCONTENT (0x2) renders the window contents even if it is -//! occluded or off-screen. The result is encoded as base64 PNG in memory. +//! `PW_RENDERFULLCONTENT` (0x2) renders the window contents even if it is +//! occluded or off-screen for GDI-backed surfaces. The result is encoded as +//! base64 PNG in memory. +//! +//! ## UWP / DirectComposition fallback (CUA-542) +//! +//! `PrintWindow` doesn't capture DirectComposition-backed surfaces — +//! modern UWP / WinUI3 apps (Calculator, Photos, Settings, Win 11 +//! Notepad) render directly to the GPU compositor and have no GDI back +//! buffer for `PrintWindow` to copy from. Result is an all-black image. +//! +//! When the PrintWindow result comes back mostly-black (sentinel for +//! that case), we fall back to a **screen-region BitBlt**: read the +//! window's on-screen bounds via `GetWindowRect`, BitBlt the matching +//! pixels off the desktop DC. This is the same approach the Windows +//! Snipping Tool's "Window" mode uses. Trade-off: only works when the +//! window is actually on-screen and not occluded by another window. +//! For our daemon-driven agent flow that's the common case anyway — +//! the daemon lives in the user's interactive session and the target +//! is typically a visible window the agent just launched. +//! +//! The full proper fix (Windows.Graphics.Capture, which works for +//! occluded / off-screen UWP windows too) is tracked separately on +//! CUA-542; the screen-region fallback covers the same common +//! ground at a fraction of the implementation cost. use anyhow::{bail, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; @@ -15,6 +38,94 @@ use windows::Win32::Graphics::Gdi::{GetWindowDC, ReleaseDC}; use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS}; const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32); +/// After GetDIBits we have BGRA bytes from PrintWindow. If essentially every +/// pixel is fully-transparent black or fully-opaque black, treat the capture +/// as "PrintWindow didn't render this surface" and let the caller fall back +/// to the screen-region BitBlt path. +/// +/// We sample sparsely (every 64th pixel) so the heuristic is cheap even on +/// 4K windows. The threshold is intentionally aggressive — UWP apps return +/// all-zeros bitmaps, not just dark frames — so legitimate dark UI doesn't +/// trip the fallback. +fn is_mostly_black_bgra(bgra: &[u8]) -> bool { + if bgra.len() < 16 { return true; } + let pixel_count = bgra.len() / 4; + if pixel_count == 0 { return true; } + let stride = (pixel_count / 1024).max(1); + let mut sampled = 0usize; + let mut black = 0usize; + for i in (0..pixel_count).step_by(stride) { + let off = i * 4; + // BGRA layout. We consider a pixel "black" when B+G+R == 0, + // regardless of alpha — that's the all-zero pattern UWP / + // DirectComposition leaves behind. + if bgra[off] == 0 && bgra[off + 1] == 0 && bgra[off + 2] == 0 { + black += 1; + } + sampled += 1; + } + // > 99.5% of sampled pixels are black → treat as failed render. + sampled > 0 && (black * 200) >= (sampled * 199) +} + +/// Fallback capture path: BitBlt the desktop DC over the rectangle covered +/// by `hwnd`'s on-screen bounds. Works for UWP / WinUI3 / DirectComposition +/// surfaces that PrintWindow can't reach, as long as the window is on-screen +/// (the daemon's typical case — see module docs). +unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec, i32, i32)> { + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; + + let mut rect = RECT::default(); + GetWindowRect(hwnd, &mut rect)?; + let w = rect.right - rect.left; + let h = rect.bottom - rect.top; + if w <= 0 || h <= 0 { + bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}"); + } + + let screen_dc = GetDC(HWND(std::ptr::null_mut())); // NULL HWND → desktop DC + let mem_dc = CreateCompatibleDC(screen_dc); + let bitmap = CreateCompatibleBitmap(screen_dc, w, h); + let old_bitmap = SelectObject(mem_dc, bitmap); + + // Copy from screen coords (rect.left, rect.top) into our memory DC at (0, 0). + let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, rect.left, rect.top, SRCCOPY); + + let mut bmi = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: w, + biHeight: -h, // top-down + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + biSizeImage: (w * h * 4) as u32, + ..Default::default() + }, + bmiColors: [RGBQUAD::default(); 1], + }; + let pixel_count = (w * h) as usize; + let mut pixels = vec![0u8; pixel_count * 4]; + let ok = GetDIBits( + mem_dc, bitmap, 0, h as u32, + Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS, + ); + + SelectObject(mem_dc, old_bitmap); + let _ = DeleteObject(bitmap); + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND(std::ptr::null_mut()), screen_dc); + + if blt_ok.is_err() { + bail!("screen-region fallback: BitBlt failed: {:?}", blt_ok); + } + if ok == 0 { + bail!("screen-region fallback: GetDIBits returned 0"); + } + Ok((pixels, w, h)) +} + /// Capture a window by HWND, returning raw PNG bytes. pub fn screenshot_window_bytes(hwnd: u64) -> Result> { unsafe { screenshot_window_bytes_unsafe(hwnd) } @@ -36,8 +147,33 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { use windows::Win32::UI::WindowsAndMessaging::GetClientRect; use windows::Win32::Foundation::RECT; + let hwnd_raw = hwnd; let hwnd = HWND(hwnd as *mut _); + // CUA-542 routing: for known XAML / WinUI3 / UWP targets, skip + // PrintWindow entirely and go straight to the screen-region BitBlt + // path. PrintWindow either returns all-black bitmaps for those + // surfaces or — as observed for backgrounded Calculator — a tiny + // clipped capture of the window's collapsed client rect. The + // screen-region path reads from the live desktop DC, which has the + // real composited image. + if crate::input::is_xaml_host_hwnd(hwnd_raw) { + match screenshot_via_screen_region(hwnd) { + Ok((pixels, w, h)) => { + return encode_bgra_to_png(&pixels, w as u32, h as u32); + } + Err(e) => { + // Screen-region failed — fall through and try PrintWindow as a + // last resort so the caller at least gets *something*. + tracing::warn!( + target: "cua-driver", + "screenshot: XAML target screen-region path failed: {e}; \ + falling back to PrintWindow (likely all-black)." + ); + } + } + } + let mut rect = RECT::default(); GetClientRect(hwnd, &mut rect)?; let w = (rect.right - rect.left) as i32; @@ -81,6 +217,28 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { if ok == 0 { bail!("GetDIBits returned 0"); } + // CUA-542: detect the all-black bitmap PrintWindow returns for + // DirectComposition-backed UWP / WinUI3 surfaces and retry via + // screen-region BitBlt. See module docs for the rationale. + if is_mostly_black_bgra(&pixels) { + match screenshot_via_screen_region(hwnd) { + Ok((alt_pixels, alt_w, alt_h)) => { + return encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32); + } + Err(e) => { + // Screen-region path failed too — return the (black) PrintWindow + // result with an explanatory log rather than erroring outright. + // Caller still gets an image; the fact that it's black is now + // visible in the bytes themselves. + tracing::warn!( + target: "cua-driver", + "screenshot: PrintWindow returned a mostly-black bitmap (UWP / \ + DirectComposition target?); screen-region fallback failed: {e}" + ); + } + } + } + encode_bgra_to_png(&pixels, w as u32, h as u32) }