diff --git a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs index a6c920c896..2615ad863f 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/capture.rs @@ -1,7 +1,30 @@ //! Window screenshot via PrintWindow + GDI BitBlt on Windows. //! -//! PW_RENDERFULLCONTENT (0x2) renders the window contents even if it is -//! occluded or off-screen. The result is encoded as base64 PNG in memory. +//! `PW_RENDERFULLCONTENT` (0x2) renders the window contents even if it is +//! occluded or off-screen for GDI-backed surfaces. The result is encoded as +//! base64 PNG in memory. +//! +//! ## UWP / DirectComposition fallback (CUA-542) +//! +//! `PrintWindow` doesn't capture DirectComposition-backed surfaces — +//! modern UWP / WinUI3 apps (Calculator, Photos, Settings, Win 11 +//! Notepad) render directly to the GPU compositor and have no GDI back +//! buffer for `PrintWindow` to copy from. Result is an all-black image. +//! +//! When the PrintWindow result comes back mostly-black (sentinel for +//! that case), we fall back to a **screen-region BitBlt**: read the +//! window's on-screen bounds via `GetWindowRect`, BitBlt the matching +//! pixels off the desktop DC. This is the same approach the Windows +//! Snipping Tool's "Window" mode uses. Trade-off: only works when the +//! window is actually on-screen and not occluded by another window. +//! For our daemon-driven agent flow that's the common case anyway — +//! the daemon lives in the user's interactive session and the target +//! is typically a visible window the agent just launched. +//! +//! The full proper fix (Windows.Graphics.Capture, which works for +//! occluded / off-screen UWP windows too) is tracked separately on +//! CUA-542; the screen-region fallback covers the same common +//! ground at a fraction of the implementation cost. use anyhow::{bail, Result}; use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; @@ -15,6 +38,94 @@ use windows::Win32::Graphics::Gdi::{GetWindowDC, ReleaseDC}; use windows::Win32::Storage::Xps::{PrintWindow, PRINT_WINDOW_FLAGS}; const PW_RENDERFULLCONTENT: PRINT_WINDOW_FLAGS = PRINT_WINDOW_FLAGS(2u32); +/// After GetDIBits we have BGRA bytes from PrintWindow. If essentially every +/// pixel is fully-transparent black or fully-opaque black, treat the capture +/// as "PrintWindow didn't render this surface" and let the caller fall back +/// to the screen-region BitBlt path. +/// +/// We sample sparsely (every 64th pixel) so the heuristic is cheap even on +/// 4K windows. The threshold is intentionally aggressive — UWP apps return +/// all-zeros bitmaps, not just dark frames — so legitimate dark UI doesn't +/// trip the fallback. +fn is_mostly_black_bgra(bgra: &[u8]) -> bool { + if bgra.len() < 16 { return true; } + let pixel_count = bgra.len() / 4; + if pixel_count == 0 { return true; } + let stride = (pixel_count / 1024).max(1); + let mut sampled = 0usize; + let mut black = 0usize; + for i in (0..pixel_count).step_by(stride) { + let off = i * 4; + // BGRA layout. We consider a pixel "black" when B+G+R == 0, + // regardless of alpha — that's the all-zero pattern UWP / + // DirectComposition leaves behind. + if bgra[off] == 0 && bgra[off + 1] == 0 && bgra[off + 2] == 0 { + black += 1; + } + sampled += 1; + } + // > 99.5% of sampled pixels are black → treat as failed render. + sampled > 0 && (black * 200) >= (sampled * 199) +} + +/// Fallback capture path: BitBlt the desktop DC over the rectangle covered +/// by `hwnd`'s on-screen bounds. Works for UWP / WinUI3 / DirectComposition +/// surfaces that PrintWindow can't reach, as long as the window is on-screen +/// (the daemon's typical case — see module docs). +unsafe fn screenshot_via_screen_region(hwnd: HWND) -> Result<(Vec, i32, i32)> { + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; + + let mut rect = RECT::default(); + GetWindowRect(hwnd, &mut rect)?; + let w = rect.right - rect.left; + let h = rect.bottom - rect.top; + if w <= 0 || h <= 0 { + bail!("screen-region fallback: window has zero/negative bounds: {w}x{h}"); + } + + let screen_dc = GetDC(HWND(std::ptr::null_mut())); // NULL HWND → desktop DC + let mem_dc = CreateCompatibleDC(screen_dc); + let bitmap = CreateCompatibleBitmap(screen_dc, w, h); + let old_bitmap = SelectObject(mem_dc, bitmap); + + // Copy from screen coords (rect.left, rect.top) into our memory DC at (0, 0). + let blt_ok = BitBlt(mem_dc, 0, 0, w, h, screen_dc, rect.left, rect.top, SRCCOPY); + + let mut bmi = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: std::mem::size_of::() as u32, + biWidth: w, + biHeight: -h, // top-down + biPlanes: 1, + biBitCount: 32, + biCompression: BI_RGB.0, + biSizeImage: (w * h * 4) as u32, + ..Default::default() + }, + bmiColors: [RGBQUAD::default(); 1], + }; + let pixel_count = (w * h) as usize; + let mut pixels = vec![0u8; pixel_count * 4]; + let ok = GetDIBits( + mem_dc, bitmap, 0, h as u32, + Some(pixels.as_mut_ptr() as *mut _), &mut bmi, DIB_RGB_COLORS, + ); + + SelectObject(mem_dc, old_bitmap); + let _ = DeleteObject(bitmap); + let _ = DeleteDC(mem_dc); + ReleaseDC(HWND(std::ptr::null_mut()), screen_dc); + + if blt_ok.is_err() { + bail!("screen-region fallback: BitBlt failed: {:?}", blt_ok); + } + if ok == 0 { + bail!("screen-region fallback: GetDIBits returned 0"); + } + Ok((pixels, w, h)) +} + /// Capture a window by HWND, returning raw PNG bytes. pub fn screenshot_window_bytes(hwnd: u64) -> Result> { unsafe { screenshot_window_bytes_unsafe(hwnd) } @@ -36,8 +147,33 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { use windows::Win32::UI::WindowsAndMessaging::GetClientRect; use windows::Win32::Foundation::RECT; + let hwnd_raw = hwnd; let hwnd = HWND(hwnd as *mut _); + // CUA-542 routing: for known XAML / WinUI3 / UWP targets, skip + // PrintWindow entirely and go straight to the screen-region BitBlt + // path. PrintWindow either returns all-black bitmaps for those + // surfaces or — as observed for backgrounded Calculator — a tiny + // clipped capture of the window's collapsed client rect. The + // screen-region path reads from the live desktop DC, which has the + // real composited image. + if crate::input::is_xaml_host_hwnd(hwnd_raw) { + match screenshot_via_screen_region(hwnd) { + Ok((pixels, w, h)) => { + return encode_bgra_to_png(&pixels, w as u32, h as u32); + } + Err(e) => { + // Screen-region failed — fall through and try PrintWindow as a + // last resort so the caller at least gets *something*. + tracing::warn!( + target: "cua-driver", + "screenshot: XAML target screen-region path failed: {e}; \ + falling back to PrintWindow (likely all-black)." + ); + } + } + } + let mut rect = RECT::default(); GetClientRect(hwnd, &mut rect)?; let w = (rect.right - rect.left) as i32; @@ -81,6 +217,28 @@ unsafe fn screenshot_window_bytes_unsafe(hwnd: u64) -> Result> { if ok == 0 { bail!("GetDIBits returned 0"); } + // CUA-542: detect the all-black bitmap PrintWindow returns for + // DirectComposition-backed UWP / WinUI3 surfaces and retry via + // screen-region BitBlt. See module docs for the rationale. + if is_mostly_black_bgra(&pixels) { + match screenshot_via_screen_region(hwnd) { + Ok((alt_pixels, alt_w, alt_h)) => { + return encode_bgra_to_png(&alt_pixels, alt_w as u32, alt_h as u32); + } + Err(e) => { + // Screen-region path failed too — return the (black) PrintWindow + // result with an explanatory log rather than erroring outright. + // Caller still gets an image; the fact that it's black is now + // visible in the bytes themselves. + tracing::warn!( + target: "cua-driver", + "screenshot: PrintWindow returned a mostly-black bitmap (UWP / \ + DirectComposition target?); screen-region fallback failed: {e}" + ); + } + } + } + encode_bgra_to_png(&pixels, w as u32, h as u32) }