diff --git a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs index 22e507a148..ecad0c4379 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/capture.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/capture.rs @@ -27,6 +27,12 @@ pub fn screenshot_window_bytes(window_id: u32) -> anyhow::Result> { .status()?; if !status.success() { + if crate::display_state::main_display_asleep() { + anyhow::bail!( + "screencapture failed for window {window_id}: {}", + crate::display_state::ASLEEP_CAPTURE_HINT + ); + } anyhow::bail!("screencapture failed for window {window_id}"); } @@ -59,6 +65,12 @@ pub fn screenshot_display_bytes() -> anyhow::Result> { .status()?; if !status.success() { + if crate::display_state::main_display_asleep() { + anyhow::bail!( + "screencapture failed for main display: {}", + crate::display_state::ASLEEP_CAPTURE_HINT + ); + } anyhow::bail!("screencapture failed for main display"); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/display_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/display_state.rs new file mode 100644 index 0000000000..8f8a85a192 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/display_state.rs @@ -0,0 +1,68 @@ +//! Main-display power state. +//! +//! When the display is asleep (idle sleep, lid closed without an awake +//! external display), two things break silently: +//! * `screencapture` fails for every window, so `get_window_state` / +//! `zoom` / `debug_image_out` return opaque errors, and +//! * posted CGEvents may never be rendered by the target app, and the +//! agent has no screenshot to verify them against. +//! +//! An agent that doesn't know the display is asleep burns its budget +//! re-deriving coordinates and re-posting clicks into the void. Capture +//! errors and action results on macOS therefore carry an explicit +//! "display is asleep" marker while this state holds. + +use std::os::raw::c_uint; + +extern "C" { + fn CGMainDisplayID() -> c_uint; + fn CGDisplayIsAsleep(display: c_uint) -> u32; +} + +/// True when the main display is asleep. Pure WindowServer query ā€” cheap +/// enough to call on every action/capture result. +pub fn main_display_asleep() -> bool { + unsafe { CGDisplayIsAsleep(CGMainDisplayID()) != 0 } +} + +/// Hint appended to capture errors while the display sleeps. +pub const ASLEEP_CAPTURE_HINT: &str = + "the main display is asleep, so macOS cannot capture windows. Wake it \ + (user presence or `caffeinate -u -t 1`) and retry"; + +const ASLEEP_ACTION_SUFFIX: &str = + "\n\nšŸ˜“ Main display is ASLEEP: the event was posted, but the app may \ + never render it and no screenshot can verify it. Wake the display \ + (user presence or `caffeinate -u -t 1`) before retrying or verifying ā€” \ + do NOT re-derive coordinates from this failure."; + +/// Suffix for action success messages: empty while the display is awake. +pub fn asleep_suffix() -> &'static str { + asleep_suffix_for(main_display_asleep()) +} + +/// Pure mapping used by `asleep_suffix` ā€” split out for unit testing. +fn asleep_suffix_for(asleep: bool) -> &'static str { + if asleep { + ASLEEP_ACTION_SUFFIX + } else { + "" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn suffix_empty_when_awake() { + assert_eq!(asleep_suffix_for(false), ""); + } + + #[test] + fn suffix_warns_when_asleep() { + let s = asleep_suffix_for(true); + assert!(s.contains("ASLEEP")); + assert!(s.contains("caffeinate")); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs index f5015378ff..27e509ef80 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -18,6 +18,8 @@ pub mod input; pub mod cursor; #[cfg(target_os = "macos")] pub mod capture; + +pub mod display_state; #[cfg(target_os = "macos")] pub mod browser; #[cfg(target_os = "macos")] diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs index 89e9566704..7f33cb161a 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -279,13 +279,15 @@ impl Tool for ClickTool { Ok(Ok(Some(pid))) => ToolResult::text(format!( "āœ… Sent {button_label} at desktop-pixel ({sx_shot:.0},{sy_shot:.0}) \ ā†’ screen-point ({sx:.0},{sy:.0}) on pid {pid} (desktop scope; \ - not driver-verified ā€” confirm via screenshot)." + not driver-verified ā€” confirm via screenshot).{}", + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), Ok(Ok(None)) => ToolResult::text(format!( "āœ… Sent screen-absolute {button_label} at desktop-pixel \ ({sx_shot:.0},{sy_shot:.0}) ā†’ screen-point ({sx:.0},{sy:.0}) \ - (desktop scope, no window under point; not driver-verified)." + (desktop scope, no window under point; not driver-verified).{}", + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")), @@ -406,7 +408,8 @@ impl Tool for ClickTool { return match result { Ok(Ok(())) => ToolResult::text(format!( "āœ… Posted middle-click to pid {pid} at element [{idx}] center \ - (background CGEvent; not driver-verified ā€” confirm via screenshot)." + (background CGEvent; not driver-verified ā€” confirm via screenshot).{}", + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), Ok(Err(e)) => ToolResult::error(format!("Middle-click failed: {e}")), @@ -712,8 +715,9 @@ impl Tool for ClickTool { }; ToolResult::text(format!( "āœ… Posted {button_label} to pid {pid} ({mode_label}; \ - not driver-verified ā€” confirm via screenshot).{}", - changes.result_suffix() + not driver-verified ā€” confirm via screenshot).{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": path, "verified": false, "effect": "unverifiable" })) } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs index 3708e5ec61..4def8f7a77 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/double_click.rs @@ -194,7 +194,10 @@ impl Tool for DoubleClickTool { let mode_label = if fg { " (delivery_mode:foreground)" } else { "" }; match result { - Ok(Ok(())) => ToolResult::text(format!("āœ… Double-clicked at ({screen_x:.1}, {screen_y:.1}){mode_label}.")) + Ok(Ok(())) => ToolResult::text(format!( + "āœ… Double-clicked at ({screen_x:.1}, {screen_y:.1}){mode_label}.{}", + crate::display_state::asleep_suffix() + )) .with_structured(serde_json::json!({ "path": if fg { "cgevent_fg" } else { "cgevent" }, "verified": false, "effect": "unverifiable" })), @@ -244,5 +247,8 @@ fn ax_double_click(pid: i32, wid: u32, element_ptr: usize, idx: usize, cursor_ke screen coordinates as window-local for element [{idx}]." ))?; crate::input::mouse::click_at_xy_with_window_local(pid, cx, cy, wx, wy, wid, 2, &[])?; - Ok(format!("āœ… Double-clicked element [{idx}] at ({cx:.1}, {cy:.1}).")) + Ok(format!( + "āœ… Double-clicked element [{idx}] at ({cx:.1}, {cy:.1}).{}", + crate::display_state::asleep_suffix() + )) } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs index 09c157ff51..e43a6e8106 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs @@ -284,7 +284,7 @@ impl Tool for DragTool { from_sx as i64, from_sy as i64, to_sx as i64, to_sy as i64, changes.result_suffix(), - )) + ) + crate::display_state::asleep_suffix()) .with_structured(serde_json::json!({ "path": if fg { "cgevent_fg" } else { "cgevent" }, "verified": false, "effect": "unverifiable" })), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index 953c8f8147..eda471e605 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -184,6 +184,10 @@ impl Tool for GetWindowStateTool { // screenshot_out_file). With `screenshot_out_file` set, write to disk and // surface the path instead of embedding base64; otherwise embed base64. let max_dim = effective_max_dim; + // Why the screenshot is missing, when it is ā€” surfaced to the caller so + // a capture failure (permissions, display asleep, WindowServer refusal) + // is never silent while `escalation` tells the agent to act off it. + let mut screenshot_error: Option = None; // Returns (b64_or_path, final_w, final_h, Option, is_file_path) let screenshot = if should_capture { let out_file = screenshot_out_file.clone(); @@ -215,10 +219,12 @@ impl Tool for GetWindowStateTool { } Ok(Err(e)) => { tracing::warn!("Screenshot failed for window {window_id}: {e}"); + screenshot_error = Some(e.to_string()); None } Err(e) => { tracing::warn!("Screenshot task error for window {window_id}: {e}"); + screenshot_error = Some(e.to_string()); None } } @@ -251,13 +257,23 @@ impl Tool for GetWindowStateTool { content.push(Content::text(summary)); } else if let Some(ref r) = tree_result { let element_count = self.state.element_cache.element_count(pid, window_id); + let capture_note = screenshot_error + .as_ref() + .map(|e| format!("\n\nāš ļø screenshot unavailable: {e}")) + .unwrap_or_default(); content.push(Content::text(format!( - "window_id={window_id} pid={pid} elements={element_count}\n\n{}", + "window_id={window_id} pid={pid} elements={element_count}{capture_note}\n\n{}", r.tree_markdown ))); } if content.is_empty() { + if crate::display_state::main_display_asleep() { + return ToolResult::error(format!( + "No content produced (neither AX tree nor screenshot succeeded) ā€” {}", + crate::display_state::ASLEEP_CAPTURE_HINT + )); + } return ToolResult::error("No content produced (neither AX tree nor screenshot succeeded)"); } @@ -335,6 +351,9 @@ impl Tool for GetWindowStateTool { in this response (an element px action)." }); } + if let Some(ref err) = screenshot_error { + structured["screenshot_error"] = serde_json::json!(err); + } if let Some((sw, sh)) = screenshot_dims { structured["screenshot_width"] = serde_json::json!(sw); structured["screenshot_height"] = serde_json::json!(sh); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs index c6aed03836..6300935af4 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/hotkey.rs @@ -207,8 +207,9 @@ impl Tool for HotkeyTool { }); } ToolResult::text(format!( - "Pressed {key_display} on pid {pid}{label}.{}", - changes.result_suffix() + "Pressed {key_display} on pid {pid}{label}.{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )).with_structured(structured) } Ok(Err(e)) => ToolResult::error(format!("hotkey failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs index ebcf1ecd21..db5e58cd12 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/press_key.rs @@ -214,8 +214,9 @@ impl Tool for PressKeyTool { }); } ToolResult::text(format!( - "āœ… Pressed {display_key} on pid {pid}{label}.{}", - changes.result_suffix() + "āœ… Pressed {display_key} on pid {pid}{label}.{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )).with_structured(structured) } Ok(Err(e)) => ToolResult::error(format!("press_key failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs index 070196f4ff..dce9c00b33 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs @@ -295,8 +295,9 @@ impl Tool for ScrollTool { Ok(Ok(())) => ToolResult::text(format!( "āœ… Sent {direction} scroll by {by} Ɨ {amount} via pixel wheel at \ ({screen_x:.0}, {screen_y:.0}){mode_label} (background CGEvent; not \ - driver-verified ā€” confirm via screenshot).{}", - changes.result_suffix() + driver-verified ā€” confirm via screenshot).{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": if fg { "cgevent_fg" } else { "cgevent" }, "verified": false, "effect": "unverifiable" @@ -361,8 +362,9 @@ impl Tool for ScrollTool { match result { Ok(Ok(())) => ToolResult::text(format!( "āœ… Sent {direction} scroll by {by} Ɨ {amount} via keystroke \ - (background; not driver-verified ā€” confirm via screenshot).{}", - changes.result_suffix() + (background; not driver-verified ā€” confirm via screenshot).{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )) .with_structured(serde_json::json!({ "path": "key_events", "verified": false })), Ok(Err(e)) => ToolResult::error(format!("Scroll failed: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index 2d9ef6d610..72c26479ac 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -285,8 +285,9 @@ impl Tool for TypeTextTool { and re-call with delivery_mode:\"foreground\" if it didn't.".to_string()) }; ToolResult::text(format!( - "{mark} {char_count} char(s){detail}.{note}{}", - changes.result_suffix() + "{mark} {char_count} char(s){detail}.{note}{}{}", + changes.result_suffix(), + crate::display_state::asleep_suffix() )) .with_structured({ // `effect` mirrors `verified`'s read-back tri-state: a TRUSTED