diff --git a/docs/scripts/modal_app.py b/docs/scripts/modal_app.py index cb317bfccb..ad1d308a0a 100644 --- a/docs/scripts/modal_app.py +++ b/docs/scripts/modal_app.py @@ -390,7 +390,7 @@ def sync_to_s3(bucket: str = S3_BUCKET_NAME): key = f"docs_db/docs.lance/{fpath.relative_to(lance_dir)}" s3.upload_file(str(fpath), bucket, key) uploaded += 1 - print(f" Uploaded docs LanceDB directory") + print(" Uploaded docs LanceDB directory") # --- code databases --- code_db_dir = Path(CODE_DB_PATH) @@ -411,7 +411,7 @@ def sync_to_s3(bucket: str = S3_BUCKET_NAME): key = f"code_db/code_index.lancedb/{fpath.relative_to(code_lance)}" s3.upload_file(str(fpath), bucket, key) uploaded += 1 - print(f" Uploaded code LanceDB directory") + print(" Uploaded code LanceDB directory") print(f"S3 sync complete: {uploaded} files uploaded to s3://{bucket}/") return {"bucket": bucket, "files_uploaded": uploaded} diff --git a/examples/sandboxes/test_linux_cloud_vm.py b/examples/sandboxes/test_linux_cloud_vm.py index 286a9a28ed..6c398ee9a4 100644 --- a/examples/sandboxes/test_linux_cloud_vm.py +++ b/examples/sandboxes/test_linux_cloud_vm.py @@ -75,7 +75,9 @@ async def test_linux_cloud_vm(): len(screenshot), time.monotonic() - t2, ) - assert screenshot[:4] == b"\x89PNG", f"Screenshot not PNG: first 4 bytes = {screenshot[:4]!r}" + assert ( + screenshot[:4] == b"\x89PNG" + ), f"Screenshot not PNG: first 4 bytes = {screenshot[:4]!r}" total_time = time.monotonic() - t0 logger.info("Test passed in %.1fs (provision=%.1fs)", total_time, provision_time) diff --git a/libs/cua-driver-rs/tests/integration/driver_client.py b/libs/cua-driver-rs/tests/integration/driver_client.py index 850b16abeb..365404a072 100644 --- a/libs/cua-driver-rs/tests/integration/driver_client.py +++ b/libs/cua-driver-rs/tests/integration/driver_client.py @@ -94,9 +94,7 @@ def _notify(self, method: str, params: Optional[dict] = None) -> None: payload["params"] = params self._write(payload) - def _call( - self, method: str, params: Optional[dict] = None, timeout: float = 20.0 - ) -> dict: + def _call(self, method: str, params: Optional[dict] = None, timeout: float = 20.0) -> dict: self._next_id += 1 request_id = self._next_id payload = {"jsonrpc": "2.0", "id": request_id, "method": method} @@ -198,8 +196,7 @@ def resolve_window_id( if require_on_current_space: preferred = [ - w for w in windows - if w.get("is_on_screen") and w.get("on_current_space") is not False + w for w in windows if w.get("is_on_screen") and w.get("on_current_space") is not False ] if preferred: preferred.sort(key=lambda w: w.get("z_index", 0), reverse=True) diff --git a/libs/cua-driver-rs/tests/integration/test_api_parity.py b/libs/cua-driver-rs/tests/integration/test_api_parity.py index 3fbe1c2c50..6276980e10 100644 --- a/libs/cua-driver-rs/tests/integration/test_api_parity.py +++ b/libs/cua-driver-rs/tests/integration/test_api_parity.py @@ -47,6 +47,7 @@ # ── helpers ──────────────────────────────────────────────────────────────────── + def _run( args: list[str], timeout: int = 20, @@ -129,9 +130,9 @@ def _swift_binary() -> str | None: # In Rust, not in Swift yet. Tests for these will fail on the Swift binary. RUST_ONLY_TOOLS = [ - "type_text_chars", # per-character delay typing; Swift uses type_text - "browser_eval", # CDP JS eval; Swift uses 'page' for browser JS - "get_accessibility_tree",# lightweight desktop AX snapshot (separate from get_window_state) + "type_text_chars", # per-character delay typing; Swift uses type_text + "browser_eval", # CDP JS eval; Swift uses 'page' for browser JS + "get_accessibility_tree", # lightweight desktop AX snapshot (separate from get_window_state) ] # In Swift, not registered in Rust yet (page.rs exists but not in register_all). @@ -142,6 +143,7 @@ def _swift_binary() -> str | None: # ── parity mixin ─────────────────────────────────────────────────────────────── + class _ParityMixin: """All API surface tests. Concrete subclasses set `cls.binary`.""" @@ -185,8 +187,14 @@ def test_list_tools_contains_required_tool_names(self) -> None: r = _run([self.binary, "list-tools"]) self.assertEqual(r.returncode, 0) for name in [ - "click", "screenshot", "list_windows", "press_key", - "get_window_state", "type_text", "hotkey", "scroll", + "click", + "screenshot", + "list_windows", + "press_key", + "get_window_state", + "type_text", + "hotkey", + "scroll", ]: self.assertIn(name, r.stdout, f"list-tools missing: {name!r}") @@ -216,7 +224,8 @@ def test_describe_each_required_tool_exits_zero(self) -> None: with self.subTest(tool=name): r = _run([self.binary, "describe", name]) self.assertEqual( - r.returncode, 0, + r.returncode, + 0, f"describe {name!r} exited {r.returncode}: {r.stderr}", ) self.assertIn(name, r.stdout) @@ -331,10 +340,7 @@ def test_call_check_permissions_returns_accessibility_and_screen_recording( try: data = json.loads(r.stdout) except json.JSONDecodeError: - self.fail( - f"call check_permissions did not output JSON.\n" - f"stdout: {r.stdout!r}" - ) + self.fail(f"call check_permissions did not output JSON.\n" f"stdout: {r.stdout!r}") self.assertIn("accessibility", data) self.assertIn("screen_recording", data) @@ -351,8 +357,11 @@ def test_call_get_config_returns_config_object(self) -> None: data = json.loads(r.stdout) # Config must have at least one of the known top-level keys. known_keys = { - "schema_version", "capture_mode", "max_image_dimension", - "agent_cursor", "telemetry_enabled", + "schema_version", + "capture_mode", + "max_image_dimension", + "agent_cursor", + "telemetry_enabled", } self.assertTrue( known_keys.intersection(data.keys()), @@ -407,8 +416,12 @@ def test_call_get_window_state_for_calculator(self) -> None: # get_window_state rs = _run( - [self.binary, "call", "get_window_state", - f'{{"pid": {pid}, "window_id": {window_id}}}'], + [ + self.binary, + "call", + "get_window_state", + f'{{"pid": {pid}, "window_id": {window_id}}}', + ], timeout=30, ) self.assertEqual(rs.returncode, 0, f"get_window_state stderr: {rs.stderr}") @@ -427,7 +440,7 @@ def test_call_press_key_missing_pid_exits_1(self) -> None: self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_click_missing_required_args_exits_1(self) -> None: - r = _run([self.binary, "call", "click", '{}']) + r = _run([self.binary, "call", "click", "{}"]) self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_type_text_missing_pid_exits_1(self) -> None: @@ -439,11 +452,11 @@ def test_call_hotkey_missing_pid_exits_1(self) -> None: self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_scroll_missing_required_args_exits_1(self) -> None: - r = _run([self.binary, "call", "scroll", '{}']) + r = _run([self.binary, "call", "scroll", "{}"]) self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_drag_missing_required_args_exits_1(self) -> None: - r = _run([self.binary, "call", "drag", '{}']) + r = _run([self.binary, "call", "drag", "{}"]) self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_get_window_state_missing_pid_exits_1(self) -> None: @@ -451,7 +464,7 @@ def test_call_get_window_state_missing_pid_exits_1(self) -> None: self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") def test_call_launch_app_no_args_exits_1(self) -> None: - r = _run([self.binary, "call", "launch_app", '{}']) + r = _run([self.binary, "call", "launch_app", "{}"]) self.assertEqual(r.returncode, 1, f"stdout: {r.stdout}") # ── CLI: implicit call (tool name as first positional) ──────────────────── @@ -853,10 +866,16 @@ def test_mcp_get_config_roundtrip(self) -> None: with self._mcp() as c: result = c.call_tool("get_config") sc = result.get("structuredContent", result) - known = {"schema_version", "capture_mode", "max_image_dimension", - "agent_cursor", "telemetry_enabled"} - self.assertTrue(known.intersection(sc.keys()), - f"get_config has no known keys: {list(sc.keys())}") + known = { + "schema_version", + "capture_mode", + "max_image_dimension", + "agent_cursor", + "telemetry_enabled", + } + self.assertTrue( + known.intersection(sc.keys()), f"get_config has no known keys: {list(sc.keys())}" + ) def test_mcp_set_config_max_image_dimension(self) -> None: with self._mcp() as c: @@ -881,6 +900,7 @@ def test_mcp_get_recording_state(self) -> None: def test_mcp_set_recording_enable_disable(self) -> None: import tempfile + with tempfile.TemporaryDirectory() as tmpdir: with self._mcp() as c: # Enable recording — output_dir is required. @@ -952,9 +972,7 @@ def test_mcp_launch_app_missing_args_raises_error(self) -> None: self._assert_tool_raises_mcp_error("launch_app", {}) def test_mcp_set_value_missing_pid_raises_error(self) -> None: - self._assert_tool_raises_mcp_error( - "set_value", {"element_index": 0, "value": "x"} - ) + self._assert_tool_raises_mcp_error("set_value", {"element_index": 0, "value": "x"}) def test_mcp_replay_trajectory_bad_dir_raises_error(self) -> None: self._assert_tool_raises_mcp_error( @@ -1000,9 +1018,7 @@ def test_mcp_get_window_state_has_required_sc_keys(self) -> None: wins = c.call_tool("list_windows", {"pid": pid})["structuredContent"]["windows"] if not wins: self.skipTest("Calculator has no windows") - state = c.call_tool("get_window_state", { - "pid": pid, "window_id": wins[0]["window_id"] - }) + state = c.call_tool("get_window_state", {"pid": pid, "window_id": wins[0]["window_id"]}) sc = state.get("structuredContent", state) for key in ("tree_markdown", "screenshot_width", "screenshot_height"): self.assertIn(key, sc, f"get_window_state structuredContent missing: {key!r}") @@ -1045,9 +1061,7 @@ def test_mcp_set_agent_cursor_motion_arc(self) -> None: # ── stdio MCP: zoom ─────────────────────────────────────────────────────── def test_mcp_zoom_missing_pid_raises_error(self) -> None: - self._assert_tool_raises_mcp_error( - "zoom", {"x1": 0, "y1": 0, "x2": 100, "y2": 100} - ) + self._assert_tool_raises_mcp_error("zoom", {"x1": 0, "y1": 0, "x2": 100, "y2": 100}) # ── stdio MCP: page ─────────────────────────────────────────────────────── @@ -1063,9 +1077,7 @@ def test_mcp_page_is_registered(self) -> None: self.assertIn("page", names, "'page' tool is not registered") def test_mcp_page_missing_pid_raises_error(self) -> None: - self._assert_tool_raises_mcp_error( - "page", {"action": "get_text"} - ) + self._assert_tool_raises_mcp_error("page", {"action": "get_text"}) # ── stdio MCP: Rust-only tools (missing in Swift) ──────────────────────── @@ -1133,9 +1145,7 @@ def test_mcp_browser_eval_is_registered(self) -> None: def test_mcp_browser_eval_missing_pid_raises_error(self) -> None: """browser_eval with no pid should return an MCP error.""" - self._assert_tool_raises_mcp_error( - "browser_eval", {"expression": "1+1"} - ) + self._assert_tool_raises_mcp_error("browser_eval", {"expression": "1+1"}) # ── stdio MCP: protocol-level contracts ────────────────────────────────── @@ -1173,6 +1183,7 @@ def test_mcp_protocol_version_echo(self) -> None: # ── concrete test classes (one per binary) ──────────────────────────────────── + class SwiftParityTests(_ParityMixin, unittest.TestCase): """Run the full parity suite against the Swift cua-driver binary.""" diff --git a/libs/cua-driver-rs/tests/integration/test_background_focus.py b/libs/cua-driver-rs/tests/integration/test_background_focus.py index b86e1fbe9a..c1aa23931f 100644 --- a/libs/cua-driver-rs/tests/integration/test_background_focus.py +++ b/libs/cua-driver-rs/tests/integration/test_background_focus.py @@ -45,9 +45,7 @@ # FocusMonitorApp lives in the Swift cua-driver tree (shared fixture). _FOCUS_APP_DIR = os.path.join(_LIBS_ROOT, "cua-driver", "Tests", "FocusMonitorApp") _FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") -_FOCUS_APP_EXE = os.path.join( - _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" -) +_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp") _LOSS_FILE = "/tmp/focus_monitor_losses.txt" SAFARI_BUNDLE = "com.apple.Safari" @@ -58,11 +56,10 @@ # Helpers # --------------------------------------------------------------------------- + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -118,9 +115,7 @@ def _open_safari_to_html(client: DriverClient) -> int: def _get_page_text(client: DriverClient, pid: int) -> str: """Return the AX tree markdown for Safari.""" window_id = resolve_window_id(client, pid) - result = client.call_tool( - "get_window_state", {"pid": pid, "window_id": window_id} - ) + result = client.call_tool("get_window_state", {"pid": pid, "window_id": window_id}) return result.get("structuredContent", result).get("tree_markdown", "") @@ -128,7 +123,7 @@ def _find_element_index(tree_markdown: str, label: str) -> int | None: """Extract the first [N] element index from a line containing `label`.""" for line in tree_markdown.split("\n"): if label in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -139,15 +134,15 @@ def _find_calc_button(tree_markdown: str, label: str) -> int | None: for line in tree_markdown.split("\n"): if "AXButton" not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if not m: continue idx = int(m.group(1)) - if f'({label})' in line: + if f"({label})" in line: return idx if f'help="{label}' in line: return idx - if f'id={label}' in line: + if f"id={label}" in line: return idx return None @@ -165,6 +160,7 @@ def _activate_focus_monitor() -> None: # Safari background focus tests # --------------------------------------------------------------------------- + class BackgroundFocusTests(unittest.TestCase): """Click & type into backgrounded Safari without stealing focus.""" @@ -200,9 +196,9 @@ def setUpClass(cls) -> None: # Confirm FocusMonitorApp is frontmost. with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" # Baseline: 0 focus losses. losses = _read_focus_losses() @@ -210,7 +206,7 @@ def setUpClass(cls) -> None: @classmethod def tearDownClass(cls) -> None: - if hasattr(cls, '_focus_proc'): + if hasattr(cls, "_focus_proc"): cls._focus_proc.terminate() try: cls._focus_proc.wait(timeout=3) @@ -232,7 +228,8 @@ def setUp(self) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"FocusMonitorApp not frontmost at test start, got {active}", ) @@ -242,37 +239,45 @@ def _assert_no_focus_loss(self, label: str) -> None: losses = _read_focus_losses() with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) - focus_restored = (active == FOCUS_MONITOR_BUNDLE) + focus_restored = active == FOCUS_MONITOR_BUNDLE loss_delta = losses - self._losses_before - print(f" [{label}] losses: {self._losses_before}->{losses} " - f"(delta={loss_delta}), frontmost: {active}, " - f"restored: {focus_restored}") + print( + f" [{label}] losses: {self._losses_before}->{losses} " + f"(delta={loss_delta}), frontmost: {active}, " + f"restored: {focus_restored}" + ) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, - f"[{label}] Focus not restored — " - f"frontmost is {active}, not FocusMonitorApp", + active, + FOCUS_MONITOR_BUNDLE, + f"[{label}] Focus not restored — " f"frontmost is {active}, not FocusMonitorApp", ) def test_01_ax_click_button(self) -> None: """AX-click the 'Click Me' button in backgrounded Safari.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "Click Me", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "Click Me", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f"\n filtered tree:\n{tree}") idx = _find_element_index(tree, "Click Me") self.assertIsNotNone(idx, "Could not find 'Click Me' button in AX tree") - result = c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "element_index": idx, - }) + result = c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "element_index": idx, + }, + ) print(f" click result: {result}") time.sleep(0.5) @@ -287,11 +292,14 @@ def test_02_type_text_chars(self) -> None: """Type into the text field in backgrounded Safari via keystroke synthesis.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "AXTextField", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "AXTextField", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f"\n filtered tree:\n{tree}") @@ -299,19 +307,23 @@ def test_02_type_text_chars(self) -> None: idx = None for line in tree.split("\n"): if "AXTextField" in line and "smart search field" not in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: idx = int(m.group(1)) break if idx is None: - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") for line in tree.split("\n"): if "AXTextField" in line and "smart search field" not in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: idx = int(m.group(1)) break @@ -320,18 +332,24 @@ def test_02_type_text_chars(self) -> None: print(f" text field element_index: {idx}") # Focus the text field via AX click first. - c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) # Type via keystroke synthesis. - result = c.call_tool("type_text_chars", { - "pid": self._safari_pid, - "text": "hello bg", - }) + result = c.call_tool( + "type_text_chars", + { + "pid": self._safari_pid, + "text": "hello bg", + }, + ) print(f" type_text_chars result: {result}") has_text = False @@ -339,7 +357,7 @@ def test_02_type_text_chars(self) -> None: time.sleep(0.5) with DriverClient(self.binary) as c: tree = _get_page_text(c, self._safari_pid) - has_text = ("hello bg" in tree) + has_text = "hello bg" in tree if has_text: break print(f" attempt {attempt+1}: text not yet visible in tree") @@ -353,11 +371,14 @@ def test_03_pixel_click_no_focus_steal(self) -> None: """Pixel-click backgrounded Safari — verify no focus steal.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "Click Me", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "Click Me", + }, + ) sc = snap.get("structuredContent", snap) width = sc.get("screenshot_width", 0) height = sc.get("screenshot_height", 0) @@ -367,12 +388,15 @@ def test_03_pixel_click_no_focus_steal(self) -> None: y = int(height * 0.35) print(f" pixel click at ({x}, {y})") - result = c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "x": x, - "y": y, - }) + result = c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "x": x, + "y": y, + }, + ) print(f" click result: {result}") time.sleep(0.5) @@ -381,9 +405,13 @@ def test_03_pixel_click_no_focus_steal(self) -> None: def test_04_press_key_tab(self) -> None: """Send Tab key to backgrounded Safari without stealing focus.""" with DriverClient(self.binary) as c: - result = c.call_tool("press_key", { - "pid": self._safari_pid, "key": "tab", - }) + result = c.call_tool( + "press_key", + { + "pid": self._safari_pid, + "key": "tab", + }, + ) print(f"\n press_key result: {result}") time.sleep(0.3) @@ -394,6 +422,7 @@ def test_04_press_key_tab(self) -> None: # Calculator AX-click background tests # --------------------------------------------------------------------------- + class CalculatorBackgroundClickTests(unittest.TestCase): """AX-click backgrounded Calculator without stealing focus.""" @@ -429,13 +458,13 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" @classmethod def tearDownClass(cls) -> None: - if hasattr(cls, '_focus_proc'): + if hasattr(cls, "_focus_proc"): cls._focus_proc.terminate() try: cls._focus_proc.wait(timeout=3) @@ -461,12 +490,14 @@ def _assert_no_focus_loss(self, label: str) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) loss_delta = losses - self._losses_before - print(f" [{label}] losses: {self._losses_before}->{losses} " - f"(delta={loss_delta}), frontmost: {active}") + print( + f" [{label}] losses: {self._losses_before}->{losses} " + f"(delta={loss_delta}), frontmost: {active}" + ) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, - f"[{label}] Focus not restored — " - f"frontmost is {active}, not FocusMonitorApp", + active, + FOCUS_MONITOR_BUNDLE, + f"[{label}] Focus not restored — " f"frontmost is {active}, not FocusMonitorApp", ) def test_01_ax_click_2_plus_2(self) -> None: @@ -490,18 +521,24 @@ def test_01_ax_click_2_plus_2(self) -> None: self.assertIsNotNone(btn_eq, "'Equals' button not found") for idx in [btn_2, btn_add, btn_2, btn_eq]: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" result tree:\n{tree}") @@ -513,25 +550,36 @@ def test_02_ax_click_3_plus_4(self) -> None: with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) # Clear first - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_c = _find_calc_button(tree, "All Clear") if btn_c is None: btn_c = _find_calc_button(tree, "Clear") if btn_c is not None: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn_c, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_c, + }, + ) time.sleep(0.3) # Find buttons for 3+4= - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_3 = _find_calc_button(tree, "3") btn_add = _find_calc_button(tree, "Add") @@ -545,18 +593,24 @@ def test_02_ax_click_3_plus_4(self) -> None: self.assertIsNotNone(btn_eq, "'Equals' button not found") for idx in [btn_3, btn_add, btn_4, btn_eq]: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" result tree:\n{tree}") diff --git a/libs/cua-driver-rs/tests/integration/test_cli.py b/libs/cua-driver-rs/tests/integration/test_cli.py index 152e5c7a81..51b29e6f38 100644 --- a/libs/cua-driver-rs/tests/integration/test_cli.py +++ b/libs/cua-driver-rs/tests/integration/test_cli.py @@ -133,14 +133,19 @@ class ServeDaemonTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.binary = default_binary_path() - import tempfile, os + import os + import tempfile + cls._sock_file = tempfile.mktemp(suffix=".sock", prefix="cua-driver-test-") def _start_daemon(self): - import subprocess, time + import subprocess + import time + proc = subprocess.Popen( [self.binary, "serve", "--socket", self._sock_file], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, ) # Wait for daemon to bind. for _ in range(30): @@ -152,6 +157,7 @@ def _start_daemon(self): def test_status_exits_1_when_no_daemon(self) -> None: import tempfile + sock = tempfile.mktemp(suffix=".sock", prefix="cua-noexist-") r = _run([self.binary, "status", "--socket", sock]) self.assertEqual(r.returncode, 1) @@ -159,6 +165,7 @@ def test_status_exits_1_when_no_daemon(self) -> None: def test_stop_exits_1_when_no_daemon(self) -> None: import tempfile + sock = tempfile.mktemp(suffix=".sock", prefix="cua-noexist2-") r = _run([self.binary, "stop", "--socket", sock]) self.assertEqual(r.returncode, 1) @@ -178,7 +185,9 @@ def test_serve_status_stop_lifecycle(self) -> None: self.assertEqual(r.returncode, 0, f"stop stderr: {r.stderr}") # Status should now show not running. - import time; time.sleep(0.2) + import time + + time.sleep(0.2) r = _run([self.binary, "status", "--socket", self._sock_file]) self.assertEqual(r.returncode, 1) finally: diff --git a/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py b/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py index 83585af4ae..7229c35c7a 100644 --- a/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py +++ b/libs/cua-driver-rs/tests/integration/test_concurrent_drivers.py @@ -46,9 +46,7 @@ def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -79,10 +77,10 @@ def _find_calc_button(tree: str, label: str) -> int | None: for line in tree.split("\n"): if "AXButton" not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if not m: continue - if f'({label})' in line or f'id={label}' in line: + if f"({label})" in line or f"id={label}" in line: return int(m.group(1)) return None @@ -125,16 +123,16 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" losses = _read_focus_losses() assert losses == 0, f"Expected 0 focus losses at start, got {losses}" @classmethod def tearDownClass(cls) -> None: - if hasattr(cls, '_focus_proc'): + if hasattr(cls, "_focus_proc"): cls._focus_proc.terminate() try: cls._focus_proc.wait(timeout=3) @@ -153,15 +151,24 @@ def setUp(self) -> None: # Clear Calculator before each test so previous display state doesn't interfere. with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_ac = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear") if btn_ac is not None: - c.call_tool("click", { - "pid": self._calc_pid, "window_id": window_id, "element_index": btn_ac, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_ac, + }, + ) time.sleep(0.2) subprocess.run( @@ -186,9 +193,13 @@ def run_driver_a() -> None: with DriverClient(self.binary) as ca: # Driver A: AX-click 2 + 2 = on Calculator. window_id = resolve_window_id(ca, self._calc_pid) - snap = ca.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = ca.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_2 = _find_calc_button(tree, "2") btn_add = _find_calc_button(tree, "Add") @@ -197,16 +208,23 @@ def run_driver_a() -> None: driver_a_error.append("Missing calculator buttons") return for idx in [btn_2, btn_add, btn_2, btn_eq]: - ca.call_tool("click", { + ca.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) + time.sleep(0.15) + snap2 = ca.call_tool( + "get_window_state", + { "pid": self._calc_pid, "window_id": window_id, - "element_index": idx, - }) - time.sleep(0.15) - snap2 = ca.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - "query": "AXStaticText", - }) + "query": "AXStaticText", + }, + ) tree2 = snap2.get("structuredContent", snap2).get("tree_markdown", "") driver_a_result["tree"] = tree2 except Exception as e: @@ -266,7 +284,8 @@ def run_driver_b() -> None: active = frontmost_bundle_id(c) print(f" losses: {self._losses_before}->{losses}, frontmost: {active}") self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"Focus stolen during concurrent operation — frontmost is {active}", ) @@ -276,15 +295,24 @@ def test_sequential_driver_reuse(self) -> None: # First session — clear Calculator. with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_ac = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear") if btn_ac is not None: - c.call_tool("click", { - "pid": self._calc_pid, "window_id": window_id, "element_index": btn_ac, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_ac, + }, + ) time.sleep(0.2) time.sleep(0.2) @@ -292,19 +320,33 @@ def test_sequential_driver_reuse(self) -> None: # Second session — press 5. with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_5 = _find_calc_button(tree, "5") self.assertIsNotNone(btn_5, "Could not find '5' button in fresh session") - c.call_tool("click", { - "pid": self._calc_pid, "window_id": window_id, "element_index": btn_5, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_5, + }, + ) time.sleep(0.3) - snap2 = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, "query": "AXStaticText", - }) + snap2 = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree2 = snap2.get("structuredContent", snap2).get("tree_markdown", "") print(f"\n After pressing 5: {tree2[:200]}") @@ -315,7 +357,8 @@ def test_sequential_driver_reuse(self) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"Focus stolen — frontmost is {active}", ) diff --git a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py index 910b5e0b2c..b0721567ca 100644 --- a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py +++ b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py @@ -36,10 +36,10 @@ CALCULATOR_BUNDLE = "com.apple.calculator" # Colour tolerances for detection -_MAGENTA_MIN = (200, 0, 200) # R≥200, G≤55, B≥200 +_MAGENTA_MIN = (200, 0, 200) # R≥200, G≤55, B≥200 _MAGENTA_MAX = (255, 55, 255) -_YELLOW_MIN = (200, 200, 0) # R≥200, G≥200, B≤55 -_YELLOW_MAX = (255, 255, 55) +_YELLOW_MIN = (200, 200, 0) # R≥200, G≥200, B≤55 +_YELLOW_MAX = (255, 255, 55) # Minimum fraction of pixels in the search region that must match the cursor colours. _MATCH_THRESHOLD = 0.002 # 0.2% of the search window @@ -51,7 +51,7 @@ def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, Only supports filter types 0 (None) and 1 (Sub) — which covers the screenshots produced by cua-driver's `screenshot` tool. Raises ValueError for unsupported formats. """ - if png_bytes[:8] != b'\x89PNG\r\n\x1a\n': + if png_bytes[:8] != b"\x89PNG\r\n\x1a\n": raise ValueError("Not a PNG file") pos = 8 @@ -62,18 +62,18 @@ def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, while pos < len(png_bytes): if pos + 8 > len(png_bytes): break - length = struct.unpack('>I', png_bytes[pos:pos+4])[0] - chunk_type = png_bytes[pos+4:pos+8] - data = png_bytes[pos+8:pos+8+length] + length = struct.unpack(">I", png_bytes[pos : pos + 4])[0] + chunk_type = png_bytes[pos + 4 : pos + 8] + data = png_bytes[pos + 8 : pos + 8 + length] pos += 12 + length - if chunk_type == b'IHDR': - width, height = struct.unpack('>II', data[:8]) + if chunk_type == b"IHDR": + width, height = struct.unpack(">II", data[:8]) bit_depth = data[8] color_type = data[9] - elif chunk_type == b'IDAT': + elif chunk_type == b"IDAT": idat_chunks.append(data) - elif chunk_type == b'IEND': + elif chunk_type == b"IEND": break if width == 0 or height == 0: @@ -81,7 +81,7 @@ def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, if bit_depth != 8: raise ValueError(f"Unsupported bit depth: {bit_depth}") - raw = zlib.decompress(b''.join(idat_chunks)) + raw = zlib.decompress(b"".join(idat_chunks)) # Channels: 2=RGB(3), 6=RGBA(4) channels = {2: 3, 6: 4}.get(color_type) @@ -96,7 +96,7 @@ def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, for _y in range(height): filt = raw[offset] offset += 1 - row = bytearray(raw[offset:offset+stride]) + row = bytearray(raw[offset : offset + stride]) offset += stride if filt == 0: @@ -118,14 +118,16 @@ def _decode_png_pixels(png_bytes: bytes) -> tuple[int, int, list[tuple[int, int, b = prev_row[i] c = prev_row[i - channels] if i >= channels else 0 p = a + b - c - pa = abs(p - a); pb = abs(p - b); pc = abs(p - c) + pa = abs(p - a) + pb = abs(p - b) + pc = abs(p - c) pr = a if pa <= pb and pa <= pc else (b if pb <= pc else c) row[i] = (row[i] + pr) & 0xFF else: raise ValueError(f"Unknown PNG filter type: {filt}") for i in range(0, stride, channels): - pixels.append((row[i], row[i+1], row[i+2])) + pixels.append((row[i], row[i + 1], row[i + 2])) prev_row = bytes(row) return width, height, pixels @@ -156,9 +158,11 @@ def _colour_in_region( continue r, g, b = pixels[idx] total += 1 - if (colour_min[0] <= r <= colour_max[0] and - colour_min[1] <= g <= colour_max[1] and - colour_min[2] <= b <= colour_max[2]): + if ( + colour_min[0] <= r <= colour_max[0] + and colour_min[1] <= g <= colour_max[1] + and colour_min[2] <= b <= colour_max[2] + ): matches += 1 return matches / total if total > 0 else 0.0 @@ -212,7 +216,9 @@ def test_custom_cursor_visible_after_click(self) -> None: # 4. Resolve the Calculator window and its screen bounds. win_id = resolve_window_id(client, self.calc_pid) - wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"]["windows"] + wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"][ + "windows" + ] win = next((w for w in wins if w["window_id"] == win_id), None) self.assertIsNotNone(win, "Could not find Calculator window") bounds = win["bounds"] @@ -220,12 +226,15 @@ def test_custom_cursor_visible_after_click(self) -> None: # 5. Click the center of the Calculator window (window-local coords). click_x = bounds["width"] / 2.0 click_y = bounds["height"] / 2.0 - r = client.call_tool("click", { - "pid": self.calc_pid, - "window_id": win_id, - "x": click_x, - "y": click_y, - }) + r = client.call_tool( + "click", + { + "pid": self.calc_pid, + "window_id": win_id, + "x": click_x, + "y": click_y, + }, + ) self.assertFalse(r.get("isError"), f"click failed: {r}") # 6. Compute expected screen coordinates of the click point. @@ -242,7 +251,7 @@ def test_custom_cursor_visible_after_click(self) -> None: if b64: png = base64.b64decode(b64) # PNG width is at bytes 16..20 in big-endian. - png_w = struct.unpack('>I', png[16:20])[0] + png_w = struct.unpack(">I", png[16:20])[0] scale = png_w / bounds["width"] if bounds["width"] > 0 else 1.0 else: scale = 1.0 @@ -274,16 +283,29 @@ def test_custom_cursor_visible_after_click(self) -> None: search_r = int(80 * ss_scale) magenta_frac = _colour_in_region( - pixels, img_w, img_h, sx, sy, search_r, - _MAGENTA_MIN, _MAGENTA_MAX, + pixels, + img_w, + img_h, + sx, + sy, + search_r, + _MAGENTA_MIN, + _MAGENTA_MAX, ) yellow_frac = _colour_in_region( - pixels, img_w, img_h, sx, sy, search_r, - _YELLOW_MIN, _YELLOW_MAX, + pixels, + img_w, + img_h, + sx, + sy, + search_r, + _YELLOW_MIN, + _YELLOW_MAX, ) self.assertGreater( - magenta_frac, _MATCH_THRESHOLD, + magenta_frac, + _MATCH_THRESHOLD, f"Magenta cursor colour NOT found near click point " f"(screen={screen_x:.0f},{screen_y:.0f} img={sx:.0f},{sy:.0f} " f"scale={ss_scale:.2f}). " @@ -292,7 +314,8 @@ def test_custom_cursor_visible_after_click(self) -> None: f"The overlay may be hidden or behind another window.", ) self.assertGreater( - yellow_frac, _MATCH_THRESHOLD, + yellow_frac, + _MATCH_THRESHOLD, f"Yellow cross NOT found in cursor region " f"(magenta={magenta_frac:.5f}, yellow={yellow_frac:.5f}). " f"Cursor shape may not have loaded correctly.", @@ -321,7 +344,9 @@ def test_default_cursor_visible_after_click(self) -> None: # 3. Resolve window and screen bounds. win_id = resolve_window_id(client, self.calc_pid) - wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"]["windows"] + wins = client.call_tool("list_windows", {"pid": self.calc_pid})["structuredContent"][ + "windows" + ] win = next((w for w in wins if w["window_id"] == win_id), None) self.assertIsNotNone(win, "Could not find Calculator window") bounds = win["bounds"] @@ -334,18 +359,21 @@ def test_default_cursor_visible_after_click(self) -> None: scr = client.call_tool("screenshot", {"pid": self.calc_pid, "window_id": win_id}) b64 = scr.get("content", [{}])[0].get("data", "") png = base64.b64decode(b64) if b64 else b"" - png_w = struct.unpack('>I', png[16:20])[0] if len(png) >= 24 else 0 + png_w = struct.unpack(">I", png[16:20])[0] if len(png) >= 24 else 0 scale = png_w / bounds["width"] if bounds["width"] > 0 and png_w > 0 else 1.0 except Exception: scale = 1.0 # 4. Click the centre of the window. - r = client.call_tool("click", { - "pid": self.calc_pid, - "window_id": win_id, - "x": click_x, - "y": click_y, - }) + r = client.call_tool( + "click", + { + "pid": self.calc_pid, + "window_id": win_id, + "x": click_x, + "y": click_y, + }, + ) self.assertFalse(r.get("isError"), f"click failed: {r}") # 5. Wait for 750ms glide + spring settle + a safety margin. @@ -373,16 +401,23 @@ def test_default_cursor_visible_after_click(self) -> None: # Default arrow: ice-blue/cyan — cursor_mid ≈ (94, 192, 232), bloom ≈ (188, 232, 252). # Accept any pixel that is clearly blue-cyan (G and B significantly exceed R). - _CYAN_MIN = (40, 150, 180) + _CYAN_MIN = (40, 150, 180) _CYAN_MAX = (180, 255, 255) cyan_frac = _colour_in_region( - pixels, img_w, img_h, sx, sy, search_r, - _CYAN_MIN, _CYAN_MAX, + pixels, + img_w, + img_h, + sx, + sy, + search_r, + _CYAN_MIN, + _CYAN_MAX, ) self.assertGreater( - cyan_frac, _MATCH_THRESHOLD, + cyan_frac, + _MATCH_THRESHOLD, f"Default ice-blue cursor NOT found near click point " f"(screen={screen_x:.0f},{screen_y:.0f} img_pt={sx:.0f},{sy:.0f} " f"scale={ss_scale:.2f}). " @@ -392,10 +427,13 @@ def test_default_cursor_visible_after_click(self) -> None: def test_set_agent_cursor_style_gradient_only(self) -> None: """set_agent_cursor_style with gradient_colors updates without error.""" - r = self.client.call_tool("set_agent_cursor_style", { - "gradient_colors": ["#FF0000", "#FF00FF", "#0000FF"], - "bloom_color": "#00FFFF", - }) + r = self.client.call_tool( + "set_agent_cursor_style", + { + "gradient_colors": ["#FF0000", "#FF00FF", "#0000FF"], + "bloom_color": "#00FFFF", + }, + ) self.assertFalse(r.get("isError"), f"gradient-only style update failed: {r}") text = r.get("content", [{}])[0].get("text", "") self.assertIn("✅", text) diff --git a/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py b/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py index f272bd859d..244d9761b5 100644 --- a/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py +++ b/libs/cua-driver-rs/tests/integration/test_double_click_delivery.py @@ -48,9 +48,7 @@ def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -143,15 +141,15 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" losses = _read_focus_losses() assert losses == 0, f"Expected 0 focus losses at start, got {losses}" @classmethod def tearDownClass(cls) -> None: - if hasattr(cls, '_focus_proc'): + if hasattr(cls, "_focus_proc"): cls._focus_proc.terminate() try: cls._focus_proc.wait(timeout=3) @@ -174,11 +172,14 @@ def setUp(self) -> None: tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear") if btn is not None: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn, + }, + ) time.sleep(0.3) subprocess.run( ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'], @@ -194,7 +195,8 @@ def _assert_no_focus_steal(self) -> None: active = frontmost_bundle_id(c) print(f" losses: {self._losses_before}->{losses}, frontmost: {active}") self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"Focus stolen — frontmost is {active}", ) @@ -202,18 +204,25 @@ def test_element_index_double_click_no_focus_steal(self) -> None: """double_click via element_index on Calculator '2' button — no focus steal.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") two_button = _find_calc_button(tree, "2") self.assertIsNotNone(two_button, "Could not locate Calculator's '2' AXButton") - result = c.call_tool("double_click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": two_button, - }) + result = c.call_tool( + "double_click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": two_button, + }, + ) self.assertIsNone(result.get("isError"), msg=result) text = result.get("content", [{}])[0].get("text", "") print(f"\n double_click result: {text}") @@ -225,9 +234,13 @@ def test_pixel_double_click_no_focus_steal(self) -> None: """double_click via pixel coords — no focus steal.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) sc = snap.get("structuredContent", snap) w = sc.get("screenshot_width", CALC_DEFAULT_W) h = sc.get("screenshot_height", CALC_DEFAULT_H) @@ -235,12 +248,15 @@ def test_pixel_double_click_no_focus_steal(self) -> None: bx, by = CALC_TWO_BUTTON_XY x = int(bx * w / CALC_DEFAULT_W) y = int(by * h / CALC_DEFAULT_H) - result = c.call_tool("double_click", { - "pid": self._calc_pid, - "window_id": window_id, - "x": x, - "y": y, - }) + result = c.call_tool( + "double_click", + { + "pid": self._calc_pid, + "window_id": window_id, + "x": x, + "y": y, + }, + ) self.assertIsNone(result.get("isError"), msg=result) text = result.get("content", [{}])[0].get("text", "") print(f"\n double_click pixel result: {text}") diff --git a/libs/cua-driver-rs/tests/integration/test_focus_steal_parity.py b/libs/cua-driver-rs/tests/integration/test_focus_steal_parity.py index 5f4dcc5a18..05669cca6a 100644 --- a/libs/cua-driver-rs/tests/integration/test_focus_steal_parity.py +++ b/libs/cua-driver-rs/tests/integration/test_focus_steal_parity.py @@ -217,9 +217,7 @@ def test_launch_with_url_preserves_frontmost(self) -> None: time.sleep(0.5) self.assertEqual(_frontmost_bundle_id(), self.PRIOR_BUNDLE) - result = self._launch( - {"bundle_id": "com.apple.Safari", "urls": ["about:blank"]} - ) + result = self._launch({"bundle_id": "com.apple.Safari", "urls": ["about:blank"]}) # NOTE: the Swift binary on macOS Sonoma+ fails this launch with # "The application 'Safari' could not be launched because it was # not found" — that's a Cryptex-app + URL-handoff regression on diff --git a/libs/cua-driver-rs/tests/integration/test_list_windows.py b/libs/cua-driver-rs/tests/integration/test_list_windows.py index a9f5c8d59b..79ee004fc5 100644 --- a/libs/cua-driver-rs/tests/integration/test_list_windows.py +++ b/libs/cua-driver-rs/tests/integration/test_list_windows.py @@ -25,7 +25,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from driver_client import DriverClient, default_binary_path - CALCULATOR_BUNDLE = "com.apple.calculator" @@ -53,13 +52,17 @@ def _call_list_windows(self, **args): def test_default_returns_layer_zero_windows_with_field_names(self): body = self._call_list_windows() windows = body["windows"] - self.assertGreater( - len(windows), 0, "expected at least Calculator's window" - ) + self.assertGreater(len(windows), 0, "expected at least Calculator's window") required = { - "window_id", "pid", "app_name", "title", "bounds", - "layer", "z_index", "is_on_screen", + "window_id", + "pid", + "app_name", + "title", + "bounds", + "layer", + "z_index", + "is_on_screen", } for w in windows: self.assertTrue( @@ -71,9 +74,7 @@ def test_default_returns_layer_zero_windows_with_field_names(self): def test_pid_filter_narrows_to_one_pid(self): body = self._call_list_windows(pid=self.calc_pid) windows = body["windows"] - self.assertGreater( - len(windows), 0, "Calculator should have at least one window" - ) + self.assertGreater(len(windows), 0, "Calculator should have at least one window") self.assertTrue( all(w["pid"] == self.calc_pid for w in windows), "pid filter leaked other pids into the result", @@ -103,7 +104,8 @@ def test_bounds_fields_present_and_non_negative(self): for field in ("x", "y", "width", "height"): self.assertIn(field, b, f"bounds missing '{field}'") self.assertGreaterEqual( - b[field], 0, + b[field], + 0, f"bounds.{field} should be non-negative, got {b[field]}", ) @@ -112,7 +114,8 @@ def test_z_index_positive(self): windows = body["windows"] for w in windows: self.assertGreater( - w["z_index"], 0, + w["z_index"], + 0, f"z_index should be > 0 (front-to-back ordering), got {w['z_index']}", ) diff --git a/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py b/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py index e7121daf0f..f55328a83f 100644 --- a/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py +++ b/libs/cua-driver-rs/tests/integration/test_overlay_z_order.py @@ -96,7 +96,7 @@ def setUp(self) -> None: time.sleep(0.5) def tearDown(self) -> None: - if hasattr(self, '_focus_proc'): + if hasattr(self, "_focus_proc"): self._focus_proc.terminate() try: self._focus_proc.wait(timeout=3) @@ -126,18 +126,20 @@ def test_overlay_at_normal_level_above_target(self) -> None: cy = b["height"] / 2.0 fg_wins_before = [ - w for w in all_before - if w["z_index"] > calc_z_before - and w["pid"] != driver_pid - and w.get("is_on_screen") + w + for w in all_before + if w["z_index"] > calc_z_before and w["pid"] != driver_pid and w.get("is_on_screen") ] - self.client.call_tool("click", { - "pid": self.calc_pid, - "window_id": calc_win_id, - "x": cx, - "y": cy, - }) + self.client.call_tool( + "click", + { + "pid": self.calc_pid, + "window_id": calc_win_id, + "x": cx, + "y": cy, + }, + ) # Let defensive-repin ticks fully settle. time.sleep(1.5) @@ -154,9 +156,7 @@ def test_overlay_at_normal_level_above_target(self) -> None: overlay_z = max(w["z_index"] for w in overlay_wins) calc_wins_after = [w for w in all_after if w["pid"] == self.calc_pid] - calc_z_after = ( - max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0 - ) + calc_z_after = max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0 self.assertGreater( overlay_z, calc_z_after, diff --git a/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py b/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py index 985047e868..04fcadf39e 100644 --- a/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py +++ b/libs/cua-driver-rs/tests/integration/test_pixel_click_delivery.py @@ -46,15 +46,15 @@ # Column centers (x): 29, 86, 144, 201 # Row centers (y): 133, 193, 253, 313, 373 CALC_BUTTONS = { - "1": (29, 313), "2": (86, 313), "3": (144, 313), + "1": (29, 313), + "2": (86, 313), + "3": (144, 313), } def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -85,10 +85,10 @@ def _find_calc_button(tree: str, label: str) -> int | None: for line in tree.split("\n"): if "AXButton" not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if not m: continue - if f'({label})' in line or f'id={label}' in line: + if f"({label})" in line or f"id={label}" in line: return int(m.group(1)) return None @@ -141,15 +141,15 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" losses = _read_focus_losses() assert losses == 0, f"Expected 0 focus losses at start, got {losses}" @classmethod def tearDownClass(cls) -> None: - if hasattr(cls, '_focus_proc'): + if hasattr(cls, "_focus_proc"): cls._focus_proc.terminate() try: cls._focus_proc.wait(timeout=3) @@ -173,11 +173,14 @@ def setUp(self) -> None: tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn = _find_calc_button(tree, "All Clear") or _find_calc_button(tree, "Clear") if btn is not None: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn, + }, + ) time.sleep(0.3) subprocess.run( ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'], @@ -203,12 +206,15 @@ def test_pixel_click_no_focus_steal(self) -> None: bx, by = CALC_BUTTONS[digit] x = int(bx * w / 230) y = int(by * h / 408) - result = c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "x": x, - "y": y, - }) + result = c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "x": x, + "y": y, + }, + ) print(f" {digit}: pixel ({x},{y})") time.sleep(0.15) @@ -220,7 +226,8 @@ def test_pixel_click_no_focus_steal(self) -> None: active = frontmost_bundle_id(c) print(f" losses: {self._losses_before}->{losses}, frontmost: {active}") self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"Focus stolen — frontmost is {active}", ) @@ -228,9 +235,13 @@ def test_ax_click_delivery(self) -> None: """AX-click 1+2 on backgrounded Calculator — verifies AX event delivery.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_1 = _find_calc_button(tree, "1") @@ -245,18 +256,24 @@ def test_ax_click_delivery(self) -> None: self.assertIsNotNone(btn_eq, "'Equals' button not found") for idx in [btn_1, btn_add, btn_2, btn_eq]: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" result tree:\n{tree}") @@ -267,7 +284,8 @@ def test_ax_click_delivery(self) -> None: active = frontmost_bundle_id(c) print(f" losses: {self._losses_before}->{losses}, frontmost: {active}") self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"Focus stolen by AX click — frontmost is {active}", ) diff --git a/libs/cua-driver-rs/tests/integration/v2/conftest.py b/libs/cua-driver-rs/tests/integration/v2/conftest.py index 47a163f35e..9481db066e 100644 --- a/libs/cua-driver-rs/tests/integration/v2/conftest.py +++ b/libs/cua-driver-rs/tests/integration/v2/conftest.py @@ -15,10 +15,10 @@ import os import subprocess import sys -import time import threading -import zipfile +import time import urllib.request +import zipfile from http.server import HTTPServer, SimpleHTTPRequestHandler from typing import Optional @@ -62,6 +62,7 @@ # ── binary ──────────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def binary() -> str: return default_binary_path() @@ -69,17 +70,20 @@ def binary() -> str: # ── driver ──────────────────────────────────────────────────────────────────── + @pytest.fixture def driver(binary): """Yield a started Driver instance.""" sys.path.insert(0, _HERE) from harness.driver import Driver + with Driver(binary) as d: yield d # ── focus monitor ───────────────────────────────────────────────────────────── + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): build_sh = os.path.join(_FOCUS_APP_DIR, "build.sh") @@ -152,8 +156,11 @@ def activate_focus_monitor(focus_monitor): """Re-activate FocusMonitorApp before each test (by pid).""" _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -161,12 +168,14 @@ def activate_focus_monitor(focus_monitor): # ── UX guard ────────────────────────────────────────────────────────────────── + @pytest.fixture def ux_guard(focus_monitor): """Start UXMonitor before the test, call assert_clean() after.""" _, sentinel_pid = focus_monitor sys.path.insert(0, _HERE) from harness.monitor import UXMonitor + mon = UXMonitor(sentinel_pid=sentinel_pid) mon.start() yield mon @@ -176,6 +185,7 @@ def ux_guard(focus_monitor): # ── local HTML server ───────────────────────────────────────────────────────── + class _SilentHandler(SimpleHTTPRequestHandler): def log_message(self, *args): pass @@ -184,9 +194,7 @@ def log_message(self, *args): @pytest.fixture(scope="session") def html_server(): """Serve the assets/ directory over HTTP; return base URL.""" - handler = lambda *args, **kwargs: _SilentHandler( - *args, directory=_ASSETS_DIR, **kwargs - ) + handler = lambda *args, **kwargs: _SilentHandler(*args, directory=_ASSETS_DIR, **kwargs) server = HTTPServer(("127.0.0.1", 0), handler) port = server.server_address[1] thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -197,6 +205,7 @@ def html_server(): # ── Tauri / Electron helpers ────────────────────────────────────────────────── + def _download_and_extract(url: str, dest_dir: str, marker_path: str) -> None: """Download a zip and extract it; skip if marker_path already exists. @@ -229,6 +238,7 @@ def _download_binary(url: str, dest_path: str) -> None: # ── Tauri app ───────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def tauri_app(): """Download, launch, and yield (proc, pid, base_url) for the Tauri test app. @@ -262,6 +272,7 @@ def tauri_app(): # ── Electron app ────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def electron_app(html_server): """Download, launch, and yield (proc, pid, base_url) for the Electron test app. @@ -297,7 +308,8 @@ def electron_app(html_server): # Get the real pid of the launched app. out = subprocess.run( ["pgrep", "-f", "desktop-test-app-electron"], - capture_output=True, text=True, + capture_output=True, + text=True, ).stdout.strip() pid = int(out.split("\n")[0]) if out else 0 diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/cv.py b/libs/cua-driver-rs/tests/integration/v2/harness/cv.py index d254ebfffe..d2bd34fa72 100644 --- a/libs/cua-driver-rs/tests/integration/v2/harness/cv.py +++ b/libs/cua-driver-rs/tests/integration/v2/harness/cv.py @@ -19,8 +19,9 @@ from typing import Optional, Tuple try: - from PIL import Image, ImageChops import numpy as np + from PIL import Image, ImageChops + _PIL_AVAILABLE = True except ImportError: _PIL_AVAILABLE = False @@ -90,7 +91,7 @@ def find_template( n_norm = (n_arr - n_mean) / n_std # Sliding-window NCC via FFT - from numpy.fft import fft2, ifft2, fftshift + from numpy.fft import fft2, fftshift, ifft2 # Zero-pad haystack to same size for FFT correlation pad_h = H + th - 1 @@ -102,12 +103,13 @@ def find_template( # Local haystack std for normalisation from numpy.lib.stride_tricks import sliding_window_view + windows = sliding_window_view(h_arr, (th, tw)) local_std = windows.reshape(-1, th * tw).std(axis=1).reshape(H - th + 1, W - tw + 1) local_std[local_std < 1e-6] = 1.0 # Trim correlation to valid region - corr_valid = corr[th - 1:th - 1 + (H - th + 1), tw - 1:tw - 1 + (W - tw + 1)] + corr_valid = corr[th - 1 : th - 1 + (H - th + 1), tw - 1 : tw - 1 + (W - tw + 1)] ncc = corr_valid / (local_std * th * tw) best_idx = np.argmax(ncc) diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/driver.py b/libs/cua-driver-rs/tests/integration/v2/harness/driver.py index cdec18f487..c59bb866b1 100644 --- a/libs/cua-driver-rs/tests/integration/v2/harness/driver.py +++ b/libs/cua-driver-rs/tests/integration/v2/harness/driver.py @@ -29,7 +29,7 @@ _INTEG_DIR = os.path.dirname(_V2_DIR) sys.path.insert(0, _INTEG_DIR) -from driver_client import DriverClient, default_binary_path, MCPCallError # noqa: E402 +from driver_client import DriverClient, MCPCallError, default_binary_path # noqa: E402 @dataclass @@ -43,9 +43,10 @@ class WindowState: def find_element(self, label: str) -> Optional[int]: """Return the first element index whose line contains `label`.""" import re + for line in self.tree.split("\n"): if label in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -53,12 +54,13 @@ def find_element(self, label: str) -> Optional[int]: def find_button(self, label: str) -> Optional[int]: """Return the first AXButton element index matching `label`.""" import re + for line in self.tree.split("\n"): if "AXButton" not in line: continue if label not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -66,12 +68,13 @@ def find_button(self, label: str) -> Optional[int]: def find_text_field(self, skip_url_bar: bool = True) -> Optional[int]: """Return the first AXTextField index (skipping Safari URL bar).""" import re + for line in self.tree.split("\n"): if "AXTextField" not in line: continue if skip_url_bar and "smart search field" in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -79,6 +82,7 @@ def find_text_field(self, skip_url_bar: bool = True) -> Optional[int]: def ax_value(self, element_index: int) -> Optional[str]: """Return the value= attribute of the element at `element_index`.""" import re + prefix = f"[{element_index}]" for line in self.tree.split("\n"): if prefix not in line: @@ -138,8 +142,10 @@ def _area(w: dict) -> float: # Filter to on-screen windows with meaningful size. candidates = [ - w for w in windows - if w.get("is_on_screen") and w.get("on_current_space") is not False + w + for w in windows + if w.get("is_on_screen") + and w.get("on_current_space") is not False and _area(w) >= min_area ] if not candidates: @@ -234,14 +240,17 @@ def scroll( delta_x: float = 0.0, delta_y: float = -3.0, ) -> dict: - return self._c().call_tool("scroll", { - "pid": pid, - "window_id": window_id, - "x": x, - "y": y, - "delta_x": delta_x, - "delta_y": delta_y, - }) + return self._c().call_tool( + "scroll", + { + "pid": pid, + "window_id": window_id, + "x": x, + "y": y, + "delta_x": delta_x, + "delta_y": delta_y, + }, + ) def launch_app(self, bundle_id: str) -> dict: return self._c().call_tool("launch_app", {"bundle_id": bundle_id}) diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py b/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py index c3469a5451..36e61501f7 100644 --- a/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py +++ b/libs/cua-driver-rs/tests/integration/v2/harness/monitor.py @@ -22,19 +22,18 @@ from dataclasses import dataclass, field from typing import Optional - # Try to import PyObjC; fall back to no-op monitor on systems without it. try: - from AppKit import NSWorkspace - from AppKit import NSEvent + from AppKit import NSEvent, NSWorkspace from Quartz import ( CGWindowListCopyWindowInfo, - kCGWindowListOptionOnScreenOnly, kCGNullWindowID, kCGWindowLayer, - kCGWindowOwnerPID, + kCGWindowListOptionOnScreenOnly, kCGWindowName, + kCGWindowOwnerPID, ) + _PYOBJC_AVAILABLE = True except ImportError: _PYOBJC_AVAILABLE = False @@ -49,7 +48,7 @@ @dataclass class Violation: - kind: str # "cursor" | "frontmost" | "overlay_z" + kind: str # "cursor" | "frontmost" | "overlay_z" detail: str timestamp: float = field(default_factory=time.monotonic) @@ -189,9 +188,7 @@ def _check_frontmost(self) -> None: def _check_overlay_z(self) -> None: """Check that no cua-driver overlay window is above foreground windows.""" - wins = CGWindowListCopyWindowInfo( - kCGWindowListOptionOnScreenOnly, kCGNullWindowID - ) + wins = CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly, kCGNullWindowID) if not wins: return @@ -204,8 +201,9 @@ def _check_overlay_z(self) -> None: name = str(w.get(kCGWindowName, "") or "") layer = int(w.get(kCGWindowLayer, 0)) - is_overlay = name in ("cua-agent-cursor", "cua-overlay") or \ - "cua-driver" in str(name).lower() + is_overlay = ( + name in ("cua-agent-cursor", "cua-overlay") or "cua-driver" in str(name).lower() + ) if is_overlay: overlay_layers.append(layer) diff --git a/libs/cua-driver-rs/tests/integration/v2/harness/tree.py b/libs/cua-driver-rs/tests/integration/v2/harness/tree.py index adc2c4fe34..b69f965c7d 100644 --- a/libs/cua-driver-rs/tests/integration/v2/harness/tree.py +++ b/libs/cua-driver-rs/tests/integration/v2/harness/tree.py @@ -24,7 +24,7 @@ def _lines(tree: str) -> list[tuple[int, str]]: """Return (element_index, raw_line) pairs for every indexed line.""" result: list[tuple[int, str]] = [] for line in tree.split("\n"): - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: result.append((int(m.group(1)), line)) return result diff --git a/libs/cua-driver-rs/tests/integration/v2/test_blender.py b/libs/cua-driver-rs/tests/integration/v2/test_blender.py index b0455e6055..64bf9aa72b 100644 --- a/libs/cua-driver-rs/tests/integration/v2/test_blender.py +++ b/libs/cua-driver-rs/tests/integration/v2/test_blender.py @@ -19,19 +19,19 @@ from __future__ import annotations +import json +import os import subprocess import sys import time -import os -import json import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree -from harness.cv import decode, crop, diff_ratio, save_reference, load_reference +from harness.cv import crop, decode, diff_ratio, load_reference, save_reference +from harness.driver import Driver BLENDER_BUNDLE = "org.blenderfoundation.blender" _ASSETS_BLENDER = os.path.join(_HERE, "assets", "blender") @@ -53,6 +53,7 @@ def _save_bboxes(bboxes: dict) -> None: # ── module-level Blender setup ──────────────────────────────────────────────── + @pytest.fixture(scope="module") def blender_pid(binary, focus_monitor): """Launch Blender in background; return its pid.""" @@ -84,8 +85,11 @@ def blender_pid(binary, focus_monitor): def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -93,6 +97,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestBlenderWindow: """Basic window state and AX tree tests.""" diff --git a/libs/cua-driver-rs/tests/integration/v2/test_chrome.py b/libs/cua-driver-rs/tests/integration/v2/test_chrome.py index 8cef3b3afd..db5229ed42 100644 --- a/libs/cua-driver-rs/tests/integration/v2/test_chrome.py +++ b/libs/cua-driver-rs/tests/integration/v2/test_chrome.py @@ -9,24 +9,25 @@ from __future__ import annotations +import os import re import subprocess import sys import time -import os import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver def _parse_clicks(tree_text: str) -> int: - m = re.search(r'clicks:\s*(\d+)', tree_text) + m = re.search(r"clicks:\s*(\d+)", tree_text) return int(m.group(1)) if m else 0 + CHROME_BUNDLE = "com.google.Chrome" FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp" @@ -70,8 +71,11 @@ def chrome_pid(binary, html_server, focus_monitor): # Hand focus to FocusMonitorApp by pid to avoid activating stale instances _, fm_pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {fm_pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {fm_pid}) to true', + ], check=False, ) time.sleep(0.5) @@ -86,8 +90,11 @@ def chrome_pid(binary, html_server, focus_monitor): def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -95,13 +102,16 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestChromeButton: def test_click_increments_counter(self, driver, chrome_pid, ux_guard): wid = driver.find_window(chrome_pid) full = driver.get_window_state(chrome_pid, wid) idx = full.find_element("Click Me") - assert idx is not None, f"'Click Me' not found in tree (first 800 chars):\n{full.tree[:800]}" + assert ( + idx is not None + ), f"'Click Me' not found in tree (first 800 chars):\n{full.tree[:800]}" count_before = _parse_clicks(full.tree) driver.click(chrome_pid, wid, element_index=idx) @@ -110,7 +120,9 @@ def test_click_increments_counter(self, driver, chrome_pid, ux_guard): after = driver.get_window_state(chrome_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 1, f"Counter did not increment: {count_before} → {count_after}" + assert ( + count_after == count_before + 1 + ), f"Counter did not increment: {count_before} → {count_after}" def test_click_increments_twice(self, driver, chrome_pid, ux_guard): wid = driver.find_window(chrome_pid) @@ -127,7 +139,9 @@ def test_click_increments_twice(self, driver, chrome_pid, ux_guard): after = driver.get_window_state(chrome_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 2, f"Counter should have incremented by 2: {count_before} → {count_after}" + assert ( + count_after == count_before + 2 + ), f"Counter should have incremented by 2: {count_before} → {count_after}" class TestChromeTextInput: @@ -160,6 +174,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, chrome_pid, ux_guard): assert mid.screenshot_b64, "No screenshot" from harness.cv import decode, diff_ratio + before_img = decode(mid.screenshot_b64) x = mid.screenshot_width // 2 diff --git a/libs/cua-driver-rs/tests/integration/v2/test_electron.py b/libs/cua-driver-rs/tests/integration/v2/test_electron.py index 5c9818ceca..777ad4eaa4 100644 --- a/libs/cua-driver-rs/tests/integration/v2/test_electron.py +++ b/libs/cua-driver-rs/tests/integration/v2/test_electron.py @@ -11,23 +11,24 @@ from __future__ import annotations +import os import subprocess import sys import time -import os import pytest try: import requests as _requests + _REQUESTS_AVAILABLE = True except ImportError: _REQUESTS_AVAILABLE = False _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver _ELECTRON_API = "http://localhost:6769" @@ -55,8 +56,11 @@ def _http_reset(base_url: str = _ELECTRON_API) -> None: def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -64,6 +68,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestElectronButton: def test_click_increments_counter(self, driver, electron_app, ux_guard): _, pid, base_url = electron_app @@ -103,11 +108,14 @@ def test_type_text(self, driver, electron_app, ux_guard): try: driver.click(pid, wid, element_index=idx) time.sleep(0.4) - driver.call_tool("type_text_chars", { - "pid": pid, - "text": "hello electron", - "delay_ms": 30, - }) + driver.call_tool( + "type_text_chars", + { + "pid": pid, + "text": "hello electron", + "delay_ms": 30, + }, + ) except Exception as e: events = _http_events(base_url) key_events = [ev for ev in events if ev.get("type") in ("keydown", "keypress", "input")] @@ -135,7 +143,9 @@ def test_type_text(self, driver, electron_app, ux_guard): if after is None or not after.tree: pytest.skip("AX tree unavailable after typing (Electron AX rebuild timeout)") - print(f"\n after tree length: {len(after.tree)}, has text: {after.has_text('hello electron')}") + print( + f"\n after tree length: {len(after.tree)}, has text: {after.has_text('hello electron')}" + ) assert after.has_text("hello electron"), f"Text not in AX tree:\n{after.tree[:500]}" @@ -162,6 +172,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, electron_app, ux_guard) assert mid.screenshot_b64, "No screenshot after 5 retries" from harness.cv import decode, diff_ratio + before_img = decode(mid.screenshot_b64) x = mid.screenshot_width // 2 diff --git a/libs/cua-driver-rs/tests/integration/v2/test_safari.py b/libs/cua-driver-rs/tests/integration/v2/test_safari.py index 5a33846482..a2e39becab 100644 --- a/libs/cua-driver-rs/tests/integration/v2/test_safari.py +++ b/libs/cua-driver-rs/tests/integration/v2/test_safari.py @@ -13,31 +13,33 @@ from __future__ import annotations +import os +import re import subprocess import sys import time -import os -import re import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver def _parse_clicks(tree_text: str) -> int: """Extract the 'clicks: N' counter value from the AX tree.""" - m = re.search(r'clicks:\s*(\d+)', tree_text) + m = re.search(r"clicks:\s*(\d+)", tree_text) return int(m.group(1)) if m else 0 + SAFARI_BUNDLE = "com.apple.Safari" FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp" # ── module-level Safari setup ───────────────────────────────────────────────── + @pytest.fixture(scope="module") def safari_pid(binary, html_server, focus_monitor): """Open Safari to test_page.html; return its pid. @@ -68,8 +70,11 @@ def _reactivate_focus(focus_monitor): """Restore FocusMonitorApp as frontmost before each test (by pid).""" _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -77,6 +82,7 @@ def _reactivate_focus(focus_monitor): # ── helpers ─────────────────────────────────────────────────────────────────── + def _get_state(d: Driver, pid: int, safari_pid_fixture) -> "harness.driver.WindowState": wid = d.find_window(pid) return d.get_window_state(pid, wid) @@ -84,6 +90,7 @@ def _get_state(d: Driver, pid: int, safari_pid_fixture) -> "harness.driver.Windo # ── tests ───────────────────────────────────────────────────────────────────── + class TestSafariButton: """AX-click the 'Click Me' button without stealing focus.""" @@ -102,9 +109,9 @@ def test_click_increments_counter(self, driver, safari_pid, ux_guard): after = driver.get_window_state(safari_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 1, ( - f"Counter did not increment: {count_before} → {count_after}" - ) + assert ( + count_after == count_before + 1 + ), f"Counter did not increment: {count_before} → {count_after}" def test_click_increments_twice(self, driver, safari_pid, ux_guard): wid = driver.find_window(safari_pid) @@ -123,9 +130,9 @@ def test_click_increments_twice(self, driver, safari_pid, ux_guard): after = driver.get_window_state(safari_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 2, ( - f"Counter should have incremented by 2: {count_before} → {count_after}" - ) + assert ( + count_after == count_before + 2 + ), f"Counter should have incremented by 2: {count_before} → {count_after}" class TestSafariTextInput: @@ -150,9 +157,7 @@ def test_type_text_appears_in_ax_tree(self, driver, safari_pid, ux_guard): time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("hello safari"), ( - f"Typed text not visible in AX tree:\n{after.tree}" - ) + assert after.has_text("hello safari"), f"Typed text not visible in AX tree:\n{after.tree}" def test_type_text_with_type_text_tool(self, driver, safari_pid, ux_guard): """Use type_text (not type_text_chars) and verify result.""" @@ -172,9 +177,7 @@ def test_type_text_with_type_text_tool(self, driver, safari_pid, ux_guard): time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("type_text test"), ( - f"type_text result not in AX tree:\n{after.tree}" - ) + assert after.has_text("type_text test"), f"type_text result not in AX tree:\n{after.tree}" class TestSafariCheckbox: @@ -194,9 +197,7 @@ def test_checkbox_toggle(self, driver, safari_pid, ux_guard): time.sleep(0.4) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("checked"), ( - f"Checkbox did not toggle to checked:\n{after.tree}" - ) + assert after.has_text("checked"), f"Checkbox did not toggle to checked:\n{after.tree}" class TestSafariSelect: @@ -223,16 +224,21 @@ def test_select_option_b(self, driver, safari_pid, ux_guard): driver.click(safari_pid, wid, element_index=opt_b) else: # Fallback: set value directly - driver.call_tool("set_value", { - "pid": safari_pid, "window_id": wid, - "element_index": idx, "value": "b", - }) + driver.call_tool( + "set_value", + { + "pid": safari_pid, + "window_id": wid, + "element_index": idx, + "value": "b", + }, + ) time.sleep(0.4) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("Option B") or after.has_text("selected: Option B"), ( - f"Dropdown did not select Option B:\n{after.tree}" - ) + assert after.has_text("Option B") or after.has_text( + "selected: Option B" + ), f"Dropdown did not select Option B:\n{after.tree}" class TestSafariTextarea: @@ -262,19 +268,22 @@ def test_textarea_input(self, driver, safari_pid, ux_guard): time.sleep(0.8) # longer settle for WKWebView textarea focus # Step 3: type — keystrokes go to the focused element - driver.call_tool("type_text_chars", { - "pid": safari_pid, - "text": "multi line", - "delay_ms": 50, - }) + driver.call_tool( + "type_text_chars", + { + "pid": safari_pid, + "text": "multi line", + "delay_ms": 50, + }, + ) time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) val = Tree.ax_value(after.tree, ta_idx) print(f"\n textarea AX value: {val!r}") - assert after.has_text("multi line"), ( - f"Textarea text not in AX tree; val={val!r};\n{after.tree[:800]}" - ) + assert after.has_text( + "multi line" + ), f"Textarea text not in AX tree; val={val!r};\n{after.tree[:800]}" class TestSafariCanvas: @@ -295,6 +304,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, safari_pid, ux_guard): print(f"\n screenshot: {before.screenshot_width}x{before.screenshot_height}") from harness.cv import decode, diff_ratio + before_img = decode(before.screenshot_b64) # Canvas is the last section on the page. With ~80px browser chrome diff --git a/libs/cua-driver-rs/tests/integration/v2/test_tauri.py b/libs/cua-driver-rs/tests/integration/v2/test_tauri.py index e1bb465f56..ed233bdc7c 100644 --- a/libs/cua-driver-rs/tests/integration/v2/test_tauri.py +++ b/libs/cua-driver-rs/tests/integration/v2/test_tauri.py @@ -11,23 +11,24 @@ from __future__ import annotations +import os import subprocess import sys import time -import os import pytest try: import requests as _requests + _REQUESTS_AVAILABLE = True except ImportError: _REQUESTS_AVAILABLE = False _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver _TAURI_API = "http://localhost:6769" @@ -55,8 +56,11 @@ def _http_reset(base_url: str = _TAURI_API) -> None: def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -64,6 +68,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestTauriButton: def test_button_click(self, driver, tauri_app, ux_guard): _, pid, base_url = tauri_app @@ -128,9 +133,9 @@ def test_type_text(self, driver, tauri_app, ux_guard): key_events = [e for e in events if e.get("type") in ("key_down", "keydown", "input")] print(f" key events: {len(key_events)}") - assert text_in_tree or len(key_events) >= len("tauri test"), ( - f"Text not found in AX tree and insufficient key events:\n{after.tree}" - ) + assert text_in_tree or len(key_events) >= len( + "tauri test" + ), f"Text not found in AX tree and insufficient key events:\n{after.tree}" class TestTauriNoFocusSteal: diff --git a/libs/cua-driver/Tests/integration/conftest.py b/libs/cua-driver/Tests/integration/conftest.py index e2c21409d6..ddfa5194f5 100644 --- a/libs/cua-driver/Tests/integration/conftest.py +++ b/libs/cua-driver/Tests/integration/conftest.py @@ -16,16 +16,16 @@ import os import subprocess import sys -import time import threading -from http.server import HTTPServer, SimpleHTTPRequestHandler +import time import urllib.request +from http.server import HTTPServer, SimpleHTTPRequestHandler import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) -_TESTS_DIR = os.path.dirname(_HERE) # Tests/ -_DRIVER_ROOT = os.path.dirname(_TESTS_DIR) # libs/cua-driver/ +_TESTS_DIR = os.path.dirname(_HERE) # Tests/ +_DRIVER_ROOT = os.path.dirname(_TESTS_DIR) # libs/cua-driver/ sys.path.insert(0, _HERE) sys.path.insert(0, os.path.join(_HERE, "harness")) @@ -59,6 +59,7 @@ # ── binary ──────────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def binary() -> str: return default_binary_path() @@ -66,16 +67,19 @@ def binary() -> str: # ── driver ──────────────────────────────────────────────────────────────────── + @pytest.fixture def driver(binary): """Yield a started Driver instance.""" from harness.driver import Driver + with Driver(binary) as d: yield d # ── focus monitor ───────────────────────────────────────────────────────────── + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): build_sh = os.path.join(_FOCUS_APP_DIR, "build.sh") @@ -142,9 +146,12 @@ def activate_focus_monitor(focus_monitor): """Re-activate FocusMonitorApp before each test (by pid).""" _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of ' - f'(first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of ' + f"(first process whose unix id is {pid}) to true", + ], check=False, ) time.sleep(0.4) @@ -152,11 +159,13 @@ def activate_focus_monitor(focus_monitor): # ── UX guard ────────────────────────────────────────────────────────────────── + @pytest.fixture def ux_guard(focus_monitor): """Start UXMonitor before the test, call assert_clean() after.""" _, sentinel_pid = focus_monitor from harness.monitor import UXMonitor + mon = UXMonitor(sentinel_pid=sentinel_pid) mon.start() yield mon @@ -166,6 +175,7 @@ def ux_guard(focus_monitor): # ── local HTML server ───────────────────────────────────────────────────────── + class _SilentHandler(SimpleHTTPRequestHandler): def log_message(self, *args): pass @@ -174,9 +184,7 @@ def log_message(self, *args): @pytest.fixture(scope="session") def html_server(): """Serve the assets/ directory over HTTP; return base URL.""" - handler = lambda *args, **kwargs: _SilentHandler( - *args, directory=_ASSETS_DIR, **kwargs - ) + handler = lambda *args, **kwargs: _SilentHandler(*args, directory=_ASSETS_DIR, **kwargs) server = HTTPServer(("127.0.0.1", 0), handler) port = server.server_address[1] thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -187,6 +195,7 @@ def html_server(): # ── download helpers ────────────────────────────────────────────────────────── + def _download_and_extract(url: str, dest_dir: str, marker_path: str) -> None: if os.path.exists(marker_path): return @@ -210,6 +219,7 @@ def _download_binary(url: str, dest_path: str) -> None: # ── Tauri app ───────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def tauri_app(): """Download, launch, and yield (proc, pid, base_url) for the Tauri test app.""" @@ -239,6 +249,7 @@ def tauri_app(): # ── Electron app ────────────────────────────────────────────────────────────── + @pytest.fixture(scope="session") def electron_app(html_server): """Download, launch, and yield (proc, pid, base_url) for the Electron test app.""" @@ -267,7 +278,8 @@ def electron_app(html_server): out = subprocess.run( ["pgrep", "-f", "desktop-test-app-electron"], - capture_output=True, text=True, + capture_output=True, + text=True, ).stdout.strip() pid = int(out.split("\n")[0]) if out else 0 diff --git a/libs/cua-driver/Tests/integration/driver_client.py b/libs/cua-driver/Tests/integration/driver_client.py index 46728545d8..141975cc25 100644 --- a/libs/cua-driver/Tests/integration/driver_client.py +++ b/libs/cua-driver/Tests/integration/driver_client.py @@ -79,9 +79,7 @@ def _notify(self, method: str, params: Optional[dict] = None) -> None: payload["params"] = params self._write(payload) - def _call( - self, method: str, params: Optional[dict] = None, timeout: float = 20.0 - ) -> dict: + def _call(self, method: str, params: Optional[dict] = None, timeout: float = 20.0) -> dict: self._next_id += 1 request_id = self._next_id payload = {"jsonrpc": "2.0", "id": request_id, "method": method} @@ -191,8 +189,7 @@ def resolve_window_id( if require_on_current_space: preferred = [ - w for w in windows - if w.get("is_on_screen") and w.get("on_current_space") is not False + w for w in windows if w.get("is_on_screen") and w.get("on_current_space") is not False ] if preferred: preferred.sort(key=lambda w: w.get("z_index", 0), reverse=True) diff --git a/libs/cua-driver/Tests/integration/harness/cv.py b/libs/cua-driver/Tests/integration/harness/cv.py index d254ebfffe..d2bd34fa72 100644 --- a/libs/cua-driver/Tests/integration/harness/cv.py +++ b/libs/cua-driver/Tests/integration/harness/cv.py @@ -19,8 +19,9 @@ from typing import Optional, Tuple try: - from PIL import Image, ImageChops import numpy as np + from PIL import Image, ImageChops + _PIL_AVAILABLE = True except ImportError: _PIL_AVAILABLE = False @@ -90,7 +91,7 @@ def find_template( n_norm = (n_arr - n_mean) / n_std # Sliding-window NCC via FFT - from numpy.fft import fft2, ifft2, fftshift + from numpy.fft import fft2, fftshift, ifft2 # Zero-pad haystack to same size for FFT correlation pad_h = H + th - 1 @@ -102,12 +103,13 @@ def find_template( # Local haystack std for normalisation from numpy.lib.stride_tricks import sliding_window_view + windows = sliding_window_view(h_arr, (th, tw)) local_std = windows.reshape(-1, th * tw).std(axis=1).reshape(H - th + 1, W - tw + 1) local_std[local_std < 1e-6] = 1.0 # Trim correlation to valid region - corr_valid = corr[th - 1:th - 1 + (H - th + 1), tw - 1:tw - 1 + (W - tw + 1)] + corr_valid = corr[th - 1 : th - 1 + (H - th + 1), tw - 1 : tw - 1 + (W - tw + 1)] ncc = corr_valid / (local_std * th * tw) best_idx = np.argmax(ncc) diff --git a/libs/cua-driver/Tests/integration/harness/driver.py b/libs/cua-driver/Tests/integration/harness/driver.py index f3d555ed13..fa4be89739 100644 --- a/libs/cua-driver/Tests/integration/harness/driver.py +++ b/libs/cua-driver/Tests/integration/harness/driver.py @@ -28,7 +28,7 @@ _INTEG_DIR = os.path.dirname(_HERE) sys.path.insert(0, _INTEG_DIR) -from driver_client import DriverClient, default_binary_path, MCPCallError # noqa: E402 +from driver_client import DriverClient, MCPCallError, default_binary_path # noqa: E402 @dataclass @@ -42,9 +42,10 @@ class WindowState: def find_element(self, label: str) -> Optional[int]: """Return the first element index whose line contains `label`.""" import re + for line in self.tree.split("\n"): if label in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -52,12 +53,13 @@ def find_element(self, label: str) -> Optional[int]: def find_button(self, label: str) -> Optional[int]: """Return the first AXButton element index matching `label`.""" import re + for line in self.tree.split("\n"): if "AXButton" not in line: continue if label not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -65,13 +67,14 @@ def find_button(self, label: str) -> Optional[int]: def find_text_field(self, skip_url_bar: bool = True) -> Optional[int]: """Return the first AXTextField index (skipping browser URL bars).""" import re + _URL_BAR_HINTS = ("smart search field", "address and search bar", "location") for line in self.tree.split("\n"): if "AXTextField" not in line: continue if skip_url_bar and any(h in line.lower() for h in _URL_BAR_HINTS): continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -79,6 +82,7 @@ def find_text_field(self, skip_url_bar: bool = True) -> Optional[int]: def ax_value(self, element_index: int) -> Optional[str]: """Return the value= attribute of the element at `element_index`.""" import re + prefix = f"[{element_index}]" for line in self.tree.split("\n"): if prefix not in line: @@ -143,8 +147,10 @@ def _area(w: dict) -> float: # Filter to on-screen windows with meaningful size. candidates = [ - w for w in windows - if w.get("is_on_screen") and w.get("on_current_space") is not False + w + for w in windows + if w.get("is_on_screen") + and w.get("on_current_space") is not False and _area(w) >= min_area ] if not candidates: @@ -161,9 +167,7 @@ def _area(w: dict) -> float: # Probe each candidate for AX tree content; return the first one that # has a non-empty tree. This is needed for Chrome, whose AX tree is # attached to toolbar/helper windows rather than the large content window. - probe_order = [primary_wid] + [ - w["window_id"] for w in candidates[1:] - ] + probe_order = [primary_wid] + [w["window_id"] for w in candidates[1:]] # Also include off-screen windows as a last resort (Chrome toolbar # windows are often flagged off-screen even while Chrome is running). for w in sorted(windows, key=_area, reverse=True): @@ -204,7 +208,8 @@ def get_window_state( # Format: "
\n\n" where the tree starts with "- AX..." if not tree and text_content: import re - m = re.search(r'\n\n(- .+)', text_content, re.DOTALL) + + m = re.search(r"\n\n(- .+)", text_content, re.DOTALL) if m: tree = m.group(1) width = sc.get("screenshot_width", 0) @@ -272,14 +277,17 @@ def scroll( delta_x: float = 0.0, delta_y: float = -3.0, ) -> dict: - return self._c().call_tool("scroll", { - "pid": pid, - "window_id": window_id, - "x": x, - "y": y, - "delta_x": delta_x, - "delta_y": delta_y, - }) + return self._c().call_tool( + "scroll", + { + "pid": pid, + "window_id": window_id, + "x": x, + "y": y, + "delta_x": delta_x, + "delta_y": delta_y, + }, + ) def launch_app(self, bundle_id: str) -> dict: return self._c().call_tool("launch_app", {"bundle_id": bundle_id}) diff --git a/libs/cua-driver/Tests/integration/harness/monitor.py b/libs/cua-driver/Tests/integration/harness/monitor.py index c3469a5451..36e61501f7 100644 --- a/libs/cua-driver/Tests/integration/harness/monitor.py +++ b/libs/cua-driver/Tests/integration/harness/monitor.py @@ -22,19 +22,18 @@ from dataclasses import dataclass, field from typing import Optional - # Try to import PyObjC; fall back to no-op monitor on systems without it. try: - from AppKit import NSWorkspace - from AppKit import NSEvent + from AppKit import NSEvent, NSWorkspace from Quartz import ( CGWindowListCopyWindowInfo, - kCGWindowListOptionOnScreenOnly, kCGNullWindowID, kCGWindowLayer, - kCGWindowOwnerPID, + kCGWindowListOptionOnScreenOnly, kCGWindowName, + kCGWindowOwnerPID, ) + _PYOBJC_AVAILABLE = True except ImportError: _PYOBJC_AVAILABLE = False @@ -49,7 +48,7 @@ @dataclass class Violation: - kind: str # "cursor" | "frontmost" | "overlay_z" + kind: str # "cursor" | "frontmost" | "overlay_z" detail: str timestamp: float = field(default_factory=time.monotonic) @@ -189,9 +188,7 @@ def _check_frontmost(self) -> None: def _check_overlay_z(self) -> None: """Check that no cua-driver overlay window is above foreground windows.""" - wins = CGWindowListCopyWindowInfo( - kCGWindowListOptionOnScreenOnly, kCGNullWindowID - ) + wins = CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly, kCGNullWindowID) if not wins: return @@ -204,8 +201,9 @@ def _check_overlay_z(self) -> None: name = str(w.get(kCGWindowName, "") or "") layer = int(w.get(kCGWindowLayer, 0)) - is_overlay = name in ("cua-agent-cursor", "cua-overlay") or \ - "cua-driver" in str(name).lower() + is_overlay = ( + name in ("cua-agent-cursor", "cua-overlay") or "cua-driver" in str(name).lower() + ) if is_overlay: overlay_layers.append(layer) diff --git a/libs/cua-driver/Tests/integration/harness/tree.py b/libs/cua-driver/Tests/integration/harness/tree.py index adc2c4fe34..b69f965c7d 100644 --- a/libs/cua-driver/Tests/integration/harness/tree.py +++ b/libs/cua-driver/Tests/integration/harness/tree.py @@ -24,7 +24,7 @@ def _lines(tree: str) -> list[tuple[int, str]]: """Return (element_index, raw_line) pairs for every indexed line.""" result: list[tuple[int, str]] = [] for line in tree.split("\n"): - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: result.append((int(m.group(1)), line)) return result diff --git a/libs/cua-driver/Tests/integration/test_background_focus.py b/libs/cua-driver/Tests/integration/test_background_focus.py index dd1674411b..c010400cd3 100644 --- a/libs/cua-driver/Tests/integration/test_background_focus.py +++ b/libs/cua-driver/Tests/integration/test_background_focus.py @@ -39,9 +39,7 @@ _HTML_PAGE = os.path.join(_THIS_DIR, "fixtures", "interactive.html") _FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp") _FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") -_FOCUS_APP_EXE = os.path.join( - _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" -) +_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp") _LOSS_FILE = "/tmp/focus_monitor_losses.txt" SAFARI_BUNDLE = "com.apple.Safari" @@ -52,6 +50,7 @@ # Helpers # --------------------------------------------------------------------------- + def _tool_text(result: dict) -> str: """Extract text from a tool call result's content array.""" for item in result.get("content", []): @@ -59,11 +58,10 @@ def _tool_text(result: dict) -> str: return item.get("text", "") return "" + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -100,12 +98,15 @@ def _open_safari_to_html(client: DriverClient) -> int: document load back to whatever was frontmost before the call. """ file_url = f"file://{_HTML_PAGE}" - result = client.call_tool("launch_app", { - "bundle_id": SAFARI_BUNDLE, - "urls": [file_url], - }) + result = client.call_tool( + "launch_app", + { + "bundle_id": SAFARI_BUNDLE, + "urls": [file_url], + }, + ) text = _tool_text(result) - m = re.search(r'pid[=:\s]+(\d+)', text, re.IGNORECASE) + m = re.search(r"pid[=:\s]+(\d+)", text, re.IGNORECASE) if not m: raise RuntimeError(f"launch_app did not return a pid: {text[:200]}") time.sleep(2.0) # let page load @@ -115,9 +116,7 @@ def _open_safari_to_html(client: DriverClient) -> int: def _get_page_text(client: DriverClient, pid: int) -> str: """Return the AX tree markdown for Safari.""" window_id = resolve_window_id(client, pid) - result = client.call_tool( - "get_window_state", {"pid": pid, "window_id": window_id} - ) + result = client.call_tool("get_window_state", {"pid": pid, "window_id": window_id}) return result.get("structuredContent", result).get("tree_markdown", "") @@ -125,7 +124,7 @@ def _find_element_index(tree_markdown: str, label: str) -> int | None: """Extract the first [N] element index from a line containing `label`.""" for line in tree_markdown.split("\n"): if label in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -143,25 +142,25 @@ def _find_calc_button(tree_markdown: str, label: str) -> int | None: for line in tree_markdown.split("\n"): if "AXButton" not in line: continue - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if not m: continue idx = int(m.group(1)) # Match (Label) pattern — exact match in parens - if f'({label})' in line: + if f"({label})" in line: return idx # Also match help text if f'help="{label}' in line: return idx # Also match id=Label - if f'id={label}' in line: + if f"id={label}" in line: return idx return None def _extract_click_count(tree_markdown: str) -> int | None: """Extract the number from 'clicks: N' in the AX tree.""" - m = re.search(r'clicks:\s*(\d+)', tree_markdown) + m = re.search(r"clicks:\s*(\d+)", tree_markdown) return int(m.group(1)) if m else None @@ -178,6 +177,7 @@ def _activate_focus_monitor() -> None: # Test class # --------------------------------------------------------------------------- + class BackgroundFocusTests(unittest.TestCase): """Click & type into backgrounded Safari without stealing focus.""" @@ -210,9 +210,9 @@ def setUpClass(cls) -> None: # Confirm FocusMonitorApp is frontmost. with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" # Baseline: 0 focus losses. losses = _read_focus_losses() @@ -229,7 +229,8 @@ def tearDownClass(cls) -> None: # Close the Safari tab we opened (best-effort) subprocess.run( [ - "osascript", "-e", + "osascript", + "-e", 'tell application "Safari" to close (every tab of window 1 ' 'whose URL contains "interactive.html")', ], @@ -252,7 +253,8 @@ def setUp(self) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"FocusMonitorApp not frontmost at test start, got {active}", ) @@ -267,15 +269,17 @@ def _assert_no_focus_loss(self, label: str) -> None: losses = _read_focus_losses() with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) - focus_restored = (active == FOCUS_MONITOR_BUNDLE) + focus_restored = active == FOCUS_MONITOR_BUNDLE loss_delta = losses - self._losses_before - print(f" [{label}] losses: {self._losses_before}->{losses} " - f"(delta={loss_delta}), frontmost: {active}, " - f"restored: {focus_restored}") + print( + f" [{label}] losses: {self._losses_before}->{losses} " + f"(delta={loss_delta}), frontmost: {active}, " + f"restored: {focus_restored}" + ) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, - f"[{label}] Focus not restored — " - f"frontmost is {active}, not FocusMonitorApp", + active, + FOCUS_MONITOR_BUNDLE, + f"[{label}] Focus not restored — " f"frontmost is {active}, not FocusMonitorApp", ) # -- AX element_index click (pure AX action, no cursor move) ----------- @@ -284,22 +288,28 @@ def test_01_ax_click_button(self) -> None: """AX-click the 'Click Me' button in backgrounded Safari.""" with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "Click Me", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "Click Me", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f"\n filtered tree:\n{tree}") idx = _find_element_index(tree, "Click Me") self.assertIsNotNone(idx, "Could not find 'Click Me' button in AX tree") - result = c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "element_index": idx, - }) + result = c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "element_index": idx, + }, + ) print(f" click result: {result}") time.sleep(0.5) @@ -322,11 +332,14 @@ def test_02_type_text_chars(self) -> None: """ with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "AXTextField", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "AXTextField", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f"\n filtered tree:\n{tree}") @@ -334,19 +347,23 @@ def test_02_type_text_chars(self) -> None: idx = None for line in tree.split("\n"): if "AXTextField" in line and "smart search field" not in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: idx = int(m.group(1)) break if idx is None: - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") for line in tree.split("\n"): if "AXTextField" in line and "smart search field" not in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: idx = int(m.group(1)) break @@ -355,18 +372,24 @@ def test_02_type_text_chars(self) -> None: print(f" text field element_index: {idx}") # Focus the text field via AX click first - c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) # Type via keystroke synthesis - result = c.call_tool("type_text_chars", { - "pid": self._safari_pid, - "text": "hello bg", - }) + result = c.call_tool( + "type_text_chars", + { + "pid": self._safari_pid, + "text": "hello bg", + }, + ) print(f" type_text_chars result: {result}") # Check if the text appears in the AX tree @@ -375,7 +398,7 @@ def test_02_type_text_chars(self) -> None: time.sleep(0.5) with DriverClient(self.binary) as c: tree = _get_page_text(c, self._safari_pid) - has_text = ("hello bg" in tree) + has_text = "hello bg" in tree if has_text: break print(f" attempt {attempt+1}: text not yet visible in tree") @@ -396,11 +419,14 @@ def test_03_pixel_click_no_focus_steal(self) -> None: """ with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._safari_pid) - snap = c.call_tool("get_window_state", { - "pid": self._safari_pid, - "window_id": window_id, - "query": "Click Me", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._safari_pid, + "window_id": window_id, + "query": "Click Me", + }, + ) sc = snap.get("structuredContent", snap) width = sc.get("screenshot_width", 0) height = sc.get("screenshot_height", 0) @@ -410,12 +436,15 @@ def test_03_pixel_click_no_focus_steal(self) -> None: y = int(height * 0.35) print(f" pixel click at ({x}, {y})") - result = c.call_tool("click", { - "pid": self._safari_pid, - "window_id": window_id, - "x": x, - "y": y, - }) + result = c.call_tool( + "click", + { + "pid": self._safari_pid, + "window_id": window_id, + "x": x, + "y": y, + }, + ) print(f" click result: {result}") time.sleep(0.5) @@ -426,9 +455,13 @@ def test_03_pixel_click_no_focus_steal(self) -> None: def test_04_press_key_tab(self) -> None: """Send Tab key to backgrounded Safari without stealing focus.""" with DriverClient(self.binary) as c: - result = c.call_tool("press_key", { - "pid": self._safari_pid, "key": "tab", - }) + result = c.call_tool( + "press_key", + { + "pid": self._safari_pid, + "key": "tab", + }, + ) print(f"\n press_key result: {result}") time.sleep(0.3) @@ -476,9 +509,9 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" @classmethod def tearDownClass(cls) -> None: @@ -506,12 +539,14 @@ def _assert_no_focus_loss(self, label: str) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) loss_delta = losses - self._losses_before - print(f" [{label}] losses: {self._losses_before}->{losses} " - f"(delta={loss_delta}), frontmost: {active}") + print( + f" [{label}] losses: {self._losses_before}->{losses} " + f"(delta={loss_delta}), frontmost: {active}" + ) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, - f"[{label}] Focus not restored — " - f"frontmost is {active}, not FocusMonitorApp", + active, + FOCUS_MONITOR_BUNDLE, + f"[{label}] Focus not restored — " f"frontmost is {active}, not FocusMonitorApp", ) def test_01_ax_click_2_plus_2(self) -> None: @@ -539,19 +574,25 @@ def test_01_ax_click_2_plus_2(self) -> None: # Press 2 + 2 = for idx in [btn_2, btn_add, btn_2, btn_eq]: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) # Read result - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" result tree:\n{tree}") @@ -563,19 +604,26 @@ def test_02_pixel_click_buttons(self) -> None: with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) # Clear calculator first via AX (C button) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_c = _find_calc_button(tree, "Clear") if btn_c is None: btn_c = _find_calc_button(tree, "All clear") if btn_c is not None: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn_c, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_c, + }, + ) time.sleep(0.3) # Re-activate focus monitor after the AX clear (may have stolen focus) @@ -583,9 +631,13 @@ def test_02_pixel_click_buttons(self) -> None: self._losses_before = _read_focus_losses() # Get fresh snapshot - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") sc = snap.get("structuredContent", snap) width = sc.get("screenshot_width", 0) @@ -596,18 +648,24 @@ def test_02_pixel_click_buttons(self) -> None: self.assertIsNotNone(btn_5, "'5' button not found") # Use AX to click 5, verify it works - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn_5, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_5, + }, + ) time.sleep(0.3) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" after AX click 5:\n{tree}") self.assertIn("5", tree, "AX click on '5' didn't register") @@ -619,25 +677,36 @@ def test_03_ax_click_3_plus_4(self) -> None: with DriverClient(self.binary) as c: window_id = resolve_window_id(c, self._calc_pid) # Clear first - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_c = _find_calc_button(tree, "All Clear") if btn_c is None: btn_c = _find_calc_button(tree, "Clear") if btn_c is not None: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": btn_c, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": btn_c, + }, + ) time.sleep(0.3) # Find buttons for 3+4= - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, "window_id": window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") btn_3 = _find_calc_button(tree, "3") btn_add = _find_calc_button(tree, "Add") @@ -651,18 +720,24 @@ def test_03_ax_click_3_plus_4(self) -> None: self.assertIsNotNone(btn_eq, "'Equals' button not found") for idx in [btn_3, btn_add, btn_4, btn_eq]: - c.call_tool("click", { - "pid": self._calc_pid, - "window_id": window_id, - "element_index": idx, - }) + c.call_tool( + "click", + { + "pid": self._calc_pid, + "window_id": window_id, + "element_index": idx, + }, + ) time.sleep(0.3) - snap = c.call_tool("get_window_state", { - "pid": self._calc_pid, - "window_id": window_id, - "query": "AXStaticText", - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._calc_pid, + "window_id": window_id, + "query": "AXStaticText", + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f" result tree:\n{tree}") diff --git a/libs/cua-driver/Tests/integration/test_background_menu_shortcut.py b/libs/cua-driver/Tests/integration/test_background_menu_shortcut.py index 6be6f49d60..79430fa746 100644 --- a/libs/cua-driver/Tests/integration/test_background_menu_shortcut.py +++ b/libs/cua-driver/Tests/integration/test_background_menu_shortcut.py @@ -39,7 +39,8 @@ def _on_screen_doc_windows(driver, pid: int) -> list[dict]: result = driver.call_tool("list_windows", {"pid": pid}) windows = result.get("structuredContent", {}).get("windows", []) return [ - w for w in windows + w + for w in windows if w.get("is_on_screen") and (w.get("bounds", {}).get("width", 0) or 0) > 100 and (w.get("bounds", {}).get("height", 0) or 0) > 100 @@ -48,6 +49,7 @@ def _on_screen_doc_windows(driver, pid: int) -> list[dict]: # ── module-scoped driver (avoids scope-mismatch with module fixtures) ───────── + @pytest.fixture(scope="module") def module_driver(binary): """Module-scoped driver used by setup fixtures (textedit_pid, textedit_wid). @@ -56,22 +58,25 @@ def module_driver(binary): module-scoped fixtures must use this dedicated module_driver instead. """ from harness.driver import Driver + with Driver(binary) as d: yield d # ── module fixtures ─────────────────────────────────────────────────────────── + def _pid_from_result(result: dict) -> int: """Extract pid from a launch_app result (structuredContent or text fallback).""" import re + pid = result.get("structuredContent", {}).get("pid", 0) if pid: return pid # Fall back: parse "pid N" from the text content for item in result.get("content", []): text = item.get("text", "") - m = re.search(r'\bpid\s+(\d+)', text) + m = re.search(r"\bpid\s+(\d+)", text) if m: return int(m.group(1)) return 0 @@ -98,14 +103,16 @@ def textedit_pid(module_driver): def textedit_wid(module_driver, textedit_pid): """Return the window_id of the largest TextEdit on-screen window.""" wins = _on_screen_doc_windows(module_driver, textedit_pid) - best = max(wins, key=lambda w: ( - w.get("bounds", {}).get("width", 0) * w.get("bounds", {}).get("height", 0) - )) + best = max( + wins, + key=lambda w: (w.get("bounds", {}).get("width", 0) * w.get("bounds", {}).get("height", 0)), + ) return best["window_id"] # ── tests ───────────────────────────────────────────────────────────────────── + def test_hotkey_without_window_id_does_not_activate( driver, focus_monitor, activate_focus_monitor, ux_guard, textedit_pid ): @@ -114,17 +121,19 @@ def test_hotkey_without_window_id_does_not_activate( Uses the auth-envelope path which delivers directly to the target PID without calling activateForMenuShortcut — FocusMonitorApp keeps focus. """ - driver.call_tool("hotkey", { - "pid": textedit_pid, - "keys": ["cmd", "z"], - }) + driver.call_tool( + "hotkey", + { + "pid": textedit_pid, + "keys": ["cmd", "z"], + }, + ) time.sleep(0.5) # ux_guard.assert_clean() fires in teardown def test_hotkey_with_window_id_fires_nsmenu( - driver, focus_monitor, activate_focus_monitor, ux_guard, - textedit_pid, textedit_wid + driver, focus_monitor, activate_focus_monitor, ux_guard, textedit_pid, textedit_wid ): """hotkey WITH window_id fires NSMenu key equivalents with zero UX violations. @@ -136,11 +145,14 @@ def test_hotkey_with_window_id_fires_nsmenu( """ count_before = len(_on_screen_doc_windows(driver, textedit_pid)) - driver.call_tool("hotkey", { - "pid": textedit_pid, - "keys": ["cmd", "n"], - "window_id": textedit_wid, - }) + driver.call_tool( + "hotkey", + { + "pid": textedit_pid, + "keys": ["cmd", "n"], + "window_id": textedit_wid, + }, + ) time.sleep(1.5) count_after = len(_on_screen_doc_windows(driver, textedit_pid)) @@ -150,11 +162,11 @@ def test_hotkey_with_window_id_fires_nsmenu( wins = _on_screen_doc_windows(driver, textedit_pid) for w in wins[count_before:]: wid = w["window_id"] - driver.call_tool("hotkey", { - "pid": textedit_pid, "keys": ["cmd", "w"], "window_id": wid}) + driver.call_tool( + "hotkey", {"pid": textedit_pid, "keys": ["cmd", "w"], "window_id": wid} + ) time.sleep(0.4) - driver.call_tool("press_key", { - "pid": textedit_pid, "key": "delete", "window_id": wid}) + driver.call_tool("press_key", {"pid": textedit_pid, "key": "delete", "window_id": wid}) time.sleep(0.3) assert count_after > count_before, ( diff --git a/libs/cua-driver/Tests/integration/test_blender.py b/libs/cua-driver/Tests/integration/test_blender.py index b0455e6055..64bf9aa72b 100644 --- a/libs/cua-driver/Tests/integration/test_blender.py +++ b/libs/cua-driver/Tests/integration/test_blender.py @@ -19,19 +19,19 @@ from __future__ import annotations +import json +import os import subprocess import sys import time -import os -import json import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree -from harness.cv import decode, crop, diff_ratio, save_reference, load_reference +from harness.cv import crop, decode, diff_ratio, load_reference, save_reference +from harness.driver import Driver BLENDER_BUNDLE = "org.blenderfoundation.blender" _ASSETS_BLENDER = os.path.join(_HERE, "assets", "blender") @@ -53,6 +53,7 @@ def _save_bboxes(bboxes: dict) -> None: # ── module-level Blender setup ──────────────────────────────────────────────── + @pytest.fixture(scope="module") def blender_pid(binary, focus_monitor): """Launch Blender in background; return its pid.""" @@ -84,8 +85,11 @@ def blender_pid(binary, focus_monitor): def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -93,6 +97,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestBlenderWindow: """Basic window state and AX tree tests.""" diff --git a/libs/cua-driver/Tests/integration/test_browser_js.py b/libs/cua-driver/Tests/integration/test_browser_js.py index 528469894e..57e1b86ef7 100644 --- a/libs/cua-driver/Tests/integration/test_browser_js.py +++ b/libs/cua-driver/Tests/integration/test_browser_js.py @@ -42,6 +42,7 @@ # Helpers # --------------------------------------------------------------------------- + def _tool_text(result: dict) -> str: """Extract the text string from a tool call result.""" for item in result.get("content", []): @@ -54,14 +55,13 @@ def _enable_chrome_apple_events() -> None: """Quit Chrome, write allow_javascript_apple_events to all profiles, relaunch.""" subprocess.run( ["osascript", "-e", 'quit app "Google Chrome"'], - check=False, timeout=10, + check=False, + timeout=10, ) time.sleep(1.5) for prefs_path in glob.glob( - os.path.expanduser( - "~/Library/Application Support/Google/Chrome/*/Preferences" - ) + os.path.expanduser("~/Library/Application Support/Google/Chrome/*/Preferences") ): profile = prefs_path.split("/")[-2] if "System" in profile or "Guest" in profile: @@ -99,14 +99,14 @@ def _main_window(client: DriverClient, pid: int) -> int: # Prefer a window whose title contains "Example Domain". for line in text.splitlines(): if "Example Domain" in line or "example.com" in line.lower(): - m = re.search(r'\[window_id:\s*(\d+)\]', line) + m = re.search(r"\[window_id:\s*(\d+)\]", line) if m: return int(m.group(1)) # Fall back to first titled window. titled = re.findall(r'"[^"]+"\s+\[window_id:\s*(\d+)\]', text) if titled: return int(titled[0]) - m = re.search(r'\[window_id:\s*(\d+)\]', text) + m = re.search(r"\[window_id:\s*(\d+)\]", text) if m: return int(m.group(1)) raise RuntimeError(f"No window found for Chrome pid {pid}") @@ -116,6 +116,7 @@ def _main_window(client: DriverClient, pid: int) -> int: # Test class # --------------------------------------------------------------------------- + class BrowserJSTests(unittest.TestCase): """Browser JS primitives work end-to-end against a live Chrome window.""" @@ -131,13 +132,16 @@ def setUpClass(cls) -> None: # launch_app opens the URL in a new window WITHOUT stealing focus — # the driver's FocusRestoreGuard catches Chrome's activate call. with DriverClient(cls.binary) as c: - result = c.call_tool("launch_app", { - "bundle_id": CHROME_BUNDLE, - "urls": [TEST_URL], - }) + result = c.call_tool( + "launch_app", + { + "bundle_id": CHROME_BUNDLE, + "urls": [TEST_URL], + }, + ) text = _tool_text(result) # Extract pid from launch_app response. - m = re.search(r'pid[=:\s]+(\d+)', text, re.IGNORECASE) + m = re.search(r"pid[=:\s]+(\d+)", text, re.IGNORECASE) cls._chrome_pid = int(m.group(1)) if m else _find_chrome(c) # Wait for Chrome to load the page and update the window title. @@ -167,39 +171,48 @@ def setUpClass(cls) -> None: def test_01_execute_javascript_arithmetic(self) -> None: """execute_javascript returns the JS evaluation result.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "execute_javascript", - "javascript": "1 + 1", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "execute_javascript", + "javascript": "1 + 1", + }, + ) self.assertFalse(result.get("isError")) self.assertIn("2", _tool_text(result)) def test_02_execute_javascript_dom_read(self) -> None: """execute_javascript can read DOM content.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "execute_javascript", - "javascript": "document.querySelector('h1').innerText", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "execute_javascript", + "javascript": "document.querySelector('h1').innerText", + }, + ) self.assertFalse(result.get("isError")) self.assertIn(EXPECTED_H1, _tool_text(result)) def test_03_execute_javascript_iife(self) -> None: """execute_javascript handles IIFE with try-catch.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "execute_javascript", - "javascript": ( - "(() => { try { return document.title; } " - "catch(e) { return 'error: ' + e; } })()" - ), - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "execute_javascript", + "javascript": ( + "(() => { try { return document.title; } " + "catch(e) { return 'error: ' + e; } })()" + ), + }, + ) self.assertFalse(result.get("isError")) # ----------------------------------------------------------------------- @@ -209,25 +222,32 @@ def test_03_execute_javascript_iife(self) -> None: def test_04_get_text_returns_body_text(self) -> None: """get_text returns document.body.innerText containing the H1.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "get_text", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "get_text", + }, + ) self.assertFalse(result.get("isError")) self.assertIn(EXPECTED_H1, _tool_text(result)) def test_05_get_text_includes_link_text(self) -> None: """get_text includes anchor text from the page.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "get_text", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "get_text", + }, + ) text = _tool_text(result) - self.assertIn(EXPECTED_LINK_TEXT, text, - f"Expected link text in body text, got: {text[:200]!r}") + self.assertIn( + EXPECTED_LINK_TEXT, text, f"Expected link text in body text, got: {text[:200]!r}" + ) # ----------------------------------------------------------------------- # page action=query_dom @@ -236,13 +256,16 @@ def test_05_get_text_includes_link_text(self) -> None: def test_06_query_dom_returns_json_array(self) -> None: """query_dom returns a JSON array of matching elements with hrefs.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "query_dom", - "css_selector": "a[href]", - "attributes": ["href"], - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "query_dom", + "css_selector": "a[href]", + "attributes": ["href"], + }, + ) self.assertFalse(result.get("isError")) text = _tool_text(result) self.assertIn("```json", text) @@ -259,12 +282,15 @@ def test_06_query_dom_returns_json_array(self) -> None: def test_07_query_dom_h1_no_attributes(self) -> None: """query_dom without attributes returns tag and text.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "query_dom", - "css_selector": "h1", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "query_dom", + "css_selector": "h1", + }, + ) self.assertFalse(result.get("isError")) text = _tool_text(result) json_block = text.split("```json")[-1].split("```")[0].strip() @@ -280,11 +306,14 @@ def test_07_query_dom_h1_no_attributes(self) -> None: def test_08_get_window_state_javascript_co_located(self) -> None: """get_window_state with javascript= returns JS result alongside AX tree.""" with DriverClient(self.binary) as c: - result = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "javascript": "document.title", - }) + result = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "javascript": "document.title", + }, + ) self.assertFalse(result.get("isError")) text = _tool_text(result) self.assertIn("## JavaScript result", text) @@ -296,11 +325,14 @@ def test_09_get_window_state_javascript_error_is_inline(self) -> None: # Chrome returns "missing value" for JS that throws — osascript exits 0. # The result section appears but may contain "missing value" or an error msg. with DriverClient(self.binary) as c: - result = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "javascript": "undefined_var_that_does_not_exist", - }) + result = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "javascript": "undefined_var_that_does_not_exist", + }, + ) # isError must be False — JS failures must not kill the whole tool call. self.assertFalse(result.get("isError")) text = _tool_text(result) @@ -314,21 +346,27 @@ def test_09_get_window_state_javascript_error_is_inline(self) -> None: def test_10_page_execute_missing_javascript_field(self) -> None: """page execute_javascript returns isError when javascript field is absent.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "execute_javascript", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "execute_javascript", + }, + ) self.assertTrue(result.get("isError")) def test_11_page_query_dom_missing_selector(self) -> None: """page query_dom returns isError when css_selector is absent.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._chrome_pid, - "window_id": self._window_id, - "action": "query_dom", - }) + result = c.call_tool( + "page", + { + "pid": self._chrome_pid, + "window_id": self._window_id, + "action": "query_dom", + }, + ) self.assertTrue(result.get("isError")) diff --git a/libs/cua-driver/Tests/integration/test_check_permissions_cli.py b/libs/cua-driver/Tests/integration/test_check_permissions_cli.py index 6576670c2f..8dc16b290b 100644 --- a/libs/cua-driver/Tests/integration/test_check_permissions_cli.py +++ b/libs/cua-driver/Tests/integration/test_check_permissions_cli.py @@ -35,7 +35,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from driver_client import default_binary_path - WARNING_MARKER = "Not running inside the cua-driver daemon process" @@ -51,9 +50,7 @@ def _run_cli(binary: str, *args: str) -> subprocess.CompletedProcess: def _stop_daemon(binary: str) -> None: """Best-effort stop; the 'not running' case is not an error here.""" - subprocess.run( - [binary, "stop"], capture_output=True, text=True, timeout=5 - ) + subprocess.run([binary, "stop"], capture_output=True, text=True, timeout=5) # Let the socket file vanish before the next call. time.sleep(0.3) @@ -64,12 +61,8 @@ def _start_daemon(binary: str) -> None: inherits the shell's TCC context, defeating the point of the test.""" # Resolve the .app path from the binary path. The build script puts # the binary at `.build/CuaDriver.app/Contents/MacOS/cua-driver`. - app_path = os.path.abspath( - os.path.join(os.path.dirname(binary), "..", "..") - ) - assert app_path.endswith(".app"), ( - f"expected binary inside CuaDriver.app, got {app_path!r}" - ) + app_path = os.path.abspath(os.path.join(os.path.dirname(binary), "..", "..")) + assert app_path.endswith(".app"), f"expected binary inside CuaDriver.app, got {app_path!r}" subprocess.run( ["open", "-a", app_path, "--args", "serve"], check=True, @@ -79,9 +72,7 @@ def _start_daemon(binary: str) -> None: # PermissionsGate before binding the socket, so this can take ~1s. deadline = time.monotonic() + 5.0 while time.monotonic() < deadline: - result = subprocess.run( - [binary, "status"], capture_output=True, text=True, timeout=2 - ) + result = subprocess.run([binary, "status"], capture_output=True, text=True, timeout=2) if result.returncode == 0: return time.sleep(0.1) diff --git a/libs/cua-driver/Tests/integration/test_chrome.py b/libs/cua-driver/Tests/integration/test_chrome.py index 26dcf066db..c5f235bbfc 100644 --- a/libs/cua-driver/Tests/integration/test_chrome.py +++ b/libs/cua-driver/Tests/integration/test_chrome.py @@ -9,24 +9,25 @@ from __future__ import annotations +import os import re import subprocess import sys import time -import os import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver def _parse_clicks(tree_text: str) -> int: - m = re.search(r'clicks:\s*(\d+)', tree_text) + m = re.search(r"clicks:\s*(\d+)", tree_text) return int(m.group(1)) if m else 0 + CHROME_BUNDLE = "com.google.Chrome" FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp" @@ -52,6 +53,7 @@ def chrome_pid(binary, html_server, focus_monitor): # Use a throwaway profile dir so Chrome never shows restore/first-run dialogs import tempfile + _tmp_profile = tempfile.mkdtemp(prefix="cua_chrome_test_") page_url = f"{html_server}/test_page.html" @@ -82,8 +84,11 @@ def chrome_pid(binary, html_server, focus_monitor): # Hand focus to FocusMonitorApp by pid to avoid activating stale instances _, fm_pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {fm_pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {fm_pid}) to true', + ], check=False, ) time.sleep(0.5) @@ -98,8 +103,11 @@ def chrome_pid(binary, html_server, focus_monitor): def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -107,13 +115,16 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestChromeButton: def test_click_increments_counter(self, driver, chrome_pid, ux_guard): wid = driver.find_window(chrome_pid, probe_ax=True) full = driver.get_window_state(chrome_pid, wid) idx = full.find_element("Click Me") - assert idx is not None, f"'Click Me' not found in tree (first 800 chars):\n{full.tree[:800]}" + assert ( + idx is not None + ), f"'Click Me' not found in tree (first 800 chars):\n{full.tree[:800]}" count_before = _parse_clicks(full.tree) driver.click(chrome_pid, wid, element_index=idx) @@ -122,7 +133,9 @@ def test_click_increments_counter(self, driver, chrome_pid, ux_guard): after = driver.get_window_state(chrome_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 1, f"Counter did not increment: {count_before} → {count_after}" + assert ( + count_after == count_before + 1 + ), f"Counter did not increment: {count_before} → {count_after}" def test_click_increments_twice(self, driver, chrome_pid, ux_guard): wid = driver.find_window(chrome_pid, probe_ax=True) @@ -139,7 +152,9 @@ def test_click_increments_twice(self, driver, chrome_pid, ux_guard): after = driver.get_window_state(chrome_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 2, f"Counter should have incremented by 2: {count_before} → {count_after}" + assert ( + count_after == count_before + 2 + ), f"Counter should have incremented by 2: {count_before} → {count_after}" class TestChromeTextInput: @@ -172,6 +187,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, chrome_pid, ux_guard): assert mid.screenshot_b64, "No screenshot" from harness.cv import decode, diff_ratio + before_img = decode(mid.screenshot_b64) x = mid.screenshot_width // 2 diff --git a/libs/cua-driver/Tests/integration/test_chrome_minimized_nav.py b/libs/cua-driver/Tests/integration/test_chrome_minimized_nav.py index 0a5816ea2b..d06dc15e56 100644 --- a/libs/cua-driver/Tests/integration/test_chrome_minimized_nav.py +++ b/libs/cua-driver/Tests/integration/test_chrome_minimized_nav.py @@ -38,9 +38,7 @@ _REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) _FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp") _FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") -_FOCUS_APP_EXE = os.path.join( - _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" -) +_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp") _LOSS_FILE = "/tmp/focus_monitor_losses.txt" CHROME_BUNDLE = "com.google.Chrome" @@ -51,11 +49,10 @@ # Helpers # --------------------------------------------------------------------------- + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -85,7 +82,7 @@ def _read_focus_losses() -> int: def _find_element_index(tree: str, label: str) -> int | None: for line in tree.split("\n"): if label in line: - m = re.search(r'\[(\d+)\]', line) + m = re.search(r"\[(\d+)\]", line) if m: return int(m.group(1)) return None @@ -94,9 +91,7 @@ def _find_element_index(tree: str, label: str) -> int | None: def _chrome_is_minimized(client: DriverClient, pid: int) -> bool: """Check if Chrome's main window is minimized by looking for AXMinimized.""" window_id = resolve_window_id(client, pid, require_on_current_space=False) - snap = client.call_tool( - "get_window_state", {"pid": pid, "window_id": window_id} - ) + snap = client.call_tool("get_window_state", {"pid": pid, "window_id": window_id}) tree = snap.get("structuredContent", snap).get("tree_markdown", "") # A minimized window typically won't have on-screen content, # but we can check the has_screenshot field — if no on-screen @@ -117,6 +112,7 @@ def _activate_focus_monitor() -> None: # Test class # --------------------------------------------------------------------------- + class ChromeMinimizedNavTests(unittest.TestCase): """Interact with a minimized Chrome without deminiaturizing or stealing focus.""" @@ -141,20 +137,26 @@ def setUpClass(cls) -> None: if chrome: cls._chrome_pid = chrome[0]["pid"] else: - result = c.call_tool("launch_app", { - "bundle_id": CHROME_BUNDLE, - "urls": ["about:blank"], - }) + result = c.call_tool( + "launch_app", + { + "bundle_id": CHROME_BUNDLE, + "urls": ["about:blank"], + }, + ) cls._chrome_pid = result["structuredContent"]["pid"] time.sleep(2.0) print(f"\n Chrome pid: {cls._chrome_pid}") # Ensure Chrome has an about:blank window via launch_app (no focus steal). with DriverClient(cls.binary) as c: - c.call_tool("launch_app", { - "bundle_id": CHROME_BUNDLE, - "urls": ["about:blank"], - }) + c.call_tool( + "launch_app", + { + "bundle_id": CHROME_BUNDLE, + "urls": ["about:blank"], + }, + ) time.sleep(2.0) # Snapshot AX tree BEFORE minimizing to cache the omnibox index. @@ -163,10 +165,13 @@ def setUpClass(cls) -> None: # resolve cached indices. with DriverClient(cls.binary) as c: cls._chrome_window_id = resolve_window_id(c, cls._chrome_pid) - snap = c.call_tool("get_window_state", { - "pid": cls._chrome_pid, - "window_id": cls._chrome_window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": cls._chrome_pid, + "window_id": cls._chrome_window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") cls._omnibox_idx = _find_element_index(tree, "Address and search bar") if cls._omnibox_idx is None: @@ -176,10 +181,13 @@ def setUpClass(cls) -> None: # Minimize Chrome via Cmd+M. with DriverClient(cls.binary) as c: - c.call_tool("hotkey", { - "pid": cls._chrome_pid, - "keys": ["cmd", "m"], - }) + c.call_tool( + "hotkey", + { + "pid": cls._chrome_pid, + "keys": ["cmd", "m"], + }, + ) time.sleep(1.5) print(" Chrome minimized via Cmd+M") @@ -190,9 +198,9 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost, got {active}" @classmethod def tearDownClass(cls) -> None: @@ -203,8 +211,7 @@ def tearDownClass(cls) -> None: cls._focus_proc.kill() # Close the Chrome window we opened. subprocess.run( - ["osascript", "-e", - 'tell application "Google Chrome" to close window 1'], + ["osascript", "-e", 'tell application "Google Chrome" to close window 1'], check=False, ) try: @@ -216,10 +223,13 @@ def setUp(self) -> None: # Minimize Chrome via hotkey — cua-driver delivers Cmd+M to the pid # via CGEvent.postToPid without stealing focus. with DriverClient(self.binary) as c: - c.call_tool("hotkey", { - "pid": self._chrome_pid, - "keys": ["cmd", "m"], - }) + c.call_tool( + "hotkey", + { + "pid": self._chrome_pid, + "keys": ["cmd", "m"], + }, + ) time.sleep(1.5) _activate_focus_monitor() self._losses_before = _read_focus_losses() @@ -231,10 +241,13 @@ def _assert_focus_preserved(self, label: str) -> None: with DriverClient(self.binary) as c: active = frontmost_bundle_id(c) loss_delta = losses - self._losses_before - print(f" [{label}] losses: {self._losses_before}->{losses} " - f"(delta={loss_delta}), frontmost: {active}") + print( + f" [{label}] losses: {self._losses_before}->{losses} " + f"(delta={loss_delta}), frontmost: {active}" + ) self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, + active, + FOCUS_MONITOR_BUNDLE, f"[{label}] Focus stolen — frontmost is {active}", ) @@ -242,10 +255,13 @@ def _assert_chrome_still_minimized(self, label: str) -> None: """Chrome window is still minimized — verified by checking if get_window_state returns has_screenshot=false (no on-screen window).""" with DriverClient(self.binary) as c: - snap = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + }, + ) sc = snap.get("structuredContent", snap) has_shot = sc.get("has_screenshot", False) is_minimized = not has_shot @@ -260,10 +276,13 @@ def _assert_chrome_still_minimized(self, label: str) -> None: def test_01_get_window_state_safe(self) -> None: """AX snapshot of minimized Chrome should not deminiaturize.""" with DriverClient(self.binary) as c: - snap = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") print(f"\n tree length: {len(tree)} chars") @@ -281,15 +300,21 @@ def test_02_ax_click_omnibox(self) -> None: with DriverClient(self.binary) as c: # Refresh element cache. - c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - }) - result = c.call_tool("click", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - "element_index": idx, - }) + c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + }, + ) + result = c.call_tool( + "click", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + "element_index": idx, + }, + ) text = "" for item in result.get("content", []): if item.get("type") == "text": @@ -310,16 +335,22 @@ def test_03_set_value_omnibox(self) -> None: self.skipTest("Omnibox not found in pre-minimize AX tree") with DriverClient(self.binary) as c: - c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - }) - result = c.call_tool("set_value", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - "element_index": idx, - "value": "https://example.com", - }) + c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + }, + ) + result = c.call_tool( + "set_value", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + "element_index": idx, + "value": "https://example.com", + }, + ) text = "" for item in result.get("content", []): if item.get("type") == "text": @@ -335,10 +366,13 @@ def test_03_set_value_omnibox(self) -> None: def test_04_type_without_omnibox_focus(self) -> None: """Type keys to minimized Chrome without focusing omnibox — should stay minimized.""" with DriverClient(self.binary) as c: - c.call_tool("type_text_chars", { - "pid": self._chrome_pid, - "text": "hello", - }) + c.call_tool( + "type_text_chars", + { + "pid": self._chrome_pid, + "text": "hello", + }, + ) time.sleep(0.5) self._assert_chrome_still_minimized("04_type_no_omnibox") @@ -355,10 +389,13 @@ def test_05_focus_without_raise_safe(self) -> None: with DriverClient(self.binary) as c: # get_window_state uses FocusWithoutRaise internally for # AX enablement. Just verify the snapshot path is safe. - snap = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": self._chrome_window_id, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": self._chrome_window_id, + }, + ) count = snap.get("structuredContent", snap).get("element_count", 0) print(f"\n element_count: {count}") diff --git a/libs/cua-driver/Tests/integration/test_click_opens_new_window.py b/libs/cua-driver/Tests/integration/test_click_opens_new_window.py index c8c7df5222..5700eae98c 100644 --- a/libs/cua-driver/Tests/integration/test_click_opens_new_window.py +++ b/libs/cua-driver/Tests/integration/test_click_opens_new_window.py @@ -47,9 +47,7 @@ _REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) _FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp") _FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") -_FOCUS_APP_EXE = os.path.join( - _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" -) +_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp") _LOSS_FILE = "/tmp/focus_monitor_losses.txt" _UTM_BUNDLE = "com.utmapp.UTM" @@ -62,6 +60,7 @@ # Helpers # --------------------------------------------------------------------------- + def _tool_text(result: dict) -> str: for item in result.get("content", []): if item.get("type") == "text": @@ -122,6 +121,7 @@ def _kill(name: str) -> None: # Test # --------------------------------------------------------------------------- + class TestBrowseUTMGalleryUXGuard(unittest.TestCase): """Clicking 'Browse UTM Gallery' in a backgrounded UTM must not steal focus. @@ -181,9 +181,9 @@ def test_browse_gallery_click_with_ux_guard(self) -> None: # Verify FocusMonitorApp is frontmost before the click. front_before = frontmost_bundle_id(c) self.assertEqual( - front_before, _FOCUS_MONITOR_BUNDLE, - f"Expected FocusMonitorApp to be frontmost before the click, " - f"got {front_before!r}", + front_before, + _FOCUS_MONITOR_BUNDLE, + f"Expected FocusMonitorApp to be frontmost before the click, " f"got {front_before!r}", ) losses_before = _read_focus_losses() @@ -208,21 +208,27 @@ def test_browse_gallery_click_with_ux_guard(self) -> None: # Click the gallery button — by element index if found, pixel otherwise. if gallery_idx is not None: - result = c.call_tool("click", { - "pid": utm_pid, - "window_id": window_id, - "element_index": gallery_idx, - }) + result = c.call_tool( + "click", + { + "pid": utm_pid, + "window_id": window_id, + "element_index": gallery_idx, + }, + ) else: # Fallback: pixel click at the approximate "Browse UTM Gallery" # button location in UTM's welcome screen. b = utm_win["bounds"] - result = c.call_tool("click", { - "pid": utm_pid, - "window_id": window_id, - "x": b["width"] / 2.0, - "y": b["height"] * 0.35, - }) + result = c.call_tool( + "click", + { + "pid": utm_pid, + "window_id": window_id, + "x": b["width"] / 2.0, + "y": b["height"] * 0.35, + }, + ) result_text = _tool_text(result) @@ -236,11 +242,13 @@ def test_browse_gallery_click_with_ux_guard(self) -> None: # 2. Result must announce the new Safari window. self.assertIn( - "🪟", result_text, + "🪟", + result_text, f"Expected 🪟 new-window notice in click result.\nGot: {result_text}", ) self.assertIn( - "Safari", result_text, + "Safari", + result_text, f"Expected 'Safari' mentioned in new-window notice.\nGot: {result_text}", ) @@ -248,7 +256,8 @@ def test_browse_gallery_click_with_ux_guard(self) -> None: time.sleep(0.3) front_after = frontmost_bundle_id(c) self.assertEqual( - front_after, _FOCUS_MONITOR_BUNDLE, + front_after, + _FOCUS_MONITOR_BUNDLE, f"Expected FocusMonitorApp to remain frontmost after re-raise, " f"got {front_after!r}.\nClick result: {result_text}", ) @@ -264,7 +273,8 @@ def test_browse_gallery_click_with_ux_guard(self) -> None: losses_after = _read_focus_losses() delta = losses_after - losses_before self.assertLessEqual( - delta, 1, + delta, + 1, f"FocusMonitorApp lost focus {delta} time(s) (max 1 allowed) " f"during the click sequence — ux_guard violated.\n" f"Click result: {result_text}", diff --git a/libs/cua-driver/Tests/integration/test_click_pixel_ax.py b/libs/cua-driver/Tests/integration/test_click_pixel_ax.py index 68cafd0f99..645cdc3c8f 100644 --- a/libs/cua-driver/Tests/integration/test_click_pixel_ax.py +++ b/libs/cua-driver/Tests/integration/test_click_pixel_ax.py @@ -111,14 +111,10 @@ def test_pixel_click_dispatches_via_ax_hit_test(self) -> None: # hit-test cache. Force `som` (tree + screenshot, the shipped # default) so this assertion doesn't flap based on whatever # capture_mode the previous session left on disk. - set_mode = self.client.call_tool( - "set_config", {"key": "capture_mode", "value": "som"} - ) + set_mode = self.client.call_tool("set_config", {"key": "capture_mode", "value": "som"}) self.assertIsNone(set_mode.get("isError"), msg=set_mode) - launch = self.client.call_tool( - "launch_app", {"bundle_id": "com.apple.calculator"} - ) + launch = self.client.call_tool("launch_app", {"bundle_id": "com.apple.calculator"}) self.assertIsNone(launch.get("isError"), msg=launch) pid = launch["structuredContent"]["pid"] @@ -129,9 +125,7 @@ def test_pixel_click_dispatches_via_ax_hit_test(self) -> None: # Populate the snapshot cache so element_index paths work if # the test ever needs them; harmless for the pixel assertion. window_id = resolve_window_id(self.client, pid) - snap = self.client.call_tool( - "get_window_state", {"pid": pid, "window_id": window_id} - ) + snap = self.client.call_tool("get_window_state", {"pid": pid, "window_id": window_id}) self.assertIsNone(snap.get("isError"), msg=snap) scale = snap["structuredContent"].get("screenshot_scale_factor") or 1.0 diff --git a/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py b/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py index c58b152e11..d63eec3571 100644 --- a/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py +++ b/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py @@ -42,9 +42,7 @@ _FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp") _FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") -_FOCUS_APP_EXE = os.path.join( - _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" -) +_FOCUS_APP_EXE = os.path.join(_FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp") _LOSS_FILE = "/tmp/focus_monitor_losses.txt" SAFARI_BUNDLE = "com.apple.Safari" @@ -53,9 +51,7 @@ def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): - subprocess.run( - [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True - ) + subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) def _launch_focus_app() -> tuple[subprocess.Popen, int]: @@ -77,7 +73,8 @@ def _launch_focus_app() -> tuple[subprocess.Popen, int]: raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time") subprocess.run( ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'], - check=False, timeout=3, + check=False, + timeout=3, ) return proc, pid @@ -92,12 +89,12 @@ def _read_focus_losses() -> int: def _safari_js(expr: str, timeout: float = 5.0) -> str: """Evaluate a JS expression in Safari's front document via osascript.""" - script = ( - f'tell application "Safari" to do JavaScript "{expr}" in front document' - ) + script = f'tell application "Safari" to do JavaScript "{expr}" in front document' r = subprocess.run( ["osascript", "-e", script], - capture_output=True, text=True, timeout=timeout, + capture_output=True, + text=True, + timeout=timeout, ) return r.stdout.strip() @@ -186,9 +183,9 @@ def setUpClass(cls) -> None: with DriverClient(cls.binary) as c: active = frontmost_bundle_id(c) - assert active == FOCUS_MONITOR_BUNDLE, ( - f"Expected FocusMonitorApp frontmost at start, got {active}" - ) + assert ( + active == FOCUS_MONITOR_BUNDLE + ), f"Expected FocusMonitorApp frontmost at start, got {active}" losses = _read_focus_losses() assert losses == 0, f"Expected 0 focus losses at start, got {losses}" @@ -220,8 +217,7 @@ def test_slider_drag_to_right_edge(self) -> None: """Drag the thumb from mid-track to past the right edge — value goes to 100.""" before = _slider_value() self.assertEqual( - before, 50, - f"slider didn't reset to 50 (got {before}) — fixture or JS bridge off" + before, 50, f"slider didn't reset to 50 (got {before}) — fixture or JS bridge off" ) with DriverClient(self.binary) as c: @@ -234,28 +230,25 @@ def test_slider_drag_to_right_edge(self) -> None: scale = sc.get("screenshot_scale_factor", 2) # Window origin in screen points (top-left). - windows = c.call_tool( - "list_windows", {"pid": self._safari_pid} - )["structuredContent"]["windows"] - win = next( - w for w in windows - if w["window_id"] == window_id - ) + windows = c.call_tool("list_windows", {"pid": self._safari_pid})["structuredContent"][ + "windows" + ] + win = next(w for w in windows if w["window_id"] == window_id) win_x = win["bounds"]["x"] win_y = win["bounds"]["y"] rect = _slider_screen_rect() # Slider center in screen points. cx = rect["screenLeft"] + rect["left"] + rect["width"] / 2 - cy = rect["screenTop"] + rect["top"] + rect["height"] / 2 + cy = rect["screenTop"] + rect["top"] + rect["height"] / 2 # End point: 50 px past the right edge of the slider, same y. ex = rect["screenLeft"] + rect["left"] + rect["width"] + 50 ey = cy # Convert screen points → window-local image pixels. from_x = (cx - win_x) * scale from_y = (cy - win_y) * scale - to_x = (ex - win_x) * scale - to_y = (ey - win_y) * scale + to_x = (ex - win_x) * scale + to_y = (ey - win_y) * scale print( f"\n slider rect (screen pts): " @@ -276,9 +269,7 @@ def test_slider_drag_to_right_edge(self) -> None: "steps": 24, }, ) - print( - f" drag result: {result.get('content', [{}])[0].get('text', '')[:200]}" - ) + print(f" drag result: {result.get('content', [{}])[0].get('text', '')[:200]}") time.sleep(0.4) after = _slider_value() @@ -286,8 +277,7 @@ def test_slider_drag_to_right_edge(self) -> None: # Drag landed AND clamped at the slider's max (100). self.assertGreaterEqual( - after, 80, - f"Drag did not move the slider far enough — value is {after} (was {before})" + after, 80, f"Drag did not move the slider far enough — value is {after} (was {before})" ) # Focus invariant: FocusMonitorApp stayed frontmost throughout. @@ -296,12 +286,12 @@ def test_slider_drag_to_right_edge(self) -> None: losses = _read_focus_losses() print(f" losses: {self._losses_before}→{losses}, frontmost: {active}") self.assertEqual( - active, FOCUS_MONITOR_BUNDLE, - f"Focus stolen — frontmost is {active} (expected FocusMonitorApp)" + active, + FOCUS_MONITOR_BUNDLE, + f"Focus stolen — frontmost is {active} (expected FocusMonitorApp)", ) self.assertEqual( - losses, self._losses_before, - f"Focus losses increased: {self._losses_before} → {losses}" + losses, self._losses_before, f"Focus losses increased: {self._losses_before} → {losses}" ) diff --git a/libs/cua-driver/Tests/integration/test_electron.py b/libs/cua-driver/Tests/integration/test_electron.py index 5c9818ceca..777ad4eaa4 100644 --- a/libs/cua-driver/Tests/integration/test_electron.py +++ b/libs/cua-driver/Tests/integration/test_electron.py @@ -11,23 +11,24 @@ from __future__ import annotations +import os import subprocess import sys import time -import os import pytest try: import requests as _requests + _REQUESTS_AVAILABLE = True except ImportError: _REQUESTS_AVAILABLE = False _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver _ELECTRON_API = "http://localhost:6769" @@ -55,8 +56,11 @@ def _http_reset(base_url: str = _ELECTRON_API) -> None: def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -64,6 +68,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestElectronButton: def test_click_increments_counter(self, driver, electron_app, ux_guard): _, pid, base_url = electron_app @@ -103,11 +108,14 @@ def test_type_text(self, driver, electron_app, ux_guard): try: driver.click(pid, wid, element_index=idx) time.sleep(0.4) - driver.call_tool("type_text_chars", { - "pid": pid, - "text": "hello electron", - "delay_ms": 30, - }) + driver.call_tool( + "type_text_chars", + { + "pid": pid, + "text": "hello electron", + "delay_ms": 30, + }, + ) except Exception as e: events = _http_events(base_url) key_events = [ev for ev in events if ev.get("type") in ("keydown", "keypress", "input")] @@ -135,7 +143,9 @@ def test_type_text(self, driver, electron_app, ux_guard): if after is None or not after.tree: pytest.skip("AX tree unavailable after typing (Electron AX rebuild timeout)") - print(f"\n after tree length: {len(after.tree)}, has text: {after.has_text('hello electron')}") + print( + f"\n after tree length: {len(after.tree)}, has text: {after.has_text('hello electron')}" + ) assert after.has_text("hello electron"), f"Text not in AX tree:\n{after.tree[:500]}" @@ -162,6 +172,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, electron_app, ux_guard) assert mid.screenshot_b64, "No screenshot after 5 retries" from harness.cv import decode, diff_ratio + before_img = decode(mid.screenshot_b64) x = mid.screenshot_width // 2 diff --git a/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py index 2ddff05cc8..bd2d75a8e3 100644 --- a/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py +++ b/libs/cua-driver/Tests/integration/test_hermes_chrome_form_fill.py @@ -61,7 +61,11 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402 +from driver_client import ( # noqa: E402 + DriverClient, + default_binary_path, + resolve_window_id, +) # --------------------------------------------------------------------------- # Paths & constants @@ -77,8 +81,8 @@ _FOCUS_APP_EXE = os.path.join( _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp" ) -_LOSS_FILE = "/tmp/focus_monitor_losses.txt" -_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" +_LOSS_FILE = "/tmp/focus_monitor_losses.txt" +_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" _FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt" _HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes") @@ -95,6 +99,7 @@ # Chrome Apple Events JS helpers # --------------------------------------------------------------------------- + def _enable_chrome_apple_events() -> None: """Quit Chrome, write allow_javascript_apple_events to all profiles. @@ -105,14 +110,13 @@ def _enable_chrome_apple_events() -> None: """ subprocess.run( ["osascript", "-e", 'quit app "Google Chrome"'], - check=False, timeout=10, + check=False, + timeout=10, ) time.sleep(1.5) for prefs_path in glob.glob( - os.path.expanduser( - "~/Library/Application Support/Google/Chrome/*/Preferences" - ) + os.path.expanduser("~/Library/Application Support/Google/Chrome/*/Preferences") ): profile = prefs_path.split("/")[-2] if "System" in profile or "Guest" in profile: @@ -141,12 +145,10 @@ def _js_chrome(expr: str) -> str: # AppleScript: tell application "Google Chrome" to execute front window's # active tab javascript "" osa = ( - "tell application \"Google Chrome\" to execute " + 'tell application "Google Chrome" to execute ' f"front window's active tab javascript {json.dumps(expr)}" ) - r = subprocess.run( - ["osascript", "-e", osa], capture_output=True, text=True, timeout=15 - ) + r = subprocess.run(["osascript", "-e", osa], capture_output=True, text=True, timeout=15) return r.stdout.strip() @@ -168,16 +170,16 @@ def _wait_for_chrome_form(timeout: float = 30.0) -> bool: # Map logical field names → HTML element IDs in form_all_inputs.html. _FIELD_TO_ID: dict[str, str] = { - "text": "f-text", - "email": "f-email", + "text": "f-text", + "email": "f-email", "password": "f-password", - "number": "f-number", - "tel": "f-tel", + "number": "f-number", + "tel": "f-tel", "textarea": "f-textarea", - "select": "f-select", - "range": "f-range", - "date": "f-date", - "color": "f-color", + "select": "f-select", + "range": "f-range", + "date": "f-date", + "color": "f-color", } @@ -208,9 +210,13 @@ def _checkbox_checked() -> bool: # FocusMonitorApp helpers # --------------------------------------------------------------------------- + def _launch_focus_app() -> tuple[subprocess.Popen, int]: proc = subprocess.Popen( - [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + [_FOCUS_APP_EXE], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, ) for _ in range(40): line = proc.stdout.readline().strip() @@ -233,6 +239,7 @@ def _read_file_int(path: str) -> int: # Base test class # --------------------------------------------------------------------------- + class _HermesChromFormBase(unittest.TestCase): """Shared setup/teardown and hermes runner for all Chrome per-input tests.""" @@ -248,9 +255,7 @@ def setUpClass(cls) -> None: "ANTHROPIC_API_KEY is not set — skipping hermes Chrome form-fill tests" ) if not os.path.exists(_HERMES_BIN): - raise unittest.SkipTest( - f"hermes not found at {_HERMES_BIN} — install hermes first" - ) + raise unittest.SkipTest(f"hermes not found at {_HERMES_BIN} — install hermes first") for f in (_LOSS_FILE, _KEY_LOSS_FILE, _FIELD_LOSS_FILE): try: @@ -260,8 +265,9 @@ def setUpClass(cls) -> None: # Rebuild FocusMonitorApp when Swift source is newer. src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift") - if (not os.path.exists(_FOCUS_APP_EXE) - or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)): + if not os.path.exists(_FOCUS_APP_EXE) or os.path.getmtime(src) > os.path.getmtime( + _FOCUS_APP_EXE + ): subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) cls.binary = default_binary_path() @@ -275,10 +281,13 @@ def setUpClass(cls) -> None: # launch_app — no activation, no window raise, no focus steal. print(f" Launching Chrome hidden with {_FORM_URL}") with DriverClient(cls.binary) as c: - result = c.call_tool("launch_app", { - "bundle_id": CHROME_BUNDLE, - "urls": [_FORM_URL], - }) + result = c.call_tool( + "launch_app", + { + "bundle_id": CHROME_BUNDLE, + "urls": [_FORM_URL], + }, + ) sc = result.get("structuredContent", {}) cls._chrome_pid = sc.get("pid", 0) if not cls._chrome_pid: @@ -310,13 +319,15 @@ def setUpClass(cls) -> None: # baseline in setUp(), so startup-phase transients are not counted # as test failures. prev = _read_file_int(_FIELD_LOSS_FILE) - for _ in range(20): # up to 10 s + for _ in range(20): # up to 10 s time.sleep(0.5) curr = _read_file_int(_FIELD_LOSS_FILE) if curr == prev: break prev = curr - print(f" FocusMonitorApp ready (field_losses baseline: {_read_file_int(_FIELD_LOSS_FILE)})") + print( + f" FocusMonitorApp ready (field_losses baseline: {_read_file_int(_FIELD_LOSS_FILE)})" + ) @classmethod def tearDownClass(cls) -> None: @@ -327,12 +338,13 @@ def tearDownClass(cls) -> None: cls._focus_proc.kill() subprocess.run( ["osascript", "-e", 'quit app "Google Chrome"'], - check=False, timeout=10, + check=False, + timeout=10, ) def setUp(self) -> None: - self._losses_before = _read_file_int(_LOSS_FILE) - self._key_losses_before = _read_file_int(_KEY_LOSS_FILE) + self._losses_before = _read_file_int(_LOSS_FILE) + self._key_losses_before = _read_file_int(_KEY_LOSS_FILE) self._field_losses_before = _read_file_int(_FIELD_LOSS_FILE) def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str: @@ -344,16 +356,25 @@ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> st result = subprocess.run( [ - _HERMES_BIN, "chat", - "--provider", "anthropic", - "-m", _MODEL, - "--toolsets", "computer_use", + _HERMES_BIN, + "chat", + "--provider", + "anthropic", + "-m", + _MODEL, + "--toolsets", + "computer_use", "--yolo", "-Q", - "--max-turns", str(max_turns), - "-q", prompt, + "--max-turns", + str(max_turns), + "-q", + prompt, ], - env=env, capture_output=True, text=True, timeout=timeout, + env=env, + capture_output=True, + text=True, + timeout=timeout, ) return (result.stdout + "\n" + result.stderr).strip() @@ -371,8 +392,8 @@ def _assert_no_new_focus_loss(self, allow_field_losses: int = 0) -> None: value for multi-action tests. """ time.sleep(0.3) - app_delta = _read_file_int(_LOSS_FILE) - self._losses_before - key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before + app_delta = _read_file_int(_LOSS_FILE) - self._losses_before + key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before msgs = [] if app_delta: @@ -385,7 +406,8 @@ def _assert_no_new_focus_loss(self, allow_field_losses: int = 0) -> None: f" (allowed: {allow_field_losses})" ) self.assertEqual( - len(msgs), 0, + len(msgs), + 0, f"Focus stolen in {self._testMethodName}: " + "; ".join(msgs) + " — background CGEvent delivery must not steal focus", @@ -405,11 +427,14 @@ def _ax_contains(self, query: str) -> bool: try: wid = self._chrome_window_id() with DriverClient(self.binary) as c: - snap = c.call_tool("get_window_state", { - "pid": self._chrome_pid, - "window_id": wid, - "query": query, - }) + snap = c.call_tool( + "get_window_state", + { + "pid": self._chrome_pid, + "window_id": wid, + "query": query, + }, + ) tree = snap.get("structuredContent", snap).get("tree_markdown", "") return len(tree.strip()) > 0 except Exception: @@ -420,6 +445,7 @@ def _ax_contains(self, query: str) -> bool: # One test class per input type # --------------------------------------------------------------------------- + class TestChromeTextInput(_HermesChromFormBase): """ in Chrome — type_text CGEvent fallback path.""" @@ -431,8 +457,7 @@ def test_text_input(self) -> None: "then type 'Hello Chrome' into it. Stop after typing." ) val = _field("text") - self.assertEqual(val, "Hello Chrome", - msg=f"text='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "Hello Chrome", msg=f"text='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -447,8 +472,7 @@ def test_email_input(self) -> None: "Stop after typing." ) val = _field("email") - self.assertEqual(val, "agent@chrome.com", - msg=f"email='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "agent@chrome.com", msg=f"email='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -463,8 +487,7 @@ def test_password_input(self) -> None: "Stop after typing." ) val = _field("password") - self.assertEqual(val, "Secure123!", - msg=f"password='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "Secure123!", msg=f"password='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -479,8 +502,7 @@ def test_number_input(self) -> None: "Stop after typing." ) val = _field("number") - self.assertEqual(val, "99", - msg=f"number='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "99", msg=f"number='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -495,8 +517,7 @@ def test_textarea(self) -> None: "Stop after typing." ) val = _field("textarea") - self.assertIn("chrome bg ok", val, - msg=f"textarea='{val}'\nhermes: {out[-400:]}") + self.assertIn("chrome bg ok", val, msg=f"textarea='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -510,8 +531,7 @@ def test_checkbox(self) -> None: "Click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). " "Stop after clicking." ) - self.assertTrue(_checkbox_checked(), - msg=f"checkbox not checked\nhermes: {out[-400:]}") + self.assertTrue(_checkbox_checked(), msg=f"checkbox not checked\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -534,8 +554,7 @@ def test_select_dropdown(self) -> None: timeout=180, ) val = _field("select") - self.assertEqual(val, "blue", - msg=f"select='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "blue", msg=f"select='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -556,8 +575,7 @@ def test_submit_button(self) -> None: '(function(){var e=document.getElementById("result");' 'return (e && e.textContent.trim().length > 0) ? "true" : "false";})()' ) - self.assertEqual(submitted, "true", - msg=f"form not submitted\nhermes: {out[-400:]}") + self.assertEqual(submitted, "true", msg=f"form not submitted\nhermes: {out[-400:]}") self._assert_no_new_focus_loss(allow_field_losses=3) @@ -585,17 +603,17 @@ def test_full_form_flow(self) -> None: timeout=360, ) # Verify all three fields were written. - text_val = _field("text") + text_val = _field("text") email_val = _field("email") - num_val = _field("number") + num_val = _field("number") errors: list[str] = [] - if text_val != "E2E Test": errors.append(f"text='{text_val}'") - if email_val != "e2e@test.com": errors.append(f"email='{email_val}'") - if num_val != "7": errors.append(f"number='{num_val}'") - self.assertFalse( - errors, - f"Field mismatch: {'; '.join(errors)}\nhermes: {out[-600:]}" - ) + if text_val != "E2E Test": + errors.append(f"text='{text_val}'") + if email_val != "e2e@test.com": + errors.append(f"email='{email_val}'") + if num_val != "7": + errors.append(f"number='{num_val}'") + self.assertFalse(errors, f"Field mismatch: {'; '.join(errors)}\nhermes: {out[-600:]}") self._assert_no_new_focus_loss(allow_field_losses=3) diff --git a/libs/cua-driver/Tests/integration/test_hermes_form_fill.py b/libs/cua-driver/Tests/integration/test_hermes_form_fill.py index bf7b1c873c..c9933e48fe 100644 --- a/libs/cua-driver/Tests/integration/test_hermes_form_fill.py +++ b/libs/cua-driver/Tests/integration/test_hermes_form_fill.py @@ -52,8 +52,8 @@ _FOCUS_APP_EXE = os.path.join( _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp" ) -_LOSS_FILE = "/tmp/focus_monitor_losses.txt" -_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" +_LOSS_FILE = "/tmp/focus_monitor_losses.txt" +_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" _FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt" _HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes") @@ -71,6 +71,7 @@ # Helpers # --------------------------------------------------------------------------- + def _build_focus_app() -> None: if not os.path.exists(_FOCUS_APP_EXE): subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) @@ -78,7 +79,10 @@ def _build_focus_app() -> None: def _launch_focus_app() -> tuple[subprocess.Popen, int]: proc = subprocess.Popen( - [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + [_FOCUS_APP_EXE], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, ) for _ in range(40): line = proc.stdout.readline().strip() @@ -145,6 +149,7 @@ def _checkbox_checked() -> bool: # Base test class # --------------------------------------------------------------------------- + class _HermesFormBase(unittest.TestCase): """Shared setup/teardown and the hermes runner for all per-input tests.""" @@ -164,8 +169,9 @@ def setUpClass(cls) -> None: # Force rebuild FocusMonitorApp when Swift source is newer. src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift") - if (not os.path.exists(_FOCUS_APP_EXE) - or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)): + if not os.path.exists(_FOCUS_APP_EXE) or os.path.getmtime(src) > os.path.getmtime( + _FOCUS_APP_EXE + ): subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) subprocess.run(["pkill", "-x", "Safari"], check=False) @@ -189,8 +195,8 @@ def tearDownClass(cls) -> None: subprocess.run(["pkill", "-x", "Safari"], check=False) def setUp(self) -> None: - self._losses_before = _read_focus_losses() - self._key_losses_before = _read_key_losses() + self._losses_before = _read_focus_losses() + self._key_losses_before = _read_key_losses() self._field_losses_before = _read_field_losses() def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> str: @@ -202,25 +208,34 @@ def _run_hermes(self, prompt: str, timeout: int = 120, max_turns: int = 5) -> st result = subprocess.run( [ - _HERMES_BIN, "chat", - "--provider", "anthropic", - "-m", _MODEL, - "--toolsets", "computer_use", - "--yolo", # auto-approve every computer_use action - "-Q", # quiet: only final response + session line - "--max-turns", str(max_turns), - "-q", prompt, + _HERMES_BIN, + "chat", + "--provider", + "anthropic", + "-m", + _MODEL, + "--toolsets", + "computer_use", + "--yolo", # auto-approve every computer_use action + "-Q", # quiet: only final response + session line + "--max-turns", + str(max_turns), + "-q", + prompt, ], - env=env, capture_output=True, text=True, timeout=timeout, + env=env, + capture_output=True, + text=True, + timeout=timeout, ) return (result.stdout + "\n" + result.stderr).strip() def _assert_no_new_focus_loss(self) -> None: """Assert no new app-level OR key-level focus losses occurred.""" time.sleep(0.3) # let any async activation settle - app_delta = _read_focus_losses() - self._losses_before - key_delta = _read_key_losses() - self._key_losses_before - field_delta = _read_field_losses() - self._field_losses_before + app_delta = _read_focus_losses() - self._losses_before + key_delta = _read_key_losses() - self._key_losses_before + field_delta = _read_field_losses() - self._field_losses_before msgs = [] if app_delta: msgs.append(f"app lost focus {app_delta}x") @@ -229,7 +244,8 @@ def _assert_no_new_focus_loss(self) -> None: if field_delta: msgs.append(f"text-input lost first-responder {field_delta}x") self.assertEqual( - len(msgs), 0, + len(msgs), + 0, f"Focus stolen in {self._testMethodName}: " + "; ".join(msgs) + " — background events must not steal focus", @@ -240,6 +256,7 @@ def _assert_no_new_focus_loss(self) -> None: # One test class per input type # --------------------------------------------------------------------------- + class TestTextInput(_HermesFormBase): """ — fill with a short string.""" @@ -251,8 +268,7 @@ def test_text_input(self) -> None: "then type 'Hello World' into it. Stop after typing." ) val = _field("text") - self.assertEqual(val, "Hello World", - msg=f"text='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "Hello World", msg=f"text='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -267,8 +283,7 @@ def test_password_input(self) -> None: "Stop after typing." ) val = _field("password") - self.assertEqual(val, "Pass1234!", - msg=f"password='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "Pass1234!", msg=f"password='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -283,8 +298,7 @@ def test_email_input(self) -> None: "Stop after typing." ) val = _field("email") - self.assertEqual(val, "agent@example.com", - msg=f"email='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "agent@example.com", msg=f"email='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -299,8 +313,7 @@ def test_number_input(self) -> None: "Stop after typing." ) val = _field("number") - self.assertEqual(val, "42", - msg=f"number='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "42", msg=f"number='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -315,8 +328,7 @@ def test_textarea(self) -> None: "Stop after typing." ) val = _field("textarea") - self.assertIn("bg works", val, - msg=f"textarea='{val}'\nhermes: {out[-400:]}") + self.assertIn("bg works", val, msg=f"textarea='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -330,8 +342,7 @@ def test_checkbox(self) -> None: "Then AX-click the checkbox labelled 'I agree to the terms' (id='f-checkbox'). " "Stop after clicking." ) - self.assertTrue(_checkbox_checked(), - msg=f"checkbox not checked\nhermes: {out[-400:]}") + self.assertTrue(_checkbox_checked(), msg=f"checkbox not checked\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -355,8 +366,7 @@ def test_select_dropdown(self) -> None: timeout=180, ) val = _field("select") - self.assertEqual(val, "blue", - msg=f"select='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "blue", msg=f"select='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -371,8 +381,7 @@ def test_radio_button(self) -> None: "Stop after clicking." ) val = _field("radio") - self.assertEqual(val, "banana", - msg=f"radio='{val}'\nhermes: {out[-400:]}") + self.assertEqual(val, "banana", msg=f"radio='{val}'\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() @@ -387,8 +396,7 @@ def test_submit_button(self) -> None: "Click it using its element index. Stop after clicking." ) submitted = _js("(window._submitted !== undefined).toString()") - self.assertEqual(submitted, "true", - msg=f"form not submitted\nhermes: {out[-400:]}") + self.assertEqual(submitted, "true", msg=f"form not submitted\nhermes: {out[-400:]}") self._assert_no_new_focus_loss() diff --git a/libs/cua-driver/Tests/integration/test_hermes_form_fill_qwen.py b/libs/cua-driver/Tests/integration/test_hermes_form_fill_qwen.py index 288a36f4ff..9dea3b5086 100644 --- a/libs/cua-driver/Tests/integration/test_hermes_form_fill_qwen.py +++ b/libs/cua-driver/Tests/integration/test_hermes_form_fill_qwen.py @@ -39,8 +39,8 @@ _FOCUS_APP_EXE = os.path.join( _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp" ) -_LOSS_FILE = "/tmp/focus_monitor_losses.txt" -_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" +_LOSS_FILE = "/tmp/focus_monitor_losses.txt" +_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" _FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt" _HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes") @@ -55,9 +55,13 @@ # Helpers (identical to test_hermes_form_fill.py) # --------------------------------------------------------------------------- + def _launch_focus_app() -> tuple[subprocess.Popen, int]: proc = subprocess.Popen( - [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + [_FOCUS_APP_EXE], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, ) for _ in range(40): line = proc.stdout.readline().strip() @@ -103,6 +107,7 @@ def _field(name: str) -> str: # Base test class # --------------------------------------------------------------------------- + class _QwenFormBase(unittest.TestCase): """Shared setup/teardown for Qwen-backed hermes form-fill tests.""" @@ -112,7 +117,9 @@ class _QwenFormBase(unittest.TestCase): @classmethod def setUpClass(cls) -> None: # Verify the local server is reachable before spending time on setup. - import urllib.request, urllib.error + import urllib.error + import urllib.request + try: urllib.request.urlopen(_ENDPOINT.rstrip("/") + "/models", timeout=3) except urllib.error.URLError as e: @@ -126,8 +133,9 @@ def setUpClass(cls) -> None: os.remove(f) src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift") - if (not os.path.exists(_FOCUS_APP_EXE) - or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)): + if not os.path.exists(_FOCUS_APP_EXE) or os.path.getmtime(src) > os.path.getmtime( + _FOCUS_APP_EXE + ): subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) subprocess.run(["pkill", "-x", "Safari"], check=False) @@ -146,8 +154,8 @@ def tearDownClass(cls) -> None: subprocess.run(["pkill", "-x", "Safari"], check=False) def setUp(self) -> None: - self._losses_before = _read_file_int(_LOSS_FILE) - self._key_losses_before = _read_file_int(_KEY_LOSS_FILE) + self._losses_before = _read_file_int(_LOSS_FILE) + self._key_losses_before = _read_file_int(_KEY_LOSS_FILE) self._field_losses_before = _read_file_int(_FIELD_LOSS_FILE) def _run_hermes(self, prompt: str, timeout: int = 180, max_turns: int = 6) -> str: @@ -159,34 +167,46 @@ def _run_hermes(self, prompt: str, timeout: int = 180, max_turns: int = 6) -> st # (provider: custom, base_url: http://127.0.0.1:8080/v1, model: Qwen3.6…) result = subprocess.run( [ - _HERMES_BIN, "chat", - "--toolsets", "computer_use", + _HERMES_BIN, + "chat", + "--toolsets", + "computer_use", "--yolo", "-Q", - "--max-turns", str(max_turns), - "-q", prompt, + "--max-turns", + str(max_turns), + "-q", + prompt, ], - env=env, capture_output=True, text=True, timeout=timeout, + env=env, + capture_output=True, + text=True, + timeout=timeout, ) return (result.stdout + "\n" + result.stderr).strip() def _assert_no_new_focus_loss(self) -> None: time.sleep(0.3) - app_delta = _read_file_int(_LOSS_FILE) - self._losses_before - key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before + app_delta = _read_file_int(_LOSS_FILE) - self._losses_before + key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before msgs = [] - if app_delta: msgs.append(f"app lost focus {app_delta}x") - if key_delta: msgs.append(f"window lost key status {key_delta}x") - if field_delta: msgs.append(f"text-input lost first-responder {field_delta}x") - self.assertEqual(len(msgs), 0, - f"Focus stolen in {self._testMethodName}: " + "; ".join(msgs)) + if app_delta: + msgs.append(f"app lost focus {app_delta}x") + if key_delta: + msgs.append(f"window lost key status {key_delta}x") + if field_delta: + msgs.append(f"text-input lost first-responder {field_delta}x") + self.assertEqual( + len(msgs), 0, f"Focus stolen in {self._testMethodName}: " + "; ".join(msgs) + ) # --------------------------------------------------------------------------- # Tests — start with text input only; add more once text passes # --------------------------------------------------------------------------- + class TestQwenTextInput(_QwenFormBase): """ — verifies multimodal tool results work with mlx-vlm.""" @@ -198,8 +218,7 @@ def test_text_input(self) -> None: "then type 'Hello World' into it. Stop after typing." ) val = _field("text") - self.assertEqual(val, "Hello World", - msg=f"text='{val}'\nhermes: {out[-600:]}") + self.assertEqual(val, "Hello World", msg=f"text='{val}'\nhermes: {out[-600:]}") self._assert_no_new_focus_loss() @@ -214,8 +233,7 @@ def test_email_input(self) -> None: "Stop after typing." ) val = _field("email") - self.assertEqual(val, "agent@example.com", - msg=f"email='{val}'\nhermes: {out[-600:]}") + self.assertEqual(val, "agent@example.com", msg=f"email='{val}'\nhermes: {out[-600:]}") self._assert_no_new_focus_loss() diff --git a/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py index 8239deb616..1763d0e7f1 100644 --- a/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py +++ b/libs/cua-driver/Tests/integration/test_hermes_launch_safari_bg.py @@ -50,8 +50,8 @@ _FOCUS_APP_EXE = os.path.join( _FOCUS_APP_DIR, "FocusMonitorApp.app", "Contents", "MacOS", "FocusMonitorApp" ) -_LOSS_FILE = "/tmp/focus_monitor_losses.txt" -_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" +_LOSS_FILE = "/tmp/focus_monitor_losses.txt" +_KEY_LOSS_FILE = "/tmp/focus_monitor_key_losses.txt" _FIELD_LOSS_FILE = "/tmp/focus_monitor_field_losses.txt" _HERMES_BIN = os.path.expanduser("~/.hermes/hermes-agent/venv/bin/hermes") @@ -60,12 +60,13 @@ _MODEL = os.environ.get("HERMES_TEST_MODEL", "claude-haiku-4-5-20251001") _SAFARI_BUNDLE = "com.apple.Safari" -_LAUNCH_URL = "https://example.com" +_LAUNCH_URL = "https://example.com" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _read_file_int(path: str) -> int: try: with open(path) as f: @@ -93,9 +94,7 @@ def _safari_window_count_via_driver(wait_s: float = 4.0) -> int: r = c.call_tool("list_windows", {}) sc = r.get("structuredContent") or {} wins = sc.get("windows") or [] - count = sum( - 1 for w in wins if "safari" in w.get("app_name", "").lower() - ) + count = sum(1 for w in wins if "safari" in w.get("app_name", "").lower()) if count > 0 or time.time() >= deadline: return count time.sleep(0.5) @@ -103,7 +102,10 @@ def _safari_window_count_via_driver(wait_s: float = 4.0) -> int: def _launch_focus_app() -> tuple[subprocess.Popen, int]: proc = subprocess.Popen( - [_FOCUS_APP_EXE], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + [_FOCUS_APP_EXE], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, ) for _ in range(40): line = proc.stdout.readline().strip() @@ -118,6 +120,7 @@ def _launch_focus_app() -> tuple[subprocess.Popen, int]: # Test # --------------------------------------------------------------------------- + class TestHermesLaunchSafariBg(unittest.TestCase): """Verify hermes launches Safari in the background without stealing focus.""" @@ -133,8 +136,9 @@ def setUpClass(cls) -> None: # Build FocusMonitorApp if needed. src = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.swift") - if (not os.path.exists(_FOCUS_APP_EXE) - or os.path.getmtime(src) > os.path.getmtime(_FOCUS_APP_EXE)): + if not os.path.exists(_FOCUS_APP_EXE) or os.path.getmtime(src) > os.path.getmtime( + _FOCUS_APP_EXE + ): subprocess.run([os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True) # Kill Safari so we can detect a fresh launch. @@ -159,8 +163,8 @@ def setUpClass(cls) -> None: prev = cur time.sleep(0.3) - cls._losses_before = _read_file_int(_LOSS_FILE) - cls._key_losses_before = _read_file_int(_KEY_LOSS_FILE) + cls._losses_before = _read_file_int(_LOSS_FILE) + cls._key_losses_before = _read_file_int(_KEY_LOSS_FILE) cls._field_losses_before = _read_file_int(_FIELD_LOSS_FILE) @classmethod @@ -190,23 +194,32 @@ def _run_hermes(self, prompt: str, timeout: int = 240, max_turns: int = 8) -> st result = subprocess.run( [ - _HERMES_BIN, "chat", - "--provider", "anthropic", - "-m", _MODEL, - "--toolsets", "computer_use", + _HERMES_BIN, + "chat", + "--provider", + "anthropic", + "-m", + _MODEL, + "--toolsets", + "computer_use", "--yolo", "-Q", - "--max-turns", str(max_turns), - "-q", prompt, + "--max-turns", + str(max_turns), + "-q", + prompt, ], - env=env, capture_output=True, text=True, timeout=timeout, + env=env, + capture_output=True, + text=True, + timeout=timeout, ) return (result.stdout + "\n" + result.stderr).strip() def _assert_no_new_focus_loss(self, allow_field_losses: int = 1) -> None: time.sleep(0.3) - app_delta = _read_file_int(_LOSS_FILE) - self._losses_before - key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before + app_delta = _read_file_int(_LOSS_FILE) - self._losses_before + key_delta = _read_file_int(_KEY_LOSS_FILE) - self._key_losses_before field_delta = _read_file_int(_FIELD_LOSS_FILE) - self._field_losses_before msgs = [] if app_delta: @@ -215,11 +228,11 @@ def _assert_no_new_focus_loss(self, allow_field_losses: int = 1) -> None: msgs.append(f"window lost key status {key_delta}x") if field_delta > allow_field_losses: msgs.append( - f"text-input lost first-responder {field_delta}x " - f"(allowed {allow_field_losses})" + f"text-input lost first-responder {field_delta}x " f"(allowed {allow_field_losses})" ) self.assertEqual( - len(msgs), 0, + len(msgs), + 0, "Focus stolen during launch_app: " + "; ".join(msgs), ) @@ -256,7 +269,8 @@ def test_launch_safari_background_no_focus_steal(self) -> None: # the URL loads, so _safari_window_count_via_driver polls for up to 4s. window_count = _safari_window_count_via_driver(wait_s=4.0) self.assertGreater( - window_count, 0, + window_count, + 0, f"cua-driver sees no Safari windows after launch_app+URL " f"(got {window_count}). " f"Safari may have launched without a window — ensure urls=['{_LAUNCH_URL}'] " diff --git a/libs/cua-driver/Tests/integration/test_hidden_app_capture.py b/libs/cua-driver/Tests/integration/test_hidden_app_capture.py index 962c614b47..b7d7331b99 100644 --- a/libs/cua-driver/Tests/integration/test_hidden_app_capture.py +++ b/libs/cua-driver/Tests/integration/test_hidden_app_capture.py @@ -33,7 +33,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from driver_client import DriverClient, default_binary_path, reset_calculator - CALCULATOR_BUNDLE = "com.apple.calculator" @@ -52,12 +51,8 @@ def tearDown(self) -> None: def test_hidden_window_appears_in_list_windows(self) -> None: """launch_app → window appears in list_windows even if off-screen.""" - result = self.client.call_tool( - "launch_app", {"bundle_id": CALCULATOR_BUNDLE} - ) - self.assertFalse( - result.get("isError"), f"launch_app failed: {result}" - ) + result = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE}) + self.assertFalse(result.get("isError"), f"launch_app failed: {result}") pid = result["structuredContent"]["pid"] self.assertIsInstance(pid, int) self.assertGreater(pid, 0) @@ -66,9 +61,7 @@ def test_hidden_window_appears_in_list_windows(self) -> None: time.sleep(1.0) # list_windows (no filter) must include Calculator's window. - all_windows = self.client.call_tool("list_windows", {})[ - "structuredContent" - ]["windows"] + all_windows = self.client.call_tool("list_windows", {})["structuredContent"]["windows"] calc_windows = [w for w in all_windows if w["pid"] == pid] self.assertGreater( len(calc_windows), @@ -84,20 +77,16 @@ def test_hidden_window_appears_in_list_windows(self) -> None: def test_screenshot_succeeds_for_hidden_window(self) -> None: """screenshot tool captures the backing store even when is_on_screen=false.""" - result = self.client.call_tool( - "launch_app", {"bundle_id": CALCULATOR_BUNDLE} - ) + result = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE}) self.assertFalse(result.get("isError"), f"launch_app failed: {result}") pid = result["structuredContent"]["pid"] time.sleep(1.0) # Grab a window_id for Calculator. - windows_result = self.client.call_tool("list_windows", {"pid": pid})[ - "structuredContent" - ]["windows"] - self.assertGreater( - len(windows_result), 0, "No windows found for Calculator" - ) + windows_result = self.client.call_tool("list_windows", {"pid": pid})["structuredContent"][ + "windows" + ] + self.assertGreater(len(windows_result), 0, "No windows found for Calculator") window_id = windows_result[0]["window_id"] # screenshot must succeed regardless of is_on_screen. diff --git a/libs/cua-driver/Tests/integration/test_launch_app_visible.py b/libs/cua-driver/Tests/integration/test_launch_app_visible.py index c9584a1bd7..ea29065e91 100644 --- a/libs/cua-driver/Tests/integration/test_launch_app_visible.py +++ b/libs/cua-driver/Tests/integration/test_launch_app_visible.py @@ -29,7 +29,8 @@ def _on_screen_windows(driver, pid: int) -> list[dict]: result = driver.call_tool("list_windows", {"pid": pid}) windows = result.get("structuredContent", {}).get("windows", []) return [ - w for w in windows + w + for w in windows if w.get("is_on_screen") and (w.get("bounds", {}).get("width", 0) or 0) > 50 and (w.get("bounds", {}).get("height", 0) or 0) > 50 @@ -39,12 +40,13 @@ def _on_screen_windows(driver, pid: int) -> list[dict]: def _pid_from_result(result: dict) -> int: """Extract pid from a launch_app result (structuredContent or text fallback).""" import re + pid = result.get("structuredContent", {}).get("pid", 0) if pid: return pid for item in result.get("content", []): text = item.get("text", "") - m = re.search(r'\bpid\s+(\d+)', text) + m = re.search(r"\bpid\s+(\d+)", text) if m: return int(m.group(1)) return 0 @@ -52,6 +54,7 @@ def _pid_from_result(result: dict) -> int: # ── module-scoped setup ─────────────────────────────────────────────────────── + @pytest.fixture(scope="module", autouse=True) def _kill_test_apps(): """Kill test apps before the module so each test cold-launches cleanly.""" @@ -66,6 +69,7 @@ def _kill_test_apps(): # ── tests ───────────────────────────────────────────────────────────────────── + def test_textedit_cold_launch_opens_visible_window(driver, focus_monitor, ux_guard): """launch_app(TextEdit) must open an on-screen window with 0 focus losses.""" result = driver.call_tool("launch_app", {"bundle_id": TEXTEDIT_BUNDLE}) diff --git a/libs/cua-driver/Tests/integration/test_list_windows.py b/libs/cua-driver/Tests/integration/test_list_windows.py index 65b272ede1..f52937c181 100644 --- a/libs/cua-driver/Tests/integration/test_list_windows.py +++ b/libs/cua-driver/Tests/integration/test_list_windows.py @@ -26,7 +26,6 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from driver_client import DriverClient, default_binary_path, reset_calculator - CALCULATOR_BUNDLE = "com.apple.calculator" @@ -37,9 +36,7 @@ def setUp(self) -> None: # Launch Calculator so we have at least one guaranteed window to # anchor the assertions on. launch_app is idempotent; it'll # return the pid whether or not Calculator was already running. - result = self.client.call_tool( - "launch_app", {"bundle_id": CALCULATOR_BUNDLE} - ) + result = self.client.call_tool("launch_app", {"bundle_id": CALCULATOR_BUNDLE}) self.calc_pid = result["structuredContent"]["pid"] time.sleep(0.5) @@ -47,22 +44,24 @@ def tearDown(self) -> None: self.client.__exit__(None, None, None) def _call_list_windows(self, **args): - return self.client.call_tool("list_windows", args)[ - "structuredContent" - ] + return self.client.call_tool("list_windows", args)["structuredContent"] def test_default_returns_layer_zero_windows_with_new_field_names(self): body = self._call_list_windows() windows = body["windows"] - self.assertGreater( - len(windows), 0, "expected at least Calculator's window" - ) + self.assertGreater(len(windows), 0, "expected at least Calculator's window") # Field contract — window_id / app_name / title are the user-facing # renames from WindowInfo.id / owner / name. required = { - "window_id", "pid", "app_name", "title", "bounds", - "layer", "z_index", "is_on_screen", + "window_id", + "pid", + "app_name", + "title", + "bounds", + "layer", + "z_index", + "is_on_screen", } for w in windows: self.assertTrue( @@ -80,9 +79,7 @@ def test_default_returns_layer_zero_windows_with_new_field_names(self): def test_pid_filter_narrows_to_one_pid(self): body = self._call_list_windows(pid=self.calc_pid) windows = body["windows"] - self.assertGreater( - len(windows), 0, "Calculator should have at least one window" - ) + self.assertGreater(len(windows), 0, "Calculator should have at least one window") self.assertTrue( all(w["pid"] == self.calc_pid for w in windows), "pid filter leaked other pids into the result", @@ -94,9 +91,7 @@ def test_pid_filter_narrows_to_one_pid(self): def test_on_screen_only_drops_off_screen_entries(self): everything = self._call_list_windows()["windows"] - only_visible = self._call_list_windows(on_screen_only=True)[ - "windows" - ] + only_visible = self._call_list_windows(on_screen_only=True)["windows"] # The visible subset should never exceed the full set. self.assertLessEqual(len(only_visible), len(everything)) diff --git a/libs/cua-driver/Tests/integration/test_overlay_z_order.py b/libs/cua-driver/Tests/integration/test_overlay_z_order.py index ea48239e92..f2c0ff9eeb 100644 --- a/libs/cua-driver/Tests/integration/test_overlay_z_order.py +++ b/libs/cua-driver/Tests/integration/test_overlay_z_order.py @@ -31,7 +31,11 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from driver_client import DriverClient, default_binary_path, resolve_window_id # noqa: E402 +from driver_client import ( # noqa: E402 + DriverClient, + default_binary_path, + resolve_window_id, +) _THIS_DIR = os.path.dirname(os.path.abspath(__file__)) _REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) @@ -122,10 +126,9 @@ def test_overlay_at_normal_level_above_target(self) -> None: # Windows above Calculator that are NOT the driver overlay. fg_wins_before = [ - w for w in all_before - if w["z_index"] > calc_z_before - and w["pid"] != driver_pid - and w.get("is_on_screen") + w + for w in all_before + if w["z_index"] > calc_z_before and w["pid"] != driver_pid and w.get("is_on_screen") ] # Trigger a pixel click to make the overlay appear and pin above Calculator. @@ -158,9 +161,7 @@ def test_overlay_at_normal_level_above_target(self) -> None: # ── Assertion 2: overlay is above the target ──────────────────────────── calc_wins_after = [w for w in all_after if w["pid"] == self.calc_pid] - calc_z_after = ( - max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0 - ) + calc_z_after = max(w["z_index"] for w in calc_wins_after) if calc_wins_after else 0 self.assertGreater( overlay_z, calc_z_after, diff --git a/libs/cua-driver/Tests/integration/test_safari.py b/libs/cua-driver/Tests/integration/test_safari.py index 160a33e3d7..2f75b6234f 100644 --- a/libs/cua-driver/Tests/integration/test_safari.py +++ b/libs/cua-driver/Tests/integration/test_safari.py @@ -13,31 +13,33 @@ from __future__ import annotations +import os +import re import subprocess import sys import time -import os -import re import pytest _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver def _parse_clicks(tree_text: str) -> int: """Extract the 'clicks: N' counter value from the AX tree.""" - m = re.search(r'clicks:\s*(\d+)', tree_text) + m = re.search(r"clicks:\s*(\d+)", tree_text) return int(m.group(1)) if m else 0 + SAFARI_BUNDLE = "com.apple.Safari" FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp" # ── module-level Safari setup ───────────────────────────────────────────────── + @pytest.fixture(scope="module") def safari_pid(binary, html_server, focus_monitor): """Open Safari to test_page.html; return its pid. @@ -68,8 +70,11 @@ def _reactivate_focus(focus_monitor): """Restore FocusMonitorApp as frontmost before each test (by pid).""" _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -77,6 +82,7 @@ def _reactivate_focus(focus_monitor): # ── helpers ─────────────────────────────────────────────────────────────────── + def _get_state(d: Driver, pid: int, safari_pid_fixture) -> "harness.driver.WindowState": wid = d.find_window(pid) return d.get_window_state(pid, wid) @@ -84,6 +90,7 @@ def _get_state(d: Driver, pid: int, safari_pid_fixture) -> "harness.driver.Windo # ── tests ───────────────────────────────────────────────────────────────────── + class TestSafariButton: """AX-click the 'Click Me' button without stealing focus.""" @@ -102,9 +109,9 @@ def test_click_increments_counter(self, driver, safari_pid, ux_guard): after = driver.get_window_state(safari_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 1, ( - f"Counter did not increment: {count_before} → {count_after}" - ) + assert ( + count_after == count_before + 1 + ), f"Counter did not increment: {count_before} → {count_after}" def test_click_increments_twice(self, driver, safari_pid, ux_guard): wid = driver.find_window(safari_pid) @@ -123,9 +130,9 @@ def test_click_increments_twice(self, driver, safari_pid, ux_guard): after = driver.get_window_state(safari_pid, wid) count_after = _parse_clicks(after.tree) print(f"\n click counter: {count_before} → {count_after}") - assert count_after == count_before + 2, ( - f"Counter should have incremented by 2: {count_before} → {count_after}" - ) + assert ( + count_after == count_before + 2 + ), f"Counter should have incremented by 2: {count_before} → {count_after}" class TestSafariTextInput: @@ -150,9 +157,7 @@ def test_type_text_appears_in_ax_tree(self, driver, safari_pid, ux_guard): time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("hello safari"), ( - f"Typed text not visible in AX tree:\n{after.tree}" - ) + assert after.has_text("hello safari"), f"Typed text not visible in AX tree:\n{after.tree}" def test_type_text_with_type_text_tool(self, driver, safari_pid, ux_guard): """Use type_text (not type_text_chars) and verify result.""" @@ -172,9 +177,7 @@ def test_type_text_with_type_text_tool(self, driver, safari_pid, ux_guard): time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("type_text test"), ( - f"type_text result not in AX tree:\n{after.tree}" - ) + assert after.has_text("type_text test"), f"type_text result not in AX tree:\n{after.tree}" class TestSafariCheckbox: @@ -194,9 +197,7 @@ def test_checkbox_toggle(self, driver, safari_pid, ux_guard): time.sleep(0.4) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("checked"), ( - f"Checkbox did not toggle to checked:\n{after.tree}" - ) + assert after.has_text("checked"), f"Checkbox did not toggle to checked:\n{after.tree}" class TestSafariSelect: @@ -224,16 +225,21 @@ def test_select_option_b(self, driver, safari_pid, ux_guard): else: # Fallback: set value directly. Safari AXPopUpButton uses the # display text ("Option B") not the option's value attr ("b"). - driver.call_tool("set_value", { - "pid": safari_pid, "window_id": wid, - "element_index": idx, "value": "Option B", - }) + driver.call_tool( + "set_value", + { + "pid": safari_pid, + "window_id": wid, + "element_index": idx, + "value": "Option B", + }, + ) time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) - assert after.has_text("Option B") or after.has_text("selected: Option B"), ( - f"Dropdown did not select Option B:\n{after.tree}" - ) + assert after.has_text("Option B") or after.has_text( + "selected: Option B" + ), f"Dropdown did not select Option B:\n{after.tree}" class TestSafariTextarea: @@ -263,19 +269,22 @@ def test_textarea_input(self, driver, safari_pid, ux_guard): time.sleep(0.8) # longer settle for WKWebView textarea focus # Step 3: type — keystrokes go to the focused element - driver.call_tool("type_text_chars", { - "pid": safari_pid, - "text": "multi line", - "delay_ms": 50, - }) + driver.call_tool( + "type_text_chars", + { + "pid": safari_pid, + "text": "multi line", + "delay_ms": 50, + }, + ) time.sleep(0.6) after = driver.get_window_state(safari_pid, wid) val = Tree.ax_value(after.tree, ta_idx) print(f"\n textarea AX value: {val!r}") - assert after.has_text("multi line"), ( - f"Textarea text not in AX tree; val={val!r};\n{after.tree[:800]}" - ) + assert after.has_text( + "multi line" + ), f"Textarea text not in AX tree; val={val!r};\n{after.tree[:800]}" class TestSafariCanvas: @@ -296,6 +305,7 @@ def test_canvas_pixel_click_no_focus_steal(self, driver, safari_pid, ux_guard): print(f"\n screenshot: {before.screenshot_width}x{before.screenshot_height}") from harness.cv import decode, diff_ratio + before_img = decode(before.screenshot_b64) # Canvas is the last section on the page. With ~80px browser chrome diff --git a/libs/cua-driver/Tests/integration/test_tauri.py b/libs/cua-driver/Tests/integration/test_tauri.py index e1bb465f56..ed233bdc7c 100644 --- a/libs/cua-driver/Tests/integration/test_tauri.py +++ b/libs/cua-driver/Tests/integration/test_tauri.py @@ -11,23 +11,24 @@ from __future__ import annotations +import os import subprocess import sys import time -import os import pytest try: import requests as _requests + _REQUESTS_AVAILABLE = True except ImportError: _REQUESTS_AVAILABLE = False _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) -from harness.driver import Driver from harness import tree as Tree +from harness.driver import Driver _TAURI_API = "http://localhost:6769" @@ -55,8 +56,11 @@ def _http_reset(base_url: str = _TAURI_API) -> None: def _reactivate_focus(focus_monitor): _, pid = focus_monitor subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true'], + [ + "osascript", + "-e", + f'tell application "System Events" to set frontmost of (first process whose unix id is {pid}) to true', + ], check=False, ) time.sleep(0.4) @@ -64,6 +68,7 @@ def _reactivate_focus(focus_monitor): # ── tests ───────────────────────────────────────────────────────────────────── + class TestTauriButton: def test_button_click(self, driver, tauri_app, ux_guard): _, pid, base_url = tauri_app @@ -128,9 +133,9 @@ def test_type_text(self, driver, tauri_app, ux_guard): key_events = [e for e in events if e.get("type") in ("key_down", "keydown", "input")] print(f" key events: {len(key_events)}") - assert text_in_tree or len(key_events) >= len("tauri test"), ( - f"Text not found in AX tree and insufficient key events:\n{after.tree}" - ) + assert text_in_tree or len(key_events) >= len( + "tauri test" + ), f"Text not found in AX tree and insufficient key events:\n{after.tree}" class TestTauriNoFocusSteal: diff --git a/libs/cua-driver/Tests/integration/test_webkit_js.py b/libs/cua-driver/Tests/integration/test_webkit_js.py index a3d46f697c..1ecb1c99bd 100644 --- a/libs/cua-driver/Tests/integration/test_webkit_js.py +++ b/libs/cua-driver/Tests/integration/test_webkit_js.py @@ -27,13 +27,14 @@ from driver_client import DriverClient, default_binary_path CONDUCTOR_BUNDLE = "com.conductor.app" -CONDUCTOR_PATH = "/Applications/Conductor.app" +CONDUCTOR_PATH = "/Applications/Conductor.app" # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _tool_text(result: dict) -> str: for item in result.get("content", []): if item.get("type") == "text": @@ -48,13 +49,14 @@ def _conductor_installed() -> bool: def _kill_conductor() -> None: subprocess.run( ["osascript", "-e", 'tell application "Conductor" to quit'], - check=False, timeout=5, + check=False, + timeout=5, ) time.sleep(1) def _find_pid(text: str) -> int | None: - m = re.search(r'pid[=:\s]+(\d+)', text, re.IGNORECASE) + m = re.search(r"pid[=:\s]+(\d+)", text, re.IGNORECASE) return int(m.group(1)) if m else None @@ -63,14 +65,14 @@ def _main_window(client: DriverClient, pid: int) -> int | None: text = _tool_text(client.call_tool("list_windows", {"pid": pid})) for line in text.splitlines(): if "on_screen=True" in line or "is_on_screen=True" in line: - m = re.search(r'\[window_id:\s*(\d+)\]', line) + m = re.search(r"\[window_id:\s*(\d+)\]", line) if m: return int(m.group(1)) # Fall back to any titled window m = re.search(r'"[^"]+"\s+\[window_id:\s*(\d+)\]', text) if m: return int(m.group(1)) - m = re.search(r'\[window_id:\s*(\d+)\]', text) + m = re.search(r"\[window_id:\s*(\d+)\]", text) return int(m.group(1)) if m else None @@ -78,6 +80,7 @@ def _main_window(client: DriverClient, pid: int) -> int | None: # Conductor / Tauri AX fallback tests # --------------------------------------------------------------------------- + @unittest.skipUnless(_conductor_installed(), "Conductor not installed") class WebKitJSTests(unittest.TestCase): """AX-based page primitives work end-to-end against a live Conductor window.""" @@ -124,12 +127,15 @@ def tearDownClass(cls) -> None: def test_13_execute_javascript_returns_clear_error_for_wkwebview(self) -> None: """execute_javascript on a WKWebView app returns a clear error.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._pid, - "window_id": self._window_id, - "action": "execute_javascript", - "javascript": "document.title", - }) + result = c.call_tool( + "page", + { + "pid": self._pid, + "window_id": self._window_id, + "action": "execute_javascript", + "javascript": "document.title", + }, + ) self.assertTrue(result.get("isError"), "Expected isError=True for WKWebView JS") text = _tool_text(result) self.assertIn("WKWebView", text) @@ -142,11 +148,14 @@ def test_13_execute_javascript_returns_clear_error_for_wkwebview(self) -> None: def test_11_ax_get_text_works_without_inspector(self) -> None: """get_text returns page text via AX tree (no inspector needed).""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._pid, - "window_id": self._window_id, - "action": "get_text", - }) + result = c.call_tool( + "page", + { + "pid": self._pid, + "window_id": self._window_id, + "action": "get_text", + }, + ) self.assertFalse(result.get("isError"), _tool_text(result)) text = _tool_text(result) self.assertTrue(len(text.strip()) > 0, "get_text returned empty content") @@ -158,19 +167,22 @@ def test_11_ax_get_text_works_without_inspector(self) -> None: def test_12_ax_query_dom_buttons(self) -> None: """query_dom for button elements returns AX tree results.""" with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": self._pid, - "window_id": self._window_id, - "action": "query_dom", - "css_selector": "button", - }) + result = c.call_tool( + "page", + { + "pid": self._pid, + "window_id": self._window_id, + "action": "query_dom", + "css_selector": "button", + }, + ) text = _tool_text(result) self.assertNotIn("Traceback", text) # Either found buttons (role key) or returned empty — both are valid if not result.get("isError"): self.assertTrue( "AXButton" in text or "role" in text or "[]" in text, - f"Unexpected query_dom output: {text[:200]}" + f"Unexpected query_dom output: {text[:200]}", ) # ----------------------------------------------------------------------- @@ -186,7 +198,7 @@ def test_chrome_not_detected_as_wkwebview(self) -> None: chrome_pid = None for line in text.splitlines(): if "com.google.Chrome" in line: - m = re.search(r'pid[=:\s]+(\d+)', line, re.IGNORECASE) + m = re.search(r"pid[=:\s]+(\d+)", line, re.IGNORECASE) if m: chrome_pid = int(m.group(1)) break @@ -197,22 +209,24 @@ def test_chrome_not_detected_as_wkwebview(self) -> None: with DriverClient(self.binary) as c: windows_text = _tool_text(c.call_tool("list_windows", {"pid": chrome_pid})) - m = re.search(r'\[window_id:\s*(\d+)\]', windows_text) + m = re.search(r"\[window_id:\s*(\d+)\]", windows_text) if not m: self.skipTest("No Chrome window found") window_id = int(m.group(1)) with DriverClient(self.binary) as c: - result = c.call_tool("page", { - "pid": chrome_pid, - "window_id": window_id, - "action": "execute_javascript", - "javascript": "1+1", - }) + result = c.call_tool( + "page", + { + "pid": chrome_pid, + "window_id": window_id, + "action": "execute_javascript", + "javascript": "1+1", + }, + ) text = _tool_text(result) self.assertNotIn( - "WKWebView", text, - f"Chrome was incorrectly detected as WKWebView app: {text[:200]}" + "WKWebView", text, f"Chrome was incorrectly detected as WKWebView app: {text[:200]}" ) diff --git a/libs/python/cua-sandbox/cua_sandbox/transport/cloud.py b/libs/python/cua-sandbox/cua_sandbox/transport/cloud.py index 8b518c49be..89e6d3ed28 100644 --- a/libs/python/cua-sandbox/cua_sandbox/transport/cloud.py +++ b/libs/python/cua-sandbox/cua_sandbox/transport/cloud.py @@ -388,10 +388,12 @@ async def _post_with_retry(self, path: str, body: dict) -> httpx.Response: return resp if attempt == _CREATE_MAX_RETRIES - 1: break - delay = _CREATE_RETRY_BASE_S * (2 ** attempt) + delay = _CREATE_RETRY_BASE_S * (2**attempt) logger.warning( "No sandbox capacity (503), retrying in %.1fs (attempt %d/%d)", - delay, attempt + 1, _CREATE_MAX_RETRIES, + delay, + attempt + 1, + _CREATE_MAX_RETRIES, ) await asyncio.sleep(delay) return resp # return last 503 response, caller will raise_for_status diff --git a/libs/python/cua-sandbox/cua_sandbox/transport/http.py b/libs/python/cua-sandbox/cua_sandbox/transport/http.py index 2a73ecc4dc..3d1ce479f9 100644 --- a/libs/python/cua-sandbox/cua_sandbox/transport/http.py +++ b/libs/python/cua-sandbox/cua_sandbox/transport/http.py @@ -98,10 +98,14 @@ async def _cmd(self, command: str, params: Optional[Dict[str, Any]] = None) -> D resp = await self._client.post("/cmd", json=body, timeout=req_timeout) if resp.status_code < 500 or attempt == _CMD_MAX_RETRIES - 1: break - backoff = _CMD_RETRY_BACKOFF_S * (2 ** attempt) + backoff = _CMD_RETRY_BACKOFF_S * (2**attempt) logger.debug( "[http] /cmd %s returned %d, retrying in %.1fs (attempt %d/%d)", - command, resp.status_code, backoff, attempt + 1, _CMD_MAX_RETRIES, + command, + resp.status_code, + backoff, + attempt + 1, + _CMD_MAX_RETRIES, ) await asyncio.sleep(backoff) diff --git a/libs/python/cua-sandbox/tests/test_transport_http_retry.py b/libs/python/cua-sandbox/tests/test_transport_http_retry.py index 39526111e7..7d9109d661 100644 --- a/libs/python/cua-sandbox/tests/test_transport_http_retry.py +++ b/libs/python/cua-sandbox/tests/test_transport_http_retry.py @@ -11,11 +11,9 @@ from __future__ import annotations import json -from itertools import cycle import httpx import pytest - from cua_sandbox.transport.http import HTTPTransport pytestmark = pytest.mark.asyncio @@ -61,11 +59,13 @@ async def _no_sleep(_seconds): monkeypatch.setattr(http_mod.asyncio, "sleep", _no_sleep) - responses = iter([ - httpx.Response(503, text="service unavailable"), - httpx.Response(502, text="bad gateway"), - httpx.Response(200, text=_sse_body({"ok": True})), - ]) + responses = iter( + [ + httpx.Response(503, text="service unavailable"), + httpx.Response(502, text="bad gateway"), + httpx.Response(200, text=_sse_body({"ok": True})), + ] + ) calls = [] def handler(request: httpx.Request) -> httpx.Response: