From ee256395b9960d0ea4e55585802998019475144d Mon Sep 17 00:00:00 2001 From: Jeeves Assistant Date: Mon, 22 Jun 2026 06:22:42 -0500 Subject: [PATCH 1/2] fix(computer-use): preserve cua structured element bounds --- tests/tools/test_cua_backend_capture.py | 204 ++++++++++++++++++++++++ tools/computer_use/cua_backend.py | 99 +++++++++--- 2 files changed, 285 insertions(+), 18 deletions(-) create mode 100644 tests/tools/test_cua_backend_capture.py diff --git a/tests/tools/test_cua_backend_capture.py b/tests/tools/test_cua_backend_capture.py new file mode 100644 index 0000000000000..1c44832f6e56f --- /dev/null +++ b/tests/tools/test_cua_backend_capture.py @@ -0,0 +1,204 @@ +"""Regression tests for cua-driver capture routing. + +Recent cua-driver releases no longer expose the old standalone ``screenshot`` +MCP tool. Hermes must request screenshots through ``get_window_state`` with the +appropriate ``capture_mode`` instead. cua-driver 0.6.0 also returns element +geometry in structured ``elements`` records; Hermes should prefer those over the +back-compat markdown tree so SOM overlays and element-index actions stay useful. +""" + +from __future__ import annotations + +from typing import Any, Dict +from unittest.mock import MagicMock + + +_PNG_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42m" + "NkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" +) + + +def _window_list() -> Dict[str, Any]: + return { + "data": "", + "images": [], + "structuredContent": { + "windows": [ + { + "app_name": "Safari", + "pid": 1234, + "window_id": 5678, + "is_on_screen": True, + "title": "Test Page", + "z_index": 0, + } + ] + }, + "isError": False, + } + + +def _make_backend(get_window_state_response: Dict[str, Any]): + from tools.computer_use.cua_backend import CuaDriverBackend + + backend = CuaDriverBackend() + backend._session = MagicMock() + + responses = { + "list_windows": _window_list(), + "get_window_state": get_window_state_response, + } + + def _call_tool(name, args): + if name in responses: + return responses[name] + return { + "data": f"Unknown tool: {name}", + "images": [], + "structuredContent": None, + "isError": True, + } + + backend._session.call_tool.side_effect = _call_tool + return backend + + +def _tool_call_names(backend): + return [call.args[0] for call in backend._session.call_tool.call_args_list] + + +def _get_window_state_args(backend): + for call in backend._session.call_tool.call_args_list: + if call.args[0] == "get_window_state": + return call.args[1] + raise AssertionError("get_window_state was not called") + + +class TestCuaVisionCapture: + def test_vision_uses_get_window_state_not_screenshot(self): + backend = _make_backend({ + "data": "", + "images": [_PNG_B64], + "structuredContent": { + "screenshot_width": 1920, + "screenshot_height": 1080, + }, + "isError": False, + }) + + result = backend.capture(mode="vision") + + tool_names = _tool_call_names(backend) + assert "screenshot" not in tool_names + assert "get_window_state" in tool_names + assert _get_window_state_args(backend)["capture_mode"] == "vision" + assert result.png_b64 == _PNG_B64 + assert result.width == 1920 + assert result.height == 1080 + assert result.png_bytes_len > 0 + + +class TestCuaSomCapture: + def test_som_passes_capture_mode_and_returns_png_plus_elements(self): + backend = _make_backend({ + "data": "✅ Safari — 2 elements\n[1] AXButton \"Go\"\n[2] AXTextField \"Search\"", + "images": [_PNG_B64], + "structuredContent": { + "screenshot_width": 1280, + "screenshot_height": 800, + }, + "isError": False, + }) + + result = backend.capture(mode="som") + + assert _get_window_state_args(backend)["capture_mode"] == "som" + assert result.png_b64 == _PNG_B64 + assert result.width == 1280 + assert result.height == 800 + assert [element.index for element in result.elements] == [1, 2] + + def test_som_prefers_structured_elements_with_bounds(self): + backend = _make_backend({ + "data": "✅ Safari — 1 elements\n[7] AXButton \"Search\"", + "images": [_PNG_B64], + "structuredContent": { + "screenshot_width": 1280, + "screenshot_height": 800, + "elements": [ + { + "element_index": 7, + "role": "AXButton", + "label": "Search", + "frame": {"x": 101.2, "y": 202.6, "w": 88.0, "h": 34.0}, + } + ], + }, + "isError": False, + }) + + result = backend.capture(mode="som") + + assert len(result.elements) == 1 + assert result.elements[0].index == 7 + assert result.elements[0].label == "Search" + assert result.elements[0].bounds == (101, 203, 88, 34) + + +class TestCuaAxCapture: + def test_ax_still_returns_elements_without_png(self): + backend = _make_backend({ + "data": "✅ Safari — 2 elements\n[1] AXButton \"OK\"\n[2] AXStaticText \"Hello\"", + "images": [], + "structuredContent": None, + "isError": False, + }) + + result = backend.capture(mode="ax") + + assert _get_window_state_args(backend)["capture_mode"] == "ax" + assert result.png_b64 is None + assert result.png_bytes_len == 0 + assert result.width == 0 + assert result.height == 0 + assert [element.label for element in result.elements] == ["OK", "Hello"] + + def test_ax_prefers_structured_elements_without_png(self): + backend = _make_backend({ + "data": "✅ Safari — 1 elements\n[3] AXTextField \"Address\"", + "images": [], + "structuredContent": { + "elements": [ + { + "element_index": 3, + "role": "AXTextField", + "label": "Address", + "frame": {"x": 20, "y": 40, "w": 600, "h": 28}, + "enabled": True, + } + ], + }, + "isError": False, + }) + + result = backend.capture(mode="ax") + + assert result.png_b64 is None + assert len(result.elements) == 1 + assert result.elements[0].role == "AXTextField" + assert result.elements[0].bounds == (20, 40, 600, 28) + assert result.elements[0].attributes["enabled"] is True + + def test_missing_structured_content_keeps_dimensions_zero(self): + backend = _make_backend({ + "data": "✅ Safari — 0 elements\n", + "images": [], + "structuredContent": None, + "isError": False, + }) + + result = backend.capture(mode="ax") + + assert result.width == 0 + assert result.height == 0 diff --git a/tools/computer_use/cua_backend.py b/tools/computer_use/cua_backend.py index 4bacefa994bff..5c83001fe779a 100644 --- a/tools/computer_use/cua_backend.py +++ b/tools/computer_use/cua_backend.py @@ -107,6 +107,53 @@ def _parse_windows_from_text(text: str) -> List[Dict[str, Any]]: return windows +def _coerce_int(value: Any) -> int: + """Best-effort conversion for cua-driver numeric fields.""" + try: + return int(round(float(value))) + except (TypeError, ValueError): + return 0 + + +def _parse_elements_from_structured(raw_elements: Any) -> List[UIElement]: + """Parse cua-driver structured elements, preserving AX geometry. + + cua-driver 0.6.0 returns first-class ``elements`` with ``frame`` records; + the markdown tree is now primarily a back-compat rendering and may not + carry geometry. Prefer this path whenever it is available so Hermes SOM + overlays and element-index summaries are spatially useful. + """ + if not isinstance(raw_elements, list): + return [] + + elements: List[UIElement] = [] + for item in raw_elements: + if not isinstance(item, dict): + continue + frame_raw = item.get("frame") + frame: Dict[str, Any] = frame_raw if isinstance(frame_raw, dict) else {} + label = item.get("label") or item.get("title") or item.get("value") or "" + elements.append(UIElement( + index=_coerce_int(item.get("element_index", item.get("index"))), + role=str(item.get("role") or ""), + label=str(label), + bounds=( + _coerce_int(frame.get("x")), + _coerce_int(frame.get("y")), + _coerce_int(frame.get("w", frame.get("width"))), + _coerce_int(frame.get("h", frame.get("height"))), + ), + pid=_coerce_int(item.get("pid")), + window_id=_coerce_int(item.get("window_id")), + attributes={ + key: value + for key, value in item.items() + if key not in {"element_index", "index", "role", "label", "title", "value", "frame", "pid", "window_id"} + }, + )) + return elements + + def _parse_elements_from_tree(markdown: str) -> List[UIElement]: """Parse UIElement list from get_window_state AX tree markdown. @@ -427,7 +474,7 @@ def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult """Capture the frontmost on-screen window (optionally filtered by app name). Maps hermes `capture(mode, app)` → cua-driver `list_windows` + - `get_window_state` (ax/som) or `screenshot` (vision). + `get_window_state` with capture_mode set for the requested capture. """ # Step 1: enumerate on-screen windows to find target pid/window_id. lw_out = self._session.call_tool("list_windows", {"on_screen_only": True}) @@ -497,31 +544,47 @@ def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult width = height = 0 window_title = "" + # cua-driver 0.5.x no longer exposes the old standalone `screenshot` + # MCP tool. Screenshots now come from `get_window_state` when the + # requested capture_mode is `vision` or `som`. + gws_args: Dict[str, Any] = { + "pid": self._active_pid, + "window_id": self._active_window_id, + } + if mode in {"ax", "som", "vision"}: + gws_args["capture_mode"] = mode + + gws_out = self._session.call_tool("get_window_state", gws_args) + + gws_sc = gws_out.get("structuredContent") or {} + if isinstance(gws_sc, dict): + if gws_sc.get("screenshot_width"): + width = int(gws_sc["screenshot_width"]) + if gws_sc.get("screenshot_height"): + height = int(gws_sc["screenshot_height"]) + if mode == "vision": - # screenshot tool: just the PNG, no AX walk. - sc_out = self._session.call_tool( - "screenshot", - {"window_id": self._active_window_id, "format": "jpeg", "quality": 85}, - ) - if sc_out["images"]: - png_b64 = sc_out["images"][0] + # Vision mode: just the PNG, no AX walk. + if gws_out.get("images"): + png_b64 = gws_out["images"][0] else: - # get_window_state: AX tree + optional screenshot. - gws_out = self._session.call_tool( - "get_window_state", - {"pid": self._active_pid, "window_id": self._active_window_id}, - ) + # ax/som mode: AX tree + optional screenshot. text = gws_out["data"] if isinstance(gws_out["data"], str) else "" summary, tree = _split_tree_text(text) + # Extract structured elements first. cua-driver 0.6.0 carries + # geometry there while the markdown tree is only a compatibility + # rendering; falling back preserves older driver support. + structured_elements = _parse_elements_from_structured(gws_sc.get("elements")) + # Parse element count from summary e.g. "✅ AppName — 42 elements, turn 3..." m = re.search(r'(\d+)\s+elements?', summary) - if tree and not gws_out["images"]: + if tree and not gws_out.get("images"): # ax mode — no screenshot - elements = _parse_elements_from_tree(tree) - elif gws_out["images"]: + elements = structured_elements or _parse_elements_from_tree(tree) + elif gws_out.get("images"): png_b64 = gws_out["images"][0] - elements = _parse_elements_from_tree(tree) + elements = structured_elements or _parse_elements_from_tree(tree) # Extract window title from the AX tree first AXWindow line. wt = re.search(r'AXWindow\s+"([^"]+)"', tree) @@ -534,7 +597,7 @@ def capture(self, mode: str = "som", app: Optional[str] = None) -> CaptureResult raw = base64.b64decode(png_b64, validate=False) png_bytes_len = len(raw) detected_width, detected_height = _image_dimensions_from_bytes(raw) - if detected_width and detected_height: + if not (width and height) and detected_width and detected_height: width = detected_width height = detected_height except Exception: From 4fcb01e258fd0a32ca1b1a10f5072bd1be62a913 Mon Sep 17 00:00:00 2001 From: Jeeves Assistant Date: Mon, 22 Jun 2026 06:31:29 -0500 Subject: [PATCH 2/2] chore: map Jeeves Assistant attribution --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index 9dae0c8bc2914..09e1bd2d62f07 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -45,6 +45,7 @@ # Auto-extracted from noreply emails + manual overrides AUTHOR_MAP = { + "jeevesassistant00@gmail.com": "jeeves-assistant", # PR #50771 (computer-use CuaDriver structured element bounds) "rrandqua@gmail.com": "TutkuEroglu", # PR #50481 salvage (AGENTS.md stale token-lock adapter path) "pedro.m.simoes@gmail.com": "pmos69", # PR #29474 salvage (native Antigravity OAuth provider; Gemini CLI sunset #29294/#49701) "mediratta01.pally@gmail.com": "orbisai0security", # PR #9560 salvage (session.py path-traversal guard, V-009)