diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index c995c6c8c311..22259377ca7c 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -59,6 +59,7 @@ "wait", "list_apps", "focus_app", + "switch_desktop", ], "description": ( "Which action to perform. `capture` is free (no side " @@ -208,6 +209,16 @@ "matching the background co-work model." ), }, + # ── switch_desktop ────────────────────────────────────── + "direction": { + "type": "string", + "enum": ["left", "right"], + "description": ( + "Only for action='switch_desktop'. Switches to the " + "adjacent virtual desktop. Requires Windows 10+ with " + "multiple virtual desktops." + ), + }, # ── return shape ─────────────────────────────────────── "capture_after": { "type": "boolean", @@ -219,6 +230,17 @@ }, }, "required": ["action"], + "allOf": [ + { + "if": { + "properties": {"action": {"const": "switch_desktop"}}, + "required": ["action"], + }, + "then": { + "required": ["direction"], + }, + }, + ], }, } diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 1143afbe7837..c83183b62230 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -426,6 +426,13 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> res = backend.set_value(value=str(value), element=args.get("element")) return _maybe_follow_capture(backend, res, capture_after) + if action == "switch_desktop": + direction = args.get("direction", "") + if not hasattr(backend, "switch_desktop"): + return json.dumps({"error": "switch_desktop not supported by current backend"}) + res = backend.switch_desktop(str(direction)) + return _maybe_follow_capture(backend, res, capture_after) + return json.dumps({"error": f"unknown action {action!r}"}) diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 8f7506d8d4cd..f9e878f29182 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -295,6 +295,52 @@ def _press_combo(vks: List[int]) -> None: _send_inputs(seq) +def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: + """Switch virtual desktop via Ctrl+Win+Left/Right SendInput. + + The overlay subprocess (full-screen tkinter window) is killed by any + virtual-desktop transition, so we stop it before switching and restart + it on the new desktop. + """ + if direction not in ("left", "right"): + return False + + VK_CONTROL = 0x11 + VK_LWIN = 0x5B + VK_LEFT = 0x25 + VK_RIGHT = 0x27 + + vk_dir = VK_LEFT if direction == "left" else VK_RIGHT + + # Single-batch SendInput matching _press_combo semantics: + # hold modifiers → tap arrow → release, so the system input thread + # sees the same event order as a physical keyboard. + seq = [_key_event(VK_CONTROL, True), + _key_event(VK_LWIN, True), + _key_event(vk_dir, True), + _key_event(vk_dir, False), + _key_event(VK_LWIN, False), + _key_event(VK_CONTROL, False)] + + try: + overlay_client.stop() + _send_inputs(seq) + # The virtual-desktop transition is asynchronous — a brief delay + # before restarting the overlay avoids racing the DWM compositor. + time.sleep(0.15) + overlay_client._dead = False + overlay_client.start() + return True + except Exception as e: + logger.warning("switch_desktop SendInput failed: %s", e) + try: + overlay_client._dead = False + overlay_client.start() + except Exception as e2: + logger.warning("switch_desktop overlay restart also failed: %s", e2) + return False + + def _type_unicode(text: str) -> None: """Type text via KEYEVENTF_UNICODE; newlines become Return taps.""" batch: List[_INPUT] = [] @@ -990,6 +1036,23 @@ def key(self, keys: str) -> ActionResult: except Exception as e: return ActionResult(ok=False, action="key", message=f"key failed: {e}") + def switch_desktop(self, direction: str) -> ActionResult: + """Switch to adjacent virtual desktop. + + Temporarily stops the overlay subprocess before switching and + restarts it on the new desktop, because any SendInput-based + virtual-desktop transition kills the full-screen tkinter window. + """ + if direction not in ("left", "right"): + return ActionResult(ok=False, action="switch_desktop", + message=f"unknown direction {direction!r}") + self._overlay.send({"cmd": "flash", "text": f"switch desktop · {direction}", "ttl": 1.0}) + ok = _switch_desktop_via_keybd(direction, self._overlay) + return ActionResult( + ok=ok, action="switch_desktop", + message=(f"switched to {direction} virtual desktop" + if ok else "virtual desktop switch failed")) + # ── Native-value mutation ─────────────────────────────────────── def set_value(self, value: str, element: Optional[int] = None) -> ActionResult: if element is None: