From 91231ff0e167151800f28d73b5bf12f02bd5768b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Mon, 15 Jun 2026 12:49:07 +0800 Subject: [PATCH 1/3] feat(computer_use): add switch_desktop with overlay-safe restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop the overlay subprocess before switching virtual desktops, then restart it on the new desktop — avoids the tkinter display context teardown that kills the overlay during SendInput-based Ctrl+Win+Left/Right. - _switch_desktop_via_keybd(): stop overlay → two-phase SendInput → restart overlay - switch_desktop() method on WindowsUIABackend - Schema and dispatch updated with 'switch_desktop' action and 'direction' parameter Co-authored-by: lEWFkRAD --- tools/computer_use/schema.py | 11 +++++ tools/computer_use/tool.py | 7 +++ tools/computer_use/windows_backend.py | 61 +++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index c995c6c8c311..b83ab60f6047 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -59,6 +59,7 @@ "wait", "list_apps", "focus_app", + "switch_desktop", ], "description": ( "Which action to perform. `capture` is free (no side " @@ -208,6 +209,16 @@ "matching the background co-work model." ), }, + # ── switch_desktop ────────────────────────────────────── + "direction": { + "type": "string", + "enum": ["left", "right"], + "description": ( + "Only for action='switch_desktop'. Switches to the " + "adjacent virtual desktop. Requires Windows 10+ with " + "multiple virtual desktops." + ), + }, # ── return shape ─────────────────────────────────────── "capture_after": { "type": "boolean", diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index 1143afbe7837..e4be71b8f0c1 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -426,6 +426,13 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> res = backend.set_value(value=str(value), element=args.get("element")) return _maybe_follow_capture(backend, res, capture_after) + if action == "switch_desktop": + direction = args.get("direction", "") + if not hasattr(backend, "switch_desktop"): + return json.dumps({"error": "switch_desktop not supported by current backend"}) + res = backend.switch_desktop(str(direction)) + return json.dumps({"ok": res.ok, "message": res.message}) + return json.dumps({"error": f"unknown action {action!r}"}) diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 8f7506d8d4cd..2211993dccb1 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -295,6 +295,50 @@ def _press_combo(vks: List[int]) -> None: _send_inputs(seq) +def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: + """Switch virtual desktop, keeping the overlay alive across the transition. + + Any SendInput-based virtual-desktop switch kills the full-screen tkinter + overlay subprocess because the display-context change tears down its X11 + connection. Workaround: stop the overlay → switch → restart it on the + new desktop. + """ + VK_CONTROL = 0x11 + VK_LWIN = 0x5B + VK_LEFT = 0x25 + VK_RIGHT = 0x27 + + vk_dir = VK_LEFT if direction == "left" else VK_RIGHT + + press = [_key_event(VK_CONTROL, True), + _key_event(VK_LWIN, True), + _key_event(vk_dir, True)] + release = [_key_event(vk_dir, False), + _key_event(VK_LWIN, False), + _key_event(VK_CONTROL, False)] + + try: + # 1. Gracefully stop the overlay before switching + overlay_client.stop() + + # 2. Switch virtual desktop + _send_inputs(press) + time.sleep(0.08) + _send_inputs(release) + + # 3. Restart overlay on the new desktop + overlay_client._dead = False # we killed it on purpose, not a crash + overlay_client.start() + return True + except Exception: + try: + overlay_client._dead = False + overlay_client.start() + except Exception: + pass + return False + + def _type_unicode(text: str) -> None: """Type text via KEYEVENTF_UNICODE; newlines become Return taps.""" batch: List[_INPUT] = [] @@ -990,6 +1034,23 @@ def key(self, keys: str) -> ActionResult: except Exception as e: return ActionResult(ok=False, action="key", message=f"key failed: {e}") + def switch_desktop(self, direction: str) -> ActionResult: + """Switch to adjacent virtual desktop. + + Temporarily stops the overlay subprocess before switching and + restarts it on the new desktop, because any SendInput-based + virtual-desktop transition kills the full-screen tkinter window. + """ + if direction not in ("left", "right"): + return ActionResult(ok=False, action="switch_desktop", + message=f"unknown direction {direction!r}") + self._overlay.send({"cmd": "flash", "text": f"switch desktop · {direction}", "ttl": 1.0}) + ok = _switch_desktop_via_keybd(direction, self._overlay) + return ActionResult( + ok=ok, action="switch_desktop", + message=(f"switched to {direction} virtual desktop" + if ok else "virtual desktop switch failed")) + # ── Native-value mutation ─────────────────────────────────────── def set_value(self, value: str, element: Optional[int] = None) -> ActionResult: if element is None: From c7b59bc4d8a42f45b87d2fa7b4c1a4c955093d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:49:15 +0800 Subject: [PATCH 2/3] fix(computer_use): use single-batch SendInput for switch_desktop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-batch SendInput (press → sleep → release) crashes the embedded gateway (Dashboard Chat tab) because its single-channel event loop cannot handle multi-batch keyboard injection without disrupting the PTY pipeline. The full system gateway is unaffected because its multi-client dispatch loop handles concurrent channels. Switch to single-batch SendInput matching _press_combo semantics (hold modifiers → tap arrow → release). This works in both embedded and full gateway modes. --- tools/computer_use/windows_backend.py | 35 ++++++++++++--------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 2211993dccb1..62db4bb4c10c 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -296,12 +296,11 @@ def _press_combo(vks: List[int]) -> None: def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: - """Switch virtual desktop, keeping the overlay alive across the transition. + """Switch virtual desktop via Ctrl+Win+Left/Right SendInput. - Any SendInput-based virtual-desktop switch kills the full-screen tkinter - overlay subprocess because the display-context change tears down its X11 - connection. Workaround: stop the overlay → switch → restart it on the - new desktop. + The overlay subprocess (full-screen tkinter window) is killed by any + virtual-desktop transition, so we stop it before switching and restart + it on the new desktop. """ VK_CONTROL = 0x11 VK_LWIN = 0x5B @@ -310,24 +309,20 @@ def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: vk_dir = VK_LEFT if direction == "left" else VK_RIGHT - press = [_key_event(VK_CONTROL, True), - _key_event(VK_LWIN, True), - _key_event(vk_dir, True)] - release = [_key_event(vk_dir, False), - _key_event(VK_LWIN, False), - _key_event(VK_CONTROL, False)] + # Single-batch SendInput matching _press_combo semantics: + # hold modifiers → tap arrow → release, so the system input thread + # sees the same event order as a physical keyboard. + seq = [_key_event(VK_CONTROL, True), + _key_event(VK_LWIN, True), + _key_event(vk_dir, True), + _key_event(vk_dir, False), + _key_event(VK_LWIN, False), + _key_event(VK_CONTROL, False)] try: - # 1. Gracefully stop the overlay before switching overlay_client.stop() - - # 2. Switch virtual desktop - _send_inputs(press) - time.sleep(0.08) - _send_inputs(release) - - # 3. Restart overlay on the new desktop - overlay_client._dead = False # we killed it on purpose, not a crash + _send_inputs(seq) + overlay_client._dead = False overlay_client.start() return True except Exception: From 94e4f72139082fddbd51ff5ab51d88dcbd1a4ab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=81=B5=E8=B6=8A=E7=BE=BD=E6=AF=9B?= <97326386+Icather@users.noreply.github.com> Date: Mon, 15 Jun 2026 17:00:34 +0800 Subject: [PATCH 3/3] fix(computer_use): address Copilot review feedback - Validate direction before key dispatch (unknown values return False) - Log exceptions instead of silently swallowing them - Add 150ms delay before overlay restart to avoid DWM race - Route switch_desktop through _maybe_follow_capture for consistency - Add JSON Schema if/then to require direction for switch_desktop --- tools/computer_use/schema.py | 11 +++++++++++ tools/computer_use/tool.py | 2 +- tools/computer_use/windows_backend.py | 13 ++++++++++--- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tools/computer_use/schema.py b/tools/computer_use/schema.py index b83ab60f6047..22259377ca7c 100644 --- a/tools/computer_use/schema.py +++ b/tools/computer_use/schema.py @@ -230,6 +230,17 @@ }, }, "required": ["action"], + "allOf": [ + { + "if": { + "properties": {"action": {"const": "switch_desktop"}}, + "required": ["action"], + }, + "then": { + "required": ["direction"], + }, + }, + ], }, } diff --git a/tools/computer_use/tool.py b/tools/computer_use/tool.py index e4be71b8f0c1..c83183b62230 100644 --- a/tools/computer_use/tool.py +++ b/tools/computer_use/tool.py @@ -431,7 +431,7 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) -> if not hasattr(backend, "switch_desktop"): return json.dumps({"error": "switch_desktop not supported by current backend"}) res = backend.switch_desktop(str(direction)) - return json.dumps({"ok": res.ok, "message": res.message}) + return _maybe_follow_capture(backend, res, capture_after) return json.dumps({"error": f"unknown action {action!r}"}) diff --git a/tools/computer_use/windows_backend.py b/tools/computer_use/windows_backend.py index 62db4bb4c10c..f9e878f29182 100644 --- a/tools/computer_use/windows_backend.py +++ b/tools/computer_use/windows_backend.py @@ -302,6 +302,9 @@ def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: virtual-desktop transition, so we stop it before switching and restart it on the new desktop. """ + if direction not in ("left", "right"): + return False + VK_CONTROL = 0x11 VK_LWIN = 0x5B VK_LEFT = 0x25 @@ -322,15 +325,19 @@ def _switch_desktop_via_keybd(direction: str, overlay_client) -> bool: try: overlay_client.stop() _send_inputs(seq) + # The virtual-desktop transition is asynchronous — a brief delay + # before restarting the overlay avoids racing the DWM compositor. + time.sleep(0.15) overlay_client._dead = False overlay_client.start() return True - except Exception: + except Exception as e: + logger.warning("switch_desktop SendInput failed: %s", e) try: overlay_client._dead = False overlay_client.start() - except Exception: - pass + except Exception as e2: + logger.warning("switch_desktop overlay restart also failed: %s", e2) return False