feat: Computer Use Tool — macOS desktop control via Anthropic native API

agent/anthropic_adapter.py

@@ -93,6 +93,8 @@ def _supports_adaptive_thinking(model: str) -> bool:
 _COMMON_BETAS = [
     "interleaved-thinking-2025-05-14",
     "fine-grained-tool-streaming-2025-05-14",
+    "computer-use-2025-11-24",
+    "context-management-2025-06-27",
 ]
 
 # Additional beta headers required for OAuth/subscription auth.
@@ -1026,8 +1028,23 @@ def convert_messages_to_anthropic(
             continue
 
         if role == "tool":
-            # Sanitize tool_use_id and ensure non-empty content
-            result_content = content if isinstance(content, str) else json.dumps(content)
+            # Sanitize tool_use_id and ensure non-empty content.
+            # Check for multimodal content blocks (computer_use screenshots).
+            # Stored in _anthropic_content_blocks to keep "content" as a string
+            # for compatibility with trajectory/session code paths.
+            multimodal_blocks = m.get("_anthropic_content_blocks")
+            if isinstance(multimodal_blocks, list) and multimodal_blocks:
+                # Include text content alongside image blocks so Claude sees
+                # the MEDIA: path and can include it in its response for gateway.
+                text_content = content if isinstance(content, str) and content.strip() else None
+                if text_content:
+                    result_content = [{"type": "text", "text": text_content}] + multimodal_blocks
+                else:
+                    result_content = multimodal_blocks
+            elif isinstance(content, str):
+                result_content = content
+            else:
+                result_content = json.dumps(content) if content else "(no output)"
             if not result_content:
                 result_content = "(no output)"
             tool_result = {
@@ -1142,6 +1159,50 @@ def convert_messages_to_anthropic(
             fixed.append(m)
     result = fixed
 
+    # ── Image eviction: keep only the most recent N screenshots ─────
+    # computer_use screenshots (base64 images) sit inside tool_result blocks:
+    #   msg["content"] = [{"type": "tool_result", "content": [{"type": "image", ...}]}]
+    # They accumulate and are sent with every API call. Each costs ~1,465
+    # tokens; after 10+ the conversation becomes very slow even for simple
+    # text queries. Walk backward, keep the most recent _MAX_KEEP_IMAGES,
+    # replace older ones with a text placeholder.
+    #
+    # Performance vs context trade-off:
+    #   1 (default) — fastest, model only sees the latest screenshot
+    #   2-3         — model can compare before/after states (useful for
+    #                 verifying multi-step UI changes) but adds ~1.5K
+    #                 tokens per extra image, slowing every API call
+    #   5+          — rarely useful, significant latency impact
+    #
+    # The model almost always decides based on the most recent screenshot
+    # alone, so keeping 1 is the best default. Increase only if the agent
+    # needs explicit before/after comparison for a specific workflow.
+    _MAX_KEEP_IMAGES = 3
+    _image_count = 0
+    for msg in reversed(result):
+        content = msg.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict) or block.get("type") != "tool_result":
+                continue
+            inner = block.get("content")
+            if not isinstance(inner, list):
+                continue
+            has_image = any(
+                isinstance(b, dict) and b.get("type") == "image"
+                for b in inner
+            )
+            if not has_image:
+                continue
+            _image_count += 1
+            if _image_count > _MAX_KEEP_IMAGES:
+                block["content"] = [
+                    b if b.get("type") != "image"
+                    else {"type": "text", "text": "[screenshot removed to save context]"}
+                    for b in inner
+                ]
+
     return system, result
 
 
@@ -1155,6 +1216,8 @@ def build_anthropic_kwargs(
     is_oauth: bool = False,
     preserve_dots: bool = False,
     context_length: Optional[int] = None,
+    native_tools: Optional[List[Dict]] = None,
+    context_management: Optional[Dict[str, Any]] = None,
 ) -> Dict[str, Any]:
     """Build kwargs for anthropic.messages.create().
 
@@ -1168,6 +1231,10 @@ def build_anthropic_kwargs(
 
     When *preserve_dots* is True, model name dots are not converted to hyphens
     (for Alibaba/DashScope anthropic-compatible endpoints: qwen3.5-plus).
+
+    When *context_management* is provided, enables server-side context editing
+    (e.g. clearing old tool results). Only used with computer_use to reduce
+    token costs from accumulated screenshots.
     """
     system, anthropic_messages = convert_messages_to_anthropic(messages)
     anthropic_tools = convert_tools_to_anthropic(tools) if tools else []
@@ -1180,6 +1247,13 @@ def build_anthropic_kwargs(
     if context_length and effective_max_tokens > context_length:
         effective_max_tokens = max(context_length - 1, 1)
 
+    # Append native Anthropic tool types (e.g. computer_use) that bypass
+    # the OpenAI-to-Anthropic conversion — they use Anthropic's own format.
+    # Must happen BEFORE OAuth prefixing so native tools also get the mcp_
+    # prefix, keeping tool definitions consistent with message history.
+    if native_tools:
+        anthropic_tools.extend(native_tools)
+
     # ── OAuth: Claude Code identity ──────────────────────────────────
     if is_oauth:
         # 1. Prepend Claude Code system prompt identity
@@ -1203,19 +1277,25 @@ def build_anthropic_kwargs(
                 block["text"] = text
 
         # 3. Prefix tool names with mcp_ (Claude Code convention)
+        #    Skip native Anthropic tool types (e.g. computer_20251124) —
+        #    their names are fixed by the API and must not be prefixed.
+        _NATIVE_TOOL_TYPES = {"computer_20251124", "text_editor_20250124", "bash_20250124"}
         if anthropic_tools:
             for tool in anthropic_tools:
-                if "name" in tool:
+                if "name" in tool and tool.get("type") not in _NATIVE_TOOL_TYPES:
                     tool["name"] = _MCP_TOOL_PREFIX + tool["name"]
 
         # 4. Prefix tool names in message history (tool_use and tool_result blocks)
+        #    Skip native tool names (e.g. "computer") — same reason as step 3.
+        _native_tool_names = {t["name"] for t in (native_tools or []) if "name" in t}
         for msg in anthropic_messages:
             content = msg.get("content")
             if isinstance(content, list):
                 for block in content:
                     if isinstance(block, dict):
                         if block.get("type") == "tool_use" and "name" in block:
-                            if not block["name"].startswith(_MCP_TOOL_PREFIX):
+                            if (not block["name"].startswith(_MCP_TOOL_PREFIX)
+                                    and block["name"] not in _native_tool_names):
                                 block["name"] = _MCP_TOOL_PREFIX + block["name"]
                         elif block.get("type") == "tool_result" and "tool_use_id" in block:
                             pass  # tool_result uses ID, not name
@@ -1229,6 +1309,12 @@ def build_anthropic_kwargs(
     if system:
         kwargs["system"] = system
 
+    # Server-side context editing (beta) — clears old tool results to
+    # reduce token costs. Currently only enabled for computer_use sessions
+    # where accumulated screenshots bloat context rapidly.
+    if context_management:
+        kwargs["context_management"] = context_management
+
     if anthropic_tools:
         kwargs["tools"] = anthropic_tools
         # Map OpenAI tool_choice to Anthropic format

agent/context_compressor.py

@@ -174,7 +174,21 @@ def _prune_old_tool_results(
             content = msg.get("content", "")
             if not content or content == _PRUNED_TOOL_PLACEHOLDER:
                 continue
-            # Only prune if the content is substantial (>200 chars)
+            # Prune multimodal tool results (e.g. computer_use screenshots)
+            # regardless of text content length — the base64 image data in
+            # _anthropic_content_blocks is ~1MB per screenshot but the text
+            # summary is only ~85 chars, so the len(content) > 200 check
+            # below would never trigger. Strip the image blocks explicitly.
+            has_images = isinstance(msg.get("_anthropic_content_blocks"), list) and msg.get("_anthropic_content_blocks")
+            if has_images:
+                result[i] = {
+                    k: v for k, v in msg.items()
+                    if k != "_anthropic_content_blocks"
+                }
+                result[i]["content"] = _PRUNED_TOOL_PLACEHOLDER
+                pruned += 1
+                continue
+            # Only prune text-only tool results if the content is substantial (>200 chars)
             if len(content) > 200:
                 result[i] = {**msg, "content": _PRUNED_TOOL_PLACEHOLDER}
                 pruned += 1

agent/display.py

@@ -153,6 +153,50 @@ def build_tool_preview(tool_name: str, args: dict, max_len: int | None = None) -
         "clarify": "question", "skill_manage": "name",
     }
 
+    if tool_name == "computer":
+        action = args.get("action", "?")
+        coord = args.get("coordinate")
+        text = args.get("text", "")
+        if action == "screenshot":
+            return "screenshot"
+        if action == "zoom":
+            region = args.get("region")
+            return f"zoom {region}" if region else "zoom"
+        if action in ("left_click", "right_click", "double_click", "triple_click", "middle_click"):
+            label = action.replace("_", " ")
+            pos = f" ({coord[0]}, {coord[1]})" if coord and len(coord) == 2 else ""
+            mod = f" [{text}]" if text else ""
+            return f"{label}{pos}{mod}"
+        if action == "left_click_drag":
+            start = args.get("start_coordinate")
+            end = args.get("end_coordinate") or coord
+            s = f"({start[0]},{start[1]})" if start and len(start) == 2 else "?"
+            e = f"({end[0]},{end[1]})" if end and len(end) == 2 else "?"
+            return f"drag {s}->{e}"
+        if action == "type":
+            preview = _oneline(text)[:30]
+            return f'type "{preview}{"..." if len(text) > 30 else ""}"'
+        if action == "key":
+            key_combo = args.get("key", text)
+            return f"key {key_combo}"
+        if action == "hold_key":
+            key = args.get("key", text)
+            dur = args.get("duration", 1)
+            return f"hold {key} {dur}s"
+        if action == "scroll":
+            direction = args.get("scroll_direction", "down")
+            amount = args.get("scroll_amount", 3)
+            return f"scroll {direction} x{amount}"
+        if action == "wait":
+            dur = args.get("duration", 1)
+            return f"wait {dur}s"
+        if action == "mouse_move":
+            pos = f" ({coord[0]}, {coord[1]})" if coord and len(coord) == 2 else ""
+            return f"move{pos}"
+        if action in ("left_mouse_down", "left_mouse_up"):
+            return action.replace("left_mouse_", "mouse ")
+        return action
+
     if tool_name == "process":
         action = args.get("action", "")
         sid = args.get("session_id", "")
@@ -838,6 +882,47 @@ def _wrap(line: str) -> str:
             return line
         return f"{line}{failure_suffix}"
 
+    if tool_name == "computer":
+        action = args.get("action", "?")
+        coord = args.get("coordinate")
+        text = args.get("text", "")
+        _pos = f" ({coord[0]},{coord[1]})" if coord and len(coord) == 2 else ""
+        if action == "screenshot":
+            return _wrap(f"┊ 🖥️  screen    capture  {dur}")
+        if action == "zoom":
+            return _wrap(f"┊ 🖥️  zoom      region  {dur}")
+        if action in ("left_click", "right_click", "double_click", "triple_click", "middle_click"):
+            label = action.replace("_click", "").replace("_", " ")
+            mod = f" [{text}]" if text else ""
+            return _wrap(f"┊ 🖥️  click     {label}{_pos}{mod}  {dur}")
+        if action == "left_click_drag":
+            start = args.get("start_coordinate")
+            end = args.get("end_coordinate") or coord
+            s = f"({start[0]},{start[1]})" if start and len(start) == 2 else "?"
+            e = f"({end[0]},{end[1]})" if end and len(end) == 2 else "?"
+            return _wrap(f"┊ 🖥️  drag      {s}->{e}  {dur}")
+        if action == "type":
+            return _wrap(f"┊ 🖥️  type      \"{_trunc(text, 30)}\"  {dur}")
+        if action == "key":
+            key_combo = args.get("key", text)
+            return _wrap(f"┊ 🖥️  key       {key_combo}  {dur}")
+        if action == "hold_key":
+            key = args.get("key", text)
+            hold_dur = args.get("duration", 1)
+            return _wrap(f"┊ 🖥️  hold      {key} {hold_dur}s  {dur}")
+        if action == "scroll":
+            direction = args.get("scroll_direction", "down")
+            amount = args.get("scroll_amount", 3)
+            return _wrap(f"┊ 🖥️  scroll    {direction} x{amount}  {dur}")
+        if action == "wait":
+            wait_dur = args.get("duration", 1)
+            return _wrap(f"┊ 🖥️  wait      {wait_dur}s  {dur}")
+        if action == "mouse_move":
+            return _wrap(f"┊ 🖥️  move      {_pos}  {dur}")
+        if action in ("left_mouse_down", "left_mouse_up"):
+            label = "press" if "down" in action else "release"
+            return _wrap(f"┊ 🖥️  mouse     {label}{_pos}  {dur}")
+        return _wrap(f"┊ 🖥️  computer  {action}  {dur}")
     if tool_name == "web_search":
         return _wrap(f"┊ 🔍 search    {_trunc(args.get('query', ''), 42)}  {dur}")
     if tool_name == "web_extract":

agent/model_metadata.py

@@ -903,9 +903,45 @@ def estimate_tokens_rough(text: str) -> int:
 
 
 def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int:
-    """Rough token estimate for a message list (pre-flight only)."""
-    total_chars = sum(len(str(msg)) for msg in messages)
-    return total_chars // 4
+    """Rough token estimate for a message list (pre-flight only).
+
+    Excludes base64 image data from ``_anthropic_content_blocks`` which would
+    massively overcount tokens (a single screenshot's base64 is ~1MB of chars
+    but only costs ~1,465 API tokens).  Instead, each image block is counted
+    as a flat 1,500 tokens (Anthropic formula: width*height/750 for typical
+    1300x845 screenshots).
+    """
+    _IMAGE_TOKEN_ESTIMATE = 1500
+    total = 0
+    for msg in messages:
+        if not isinstance(msg, dict):
+            total += len(str(msg))
+            continue
+        # Count text content normally
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            total += len(content)
+        elif isinstance(content, list):
+            for block in content:
+                if isinstance(block, str):
+                    total += len(block)
+                elif isinstance(block, dict):
+                    total += len(block.get("text", ""))
+        # Count tool_calls args (but not the huge function schema)
+        for tc in msg.get("tool_calls", []):
+            if isinstance(tc, dict):
+                fn = tc.get("function", {})
+                total += len(fn.get("arguments", ""))
+        # Count _anthropic_content_blocks: images as flat estimate, text normally
+        for block in msg.get("_anthropic_content_blocks", []):
+            if isinstance(block, dict):
+                if block.get("type") == "image":
+                    total += _IMAGE_TOKEN_ESTIMATE * 4  # * 4 because we divide by 4 below
+                else:
+                    total += len(block.get("text", ""))
+        # Role/metadata overhead
+        total += 20  # role, tool_call_id, etc.
+    return total // 4
 
 
 def estimate_request_tokens_rough(
@@ -920,12 +956,14 @@ def estimate_request_tokens_rough(
     system prompt, conversation messages, and tool schemas.  With 50+
     tools enabled, schemas alone can add 20-30K tokens — a significant
     blind spot when only counting messages.
+
+    Uses ``estimate_messages_tokens_rough`` for messages to avoid
+    counting base64 image data as text tokens.
     """
     total_chars = 0
     if system_prompt:
         total_chars += len(system_prompt)
-    if messages:
-        total_chars += sum(len(str(msg)) for msg in messages)
+    msg_tokens = estimate_messages_tokens_rough(messages) if messages else 0
     if tools:
         total_chars += len(str(tools))
-    return total_chars // 4
+    return total_chars // 4 + msg_tokens