diff --git a/libs/python/computer-server/computer_server/mcp_server.py b/libs/python/computer-server/computer_server/mcp_server.py index ea7bbbcf1f..9cc4d8edf5 100644 --- a/libs/python/computer-server/computer_server/mcp_server.py +++ b/libs/python/computer-server/computer_server/mcp_server.py @@ -38,6 +38,7 @@ def _get_handlers(): global _handlers if _handlers is None: from .handlers.factory import HandlerFactory + _handlers = HandlerFactory.create_handlers() return _handlers @@ -56,6 +57,7 @@ async def _detect_actual_resolution_async() -> Tuple[int, int]: def _detect_actual_resolution() -> Tuple[int, int]: """Detect the actual screen resolution (sync wrapper).""" import asyncio + try: loop = asyncio.get_event_loop() if loop.is_running(): @@ -125,7 +127,7 @@ def create_mcp_server() -> FastMCP: type text, press keys, scroll, manage windows, read/write files, and run commands. Always take a screenshot first to see the current state before performing actions. - After performing actions, take another screenshot to verify the result.""" + After performing actions, take another screenshot to verify the result.""", ) # ============================================================ @@ -140,9 +142,10 @@ async def computer_screenshot() -> Image: Returns the current screen state as an image. Always call this first to see what's on screen before performing any actions. """ - from PIL import Image as PILImage from io import BytesIO + from PIL import Image as PILImage + _, automation_handler, _, _, _, _ = _get_handlers() result = await automation_handler.screenshot() image_data = base64.b64decode(result["image_data"]) @@ -167,9 +170,14 @@ async def computer_screenshot() -> Image: new_width = int(width * max_dimension / height) img = img.resize((new_width, new_height), PILImage.Resampling.LANCZOS) + # Initialize coordinate scaling if not already configured + # This ensures clicks are properly scaled from resized space to actual device space + if not _target_width: + _configure_scaling(target_width=new_width, target_height=new_height) + # Convert to RGB if necessary (for JPEG compatibility) - if img.mode in ('RGBA', 'P'): - img = img.convert('RGB') + if img.mode in ("RGBA", "P"): + img = img.convert("RGB") # Use JPEG with quality setting to ensure small file size buffered = BytesIO() @@ -218,11 +226,7 @@ async def computer_get_cursor_position() -> Dict[str, Any]: return result @mcp.tool - async def computer_click( - x: int, - y: int, - button: str = "left" - ) -> Dict[str, Any]: + async def computer_click(x: int, y: int, button: str = "left") -> Dict[str, Any]: """ Click at the specified screen coordinates. @@ -276,7 +280,7 @@ async def computer_drag( end_x: int, end_y: int, button: str = "left", - duration: float = 0.5 + duration: float = 0.5, ) -> Dict[str, Any]: """ Drag from start coordinates to end coordinates. @@ -295,14 +299,13 @@ async def computer_drag( _, automation_handler, _, _, _, _ = _get_handlers() # Move to start position first await automation_handler.move_cursor(actual_start_x, actual_start_y) - return await automation_handler.drag_to(actual_end_x, actual_end_y, button=button, duration=duration) + return await automation_handler.drag_to( + actual_end_x, actual_end_y, button=button, duration=duration + ) @mcp.tool async def computer_scroll( - x: int, - y: int, - scroll_x: int = 0, - scroll_y: int = 0 + x: int, y: int, scroll_x: int = 0, scroll_y: int = 0 ) -> Dict[str, Any]: """ Scroll at the specified position. @@ -321,9 +324,7 @@ async def computer_scroll( @mcp.tool async def computer_mouse_down( - x: Optional[int] = None, - y: Optional[int] = None, - button: str = "left" + x: Optional[int] = None, y: Optional[int] = None, button: str = "left" ) -> Dict[str, Any]: """ Press and hold a mouse button. @@ -342,9 +343,7 @@ async def computer_mouse_down( @mcp.tool async def computer_mouse_up( - x: Optional[int] = None, - y: Optional[int] = None, - button: str = "left" + x: Optional[int] = None, y: Optional[int] = None, button: str = "left" ) -> Dict[str, Any]: """ Release a mouse button. @@ -794,9 +793,7 @@ async def computer_get_accessibility_tree() -> Dict[str, Any]: @mcp.tool async def computer_find_element( - role: Optional[str] = None, - title: Optional[str] = None, - value: Optional[str] = None + role: Optional[str] = None, title: Optional[str] = None, value: Optional[str] = None ) -> Dict[str, Any]: """ Find a UI element in the accessibility tree.