diff --git a/docs/content/docs/cua-driver/reference/cli-reference.mdx b/docs/content/docs/cua-driver/reference/cli-reference.mdx index 72854e7889..16034b4b8a 100644 --- a/docs/content/docs/cua-driver/reference/cli-reference.mdx +++ b/docs/content/docs/cua-driver/reference/cli-reference.mdx @@ -363,6 +363,7 @@ The following MCP tools are callable via `cua-driver `. For input schemas - `click` — left-click by `element_index` or `(x, y)`. - `double_click` — double-click by `element_index` (AXOpen) or `(x, y)`. - `right_click` — right-click by `element_index` (AXShowMenu) or `(x, y)`. +- `drag` — press-drag-release between two pixel endpoints (marquee, drag-and-drop, slider scrub, resize handles). - `move_cursor` — warp the real cursor to `(x, y)`. **Keyboard** diff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdx index 9ad7d8c293..ccec23346e 100644 --- a/docs/content/docs/cua-driver/reference/mcp-tools.mdx +++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx @@ -186,6 +186,26 @@ Right-click by element (routes through `AXShowMenu`) or by pixel. {"pid": 844, "window_id": 10725, "element_index": 7} ``` +### drag + +Press-drag-release gesture between two pixel endpoints. macOS AX has no semantic drag action, so this is pixel-only — there is no element-indexed mode. Use it for marquee/lasso selection, drag-and-drop, slider scrubbing, resize handles, panel repositioning. Frontmost target: posts via `.cghidEventTap` (real cursor traces the path; required for AppKit drag sources / canvas viewports). Backgrounded target: posts via the auth-signed pid-routed path (cursor-neutral). + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `from_x`, `from_y` (number, required): Drag-start in window-local screenshot pixels (top-left origin). +- `to_x`, `to_y` (number, required): Drag-end in the same space. +- `window_id` (integer, optional): CGWindowID the pixels were measured against. Defaults to the frontmost window of `pid`. +- `duration_ms` (integer, optional): Wall-clock budget for the path between mouseDown and mouseUp. Default: `500`. +- `steps` (integer, optional): Number of intermediate `mouseDragged` events linearly interpolated along the path. Default: `20`. +- `modifier` (array of string, optional): `cmd` / `shift` / `option` / `ctrl`. Held across the entire gesture (option-drag duplicates, shift-drag constrains the axis). +- `button` (string, optional): `left` / `right` / `middle`. Default: `left`. +- `from_zoom` (boolean, optional): When true, all four coordinates are in the last `zoom` image's pixel space. + +```json +{"pid": 844, "window_id": 10725, "from_x": 100, "from_y": 200, "to_x": 400, "to_y": 200, "duration_ms": 600} +``` + ### move_cursor Warp the real mouse cursor to a screen-point coordinate. Does not click. diff --git a/libs/cua-driver/Skills/cua-driver/SKILL.md b/libs/cua-driver/Skills/cua-driver/SKILL.md index 092d808254..3858f74922 100644 --- a/libs/cua-driver/Skills/cua-driver/SKILL.md +++ b/libs/cua-driver/Skills/cua-driver/SKILL.md @@ -150,6 +150,7 @@ contract" above: | Find a pid | `list_apps` or `launch_app`'s return | `pgrep`, `ps`, `osascript frontmost` | | Enumerate an app's windows | `list_windows({pid})` — or read the `windows` array `launch_app` already returns | `osascript 'every window of app …'` | | Click / type / scroll / keys | `click`, `type_text`, `scroll`, `press_key`, `hotkey` | `osascript`, `cliclick`, raw `CGEvent`, `open ` | +| Drag / drag-and-drop / marquee select | `drag({pid, from_x, from_y, to_x, to_y})` (pixel-only — macOS AX has no semantic drag) | `cliclick dd:`, `osascript drag` | | Screenshot | `screenshot` or the PNG in `get_window_state` | `screencapture` | | Quit an app | ask the user first, then `hotkey({pid, keys:["cmd","q"]})` | `kill`, `killall`, `pkill` | | Hand a file/URL to an app | `launch_app({bundle_id, urls:[]})` | `open -a `, `open ` | diff --git a/libs/cua-driver/Sources/CuaDriverCore/Input/MouseInput.swift b/libs/cua-driver/Sources/CuaDriverCore/Input/MouseInput.swift index 439418757d..f9322a72c3 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Input/MouseInput.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Input/MouseInput.swift @@ -432,6 +432,209 @@ public enum MouseInput { ) } + /// Synthesize a press-drag-release gesture from `start` to `end` in + /// screen points. Emits one `mouseDown` at `start`, `steps` + /// linearly-interpolated `mouseDragged` events along the path, and + /// one `mouseUp` at `end`. `durationMs` is the wall-clock budget + /// for the path between down and up; the time is split evenly + /// across the drag steps. + /// + /// Frontmost target: posts via `.cghidEventTap` with a leading + /// `mouseMoved` so the recipient sees a real HID-origin gesture + /// (matches what AppKit drag sources, Finder selection, and + /// canvas-backed viewports expect). The user's real cursor + /// follows the drag path — unavoidable, since `cghidEventTap` is + /// the system input stream. + /// + /// Backgrounded target: posts via `postBoth` (auth-signed + /// SkyLight + public `CGEvent.postToPid`). Cursor-neutral. Some + /// surfaces filter pid-routed mouseDragged events at the + /// event-source level (same OpenGL/GHOST-style filter that + /// affects pid-routed clicks); those targets need to be frontmost + /// for drags to land. + /// + /// `modifiers` are held across every event in the gesture (down, + /// every dragged step, up), enabling option-drag (duplicate), + /// shift-drag (constrained axis), etc. + public static func drag( + from start: CGPoint, + to end: CGPoint, + toPid pid: pid_t, + button: Button = .left, + durationMs: Int = 500, + steps: Int = 20, + modifiers: [String] = [] + ) throws { + let clampedSteps = max(1, min(200, steps)) + let clampedDuration = max(0, min(10_000, durationMs)) + // Split the wall-clock budget across the dragged-step gaps. + // `clampedSteps` intermediate points produce `clampedSteps` + // gaps between down → first-drag → … → last-drag → up. + let perStepUs = useconds_t((clampedDuration * 1_000) / clampedSteps) + + let targetIsFrontmost = + NSRunningApplication(processIdentifier: pid)?.isActive ?? false + if targetIsFrontmost { + try dragFrontmostViaHIDTap( + from: start, + to: end, + button: button, + steps: clampedSteps, + perStepUs: perStepUs, + modifiers: modifiers + ) + return + } + + let (downType, upType) = nsEventTypes(for: button) + let draggedType = nsDraggedType(for: button) + let modifierFlags = modifierMask(for: modifiers) + let winNum = Int( + WindowEnumerator.frontmostWindow(forPid: pid) + .map { Int64(CGWindowID($0.id)) } ?? 0) + + let down = try buildCGEvent( + type: downType, + location: cocoaLocation(fromScreenPoint: start), + modifierFlags: modifierFlags, + clickCount: 1, + button: button, + windowNumber: winNum + ) + down.setIntegerValueField(.mouseEventClickState, value: 1) + postBoth(down, toPid: pid) + + for step in 1...clampedSteps { + let progress = Double(step) / Double(clampedSteps) + let point = CGPoint( + x: start.x + (end.x - start.x) * progress, + y: start.y + (end.y - start.y) * progress + ) + let drag = try buildCGEvent( + type: draggedType, + location: cocoaLocation(fromScreenPoint: point), + modifierFlags: modifierFlags, + clickCount: 1, + button: button, + windowNumber: winNum + ) + drag.setIntegerValueField(.mouseEventClickState, value: 1) + usleep(perStepUs) + postBoth(drag, toPid: pid) + } + + let up = try buildCGEvent( + type: upType, + location: cocoaLocation(fromScreenPoint: end), + modifierFlags: modifierFlags, + clickCount: 1, + button: button, + windowNumber: winNum + ) + up.setIntegerValueField(.mouseEventClickState, value: 1) + usleep(perStepUs) + postBoth(up, toPid: pid) + } + + /// Frontmost-target drag: route through `.cghidEventTap` so the + /// gesture originates from the system input stream — matches what + /// AppKit drag sources / Finder selection rect / canvas viewports + /// expect. The real cursor visibly traces the drag path; we + /// accept that for frontmost gestures. + private static func dragFrontmostViaHIDTap( + from start: CGPoint, + to end: CGPoint, + button: Button, + steps: Int, + perStepUs: useconds_t, + modifiers: [String] + ) throws { + let (downType, upType) = cgEventTypes(for: button) + let draggedType = cgDraggedType(for: button) + let mouseButton: CGMouseButton = { + switch button { + case .left: return .left + case .right: return .right + case .middle: return .center + } + }() + let modifierFlags = cgEventFlags(for: modifiers) + let src = CGEventSource(stateID: .hidSystemState) + + guard + let move = CGEvent( + mouseEventSource: src, + mouseType: .mouseMoved, + mouseCursorPosition: start, + mouseButton: mouseButton + ) + else { throw MouseInputError.eventCreationFailed("drag hid-tap move") } + move.flags = modifierFlags + move.post(tap: .cghidEventTap) + usleep(30_000) + + guard + let down = CGEvent( + mouseEventSource: src, + mouseType: downType, + mouseCursorPosition: start, + mouseButton: mouseButton + ) + else { throw MouseInputError.eventCreationFailed("drag hid-tap down") } + down.flags = modifierFlags + down.setIntegerValueField(.mouseEventClickState, value: 1) + down.post(tap: .cghidEventTap) + + for step in 1...steps { + let progress = Double(step) / Double(steps) + let point = CGPoint( + x: start.x + (end.x - start.x) * progress, + y: start.y + (end.y - start.y) * progress + ) + guard + let drag = CGEvent( + mouseEventSource: src, + mouseType: draggedType, + mouseCursorPosition: point, + mouseButton: mouseButton + ) + else { throw MouseInputError.eventCreationFailed("drag hid-tap step") } + drag.flags = modifierFlags + drag.setIntegerValueField(.mouseEventClickState, value: 1) + usleep(perStepUs) + drag.post(tap: .cghidEventTap) + } + + guard + let up = CGEvent( + mouseEventSource: src, + mouseType: upType, + mouseCursorPosition: end, + mouseButton: mouseButton + ) + else { throw MouseInputError.eventCreationFailed("drag hid-tap up") } + up.flags = modifierFlags + up.setIntegerValueField(.mouseEventClickState, value: 1) + usleep(perStepUs) + up.post(tap: .cghidEventTap) + } + + private static func nsDraggedType(for button: Button) -> NSEvent.EventType { + switch button { + case .left: return .leftMouseDragged + case .right: return .rightMouseDragged + case .middle: return .otherMouseDragged + } + } + + private static func cgDraggedType(for button: Button) -> CGEventType { + switch button { + case .left: return .leftMouseDragged + case .right: return .rightMouseDragged + case .middle: return .otherMouseDragged + } + } + // MARK: - Private helpers private static func buildCGEvent( diff --git a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift index 81b3270bcb..2b6f1e9a28 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift @@ -34,6 +34,7 @@ public struct ToolRegistry: Sendable { public static let actionToolNames: Set = [ "click", "right_click", + "drag", "scroll", "type_text", "type_text_chars", @@ -230,6 +231,7 @@ public struct ToolRegistry: Sendable { ClickTool.handler, DoubleClickTool.handler, RightClickTool.handler, + DragTool.handler, SetValueTool.handler, SetAgentCursorEnabledTool.handler, SetAgentCursorMotionTool.handler, diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/DragTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/DragTool.swift new file mode 100644 index 0000000000..6b709f8581 --- /dev/null +++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/DragTool.swift @@ -0,0 +1,327 @@ +import CoreGraphics +import CuaDriverCore +import Foundation +import MCP + +/// Pixel-addressed drag primitive. Two endpoints, both in window-local +/// screenshot pixels (the same space `get_window_state` returns), plus +/// the target pid. +/// +/// macOS AX has no semantic "drag" action — every drag-and-drop, marquee +/// selection, slider scrub, and resize handle is event-synthesis. So +/// drag is pixel-only by design: there is no `element_index` mode the +/// way `click` / `double_click` have. Address one element with +/// `get_window_state`, read its `bounds`, and pass the pixel coordinates +/// you want to drag from / to. +/// +/// The handler delegates to `MouseInput.drag`, which posts via +/// `.cghidEventTap` when the target is frontmost (real-cursor gesture, +/// reaches AppKit drag sources / canvas viewports) and via the +/// auth-signed pid-routed path when backgrounded (cursor-neutral). +public enum DragTool { + public static let handler = ToolHandler( + tool: Tool( + name: "drag", + description: """ + Press-drag-release gesture from (`from_x`, `from_y`) to + (`to_x`, `to_y`) in window-local screenshot pixels — + the same space the PNG `get_window_state` returns. + Top-left origin of the target's window. + + Use this for: marquee/lasso selection, drag-and-drop + between source and destination, resizing via a handle, + scrubbing a slider, repositioning a panel. macOS AX + has no semantic drag action, so drag is pixel-only — + there is no element-indexed mode. Address an element + with `get_window_state`, read its `bounds`, and pass + the pixel coordinates you want. + + `duration_ms` (default 500) is the wall-clock budget + for the path between mouse-down and mouse-up; `steps` + (default 20) is the number of intermediate `mouseDragged` + events linearly interpolated along the path. Increase + both for slower, more "human" drags; decrease for snap + gestures. `modifier` keys (cmd/shift/option/ctrl) are + held across the entire gesture — option-drag duplicates, + shift-drag constrains the axis on most surfaces. + + Frontmost target: posts via `.cghidEventTap`. The real + cursor visibly traces the drag path — unavoidable for + AppKit-style drag sources, which accept only HID-origin + events. + + Backgrounded target: posts via the auth-signed pid-routed + path. Cursor-neutral, but some surfaces (OpenGL canvases, + Blender, Unity) filter pid-routed dragged events at the + event-source level — those targets must be frontmost. + + When `from_zoom` is true, both endpoints are pixel + coordinates in the last `zoom` image for this pid; the + driver maps them back to window coordinates before + dispatching. + """, + inputSchema: [ + "type": "object", + "required": ["pid", "from_x", "from_y", "to_x", "to_y"], + "properties": [ + "pid": [ + "type": "integer", + "description": "Target process ID.", + ], + "window_id": [ + "type": "integer", + "description": + "CGWindowID for the window the pixel coordinates were measured against. Optional — when omitted the driver picks the frontmost window of `pid`. Pass when the target has multiple windows and you measured against a specific one.", + ], + "from_x": [ + "type": "number", + "description": + "Drag-start X in window-local screenshot pixels. Top-left origin.", + ], + "from_y": [ + "type": "number", + "description": + "Drag-start Y in window-local screenshot pixels. Top-left origin.", + ], + "to_x": [ + "type": "number", + "description": + "Drag-end X in window-local screenshot pixels.", + ], + "to_y": [ + "type": "number", + "description": + "Drag-end Y in window-local screenshot pixels.", + ], + "duration_ms": [ + "type": "integer", + "minimum": 0, + "maximum": 10_000, + "description": + "Wall-clock duration of the drag path between mouseDown and mouseUp. Default: 500.", + ], + "steps": [ + "type": "integer", + "minimum": 1, + "maximum": 200, + "description": + "Number of intermediate mouseDragged events linearly interpolated along the path. Default: 20.", + ], + "modifier": [ + "type": "array", + "items": ["type": "string"], + "description": + "Modifier keys held across the entire gesture: cmd/shift/option/ctrl.", + ], + "button": [ + "type": "string", + "enum": ["left", "right", "middle"], + "description": + "Mouse button used for the drag. Default: left.", + ], + "from_zoom": [ + "type": "boolean", + "description": + "When true, from_x/from_y/to_x/to_y are pixel coordinates in the last `zoom` image for this pid. The driver maps them back to window coordinates before dispatching.", + ], + ], + "additionalProperties": false, + ], + annotations: .init( + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: true + ) + ), + invoke: { arguments in + guard let rawPid = arguments?["pid"]?.intValue else { + return errorResult("Missing required integer field pid.") + } + guard let pid = Int32(exactly: rawPid) else { + return errorResult( + "pid \(rawPid) is outside the supported Int32 range.") + } + guard + let fromX = coerceDouble(arguments?["from_x"]), + let fromY = coerceDouble(arguments?["from_y"]), + let toX = coerceDouble(arguments?["to_x"]), + let toY = coerceDouble(arguments?["to_y"]) + else { + return errorResult( + "from_x, from_y, to_x, and to_y are all required (window-local pixels).") + } + + let durationMs = arguments?["duration_ms"]?.intValue ?? 500 + let steps = arguments?["steps"]?.intValue ?? 20 + let modifiers = arguments?["modifier"]?.arrayValue?.compactMap { + $0.stringValue + } ?? [] + let buttonString = arguments?["button"]?.stringValue ?? "left" + let fromZoom = arguments?["from_zoom"]?.boolValue ?? false + + let button: MouseInput.Button + switch buttonString.lowercased() { + case "left": button = .left + case "right": button = .right + case "middle": button = .middle + default: + return errorResult( + "Unknown button \"\(buttonString)\" — expected left, right, or middle.") + } + + let rawWindowId = arguments?["window_id"]?.intValue + let windowId: UInt32? + if let rawWindowId { + guard let checked = UInt32(exactly: rawWindowId) else { + return errorResult( + "window_id \(rawWindowId) is outside the supported UInt32 range.") + } + windowId = checked + } else { + windowId = nil + } + + return await performPixelDrag( + pid: pid, + windowId: windowId, + fromX: fromX, fromY: fromY, + toX: toX, toY: toY, + durationMs: durationMs, + steps: steps, + modifiers: modifiers, + button: button, + fromZoom: fromZoom + ) + } + ) + + private static func performPixelDrag( + pid: Int32, + windowId: UInt32?, + fromX: Double, fromY: Double, + toX: Double, toY: Double, + durationMs: Int, + steps: Int, + modifiers: [String], + button: MouseInput.Button, + fromZoom: Bool + ) async -> CallTool.Result { + var actualFromX = fromX + var actualFromY = fromY + var actualToX = toX + var actualToY = toY + + if fromZoom { + guard let zoom = await ImageResizeRegistry.shared.zoom(forPid: pid) else { + return errorResult( + "from_zoom=true but no zoom context for pid \(pid). Call `zoom` first.") + } + actualFromX = Double(zoom.originX) + fromX + actualFromY = Double(zoom.originY) + fromY + actualToX = Double(zoom.originX) + toX + actualToY = Double(zoom.originY) + toY + } else if let ratio = await ImageResizeRegistry.shared.ratio(forPid: pid) { + actualFromX = fromX * ratio + actualFromY = fromY * ratio + actualToX = toX * ratio + actualToY = toY * ratio + } + + let startScreen: CGPoint + let endScreen: CGPoint + do { + if let windowId { + startScreen = try WindowCoordinateSpace.screenPoint( + fromImagePixel: CGPoint(x: actualFromX, y: actualFromY), + forPid: pid, + windowId: windowId) + endScreen = try WindowCoordinateSpace.screenPoint( + fromImagePixel: CGPoint(x: actualToX, y: actualToY), + forPid: pid, + windowId: windowId) + } else { + startScreen = try WindowCoordinateSpace.screenPoint( + fromImagePixel: CGPoint(x: actualFromX, y: actualFromY), + forPid: pid) + endScreen = try WindowCoordinateSpace.screenPoint( + fromImagePixel: CGPoint(x: actualToX, y: actualToY), + forPid: pid) + } + } catch let error as WindowCoordinateSpaceError { + return errorResult(error.description) + } catch { + return errorResult("Unexpected error resolving window: \(error)") + } + + // Animate the agent cursor along the drag path so the visual + // overlay (when enabled) matches the synthesized gesture. The + // pin-above call keeps the overlay z-stacked over the target + // app for the whole gesture rather than only at the endpoints. + await MainActor.run { + AgentCursor.shared.pinAbove(pid: pid) + } + await AgentCursor.shared.animateAndWait(to: startScreen) + + do { + // Wrap the dispatch in FocusGuard so an app that + // self-activates mid-gesture (Electron / Calculator-style + // reflex `NSApp.activate`) gets demoted back to whatever + // was frontmost before the drag. Drags hold mouseDown for + // `duration_ms` (default 500) — far longer than a click — + // so the self-activation race is materially more likely + // here than for click. Pixel drags have no AX element, so + // layers 1+2 (enablement / synthetic focus) no-op; only + // layer 3 (SystemFocusStealPreventer) arms. + try await AppStateRegistry.focusGuard.withFocusSuppressed( + pid: pid, + element: nil + ) { + try MouseInput.drag( + from: startScreen, + to: endScreen, + toPid: pid, + button: button, + durationMs: durationMs, + steps: steps, + modifiers: modifiers + ) + } + await MainActor.run { + AgentCursor.shared.pinAbove(pid: pid) + } + await AgentCursor.shared.animateAndWait(to: endScreen) + await AgentCursor.shared.finishClick(pid: pid) + + let modSuffix = modifiers.isEmpty ? "" : " with \(modifiers.joined(separator: "+"))" + let buttonSuffix = button == .left ? "" : " (\(button.rawValue) button)" + let summary = + "Posted drag\(buttonSuffix)\(modSuffix) to pid \(pid) " + + "from window-pixel (\(Int(fromX)), \(Int(fromY))) " + + "→ (\(Int(toX)), \(Int(toY))), " + + "screen (\(Int(startScreen.x)), \(Int(startScreen.y))) " + + "→ (\(Int(endScreen.x)), \(Int(endScreen.y))) " + + "in \(durationMs)ms / \(steps) steps." + return CallTool.Result( + content: [.text(text: "✅ \(summary)", annotations: nil, _meta: nil)] + ) + } catch let error as MouseInputError { + return errorResult(error.description) + } catch { + return errorResult("Unexpected error: \(error)") + } + } + + private static func coerceDouble(_ value: Value?) -> Double? { + if let d = value?.doubleValue { return d } + if let i = value?.intValue { return Double(i) } + return nil + } + + private static func errorResult(_ message: String) -> CallTool.Result { + CallTool.Result( + content: [.text(text: message, annotations: nil, _meta: nil)], + isError: true + ) + } +} diff --git a/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py b/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py new file mode 100644 index 0000000000..c58b152e11 --- /dev/null +++ b/libs/cua-driver/Tests/integration/test_drag_slider_delivery.py @@ -0,0 +1,309 @@ +"""Integration test: pixel drag on backgrounded Safari range slider. + +Drives `` +in `fixtures/form_all_inputs.html`, opened in Safari with `open -g` so +Safari is NOT frontmost, with FocusMonitorApp launched on top. + +Asserts: + - Drag delivery: the slider's value changed (default 50 → ≥80 after a + rightward drag past the slider's right edge — WebKit clamps to max=100). + - No focus steal: FocusMonitorApp stays frontmost throughout. + +The drag tool is pixel-only (macOS AX has no semantic drag action), so +this test exercises the path that would otherwise be silently broken by +a wrong coord-space conversion or a focus-stealing recipe. + +Run: + scripts/test.sh test_drag_slider_delivery +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from driver_client import ( # noqa: E402 + DriverClient, + default_binary_path, + frontmost_bundle_id, + resolve_window_id, +) + +_THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(os.path.dirname(_THIS_DIR)) + +_HTML_FORM = os.path.join(_THIS_DIR, "fixtures", "form_all_inputs.html") +_FORM_URL = f"file://{_HTML_FORM}" + +_FOCUS_APP_DIR = os.path.join(_REPO_ROOT, "Tests", "FocusMonitorApp") +_FOCUS_APP_BUNDLE = os.path.join(_FOCUS_APP_DIR, "FocusMonitorApp.app") +_FOCUS_APP_EXE = os.path.join( + _FOCUS_APP_BUNDLE, "Contents", "MacOS", "FocusMonitorApp" +) +_LOSS_FILE = "/tmp/focus_monitor_losses.txt" + +SAFARI_BUNDLE = "com.apple.Safari" +FOCUS_MONITOR_BUNDLE = "com.trycua.FocusMonitorApp" + + +def _build_focus_app() -> None: + if not os.path.exists(_FOCUS_APP_EXE): + subprocess.run( + [os.path.join(_FOCUS_APP_DIR, "build.sh")], check=True + ) + + +def _launch_focus_app() -> tuple[subprocess.Popen, int]: + proc = subprocess.Popen( + [_FOCUS_APP_EXE], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + pid: int | None = None + for _ in range(40): + line = proc.stdout.readline().strip() + if line.startswith("FOCUS_PID="): + pid = int(line.split("=", 1)[1]) + break + time.sleep(0.1) + if pid is None: + proc.terminate() + raise RuntimeError("FocusMonitorApp did not print FOCUS_PID in time") + subprocess.run( + ["osascript", "-e", 'tell application "FocusMonitorApp" to activate'], + check=False, timeout=3, + ) + return proc, pid + + +def _read_focus_losses() -> int: + try: + with open(_LOSS_FILE) as f: + return int(f.read().strip()) + except (FileNotFoundError, ValueError): + return -1 + + +def _safari_js(expr: str, timeout: float = 5.0) -> str: + """Evaluate a JS expression in Safari's front document via osascript.""" + script = ( + f'tell application "Safari" to do JavaScript "{expr}" in front document' + ) + r = subprocess.run( + ["osascript", "-e", script], + capture_output=True, text=True, timeout=timeout, + ) + return r.stdout.strip() + + +def _wait_for_form(timeout: float = 25.0) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if _safari_js("typeof getFieldValues") == "function": + return True + time.sleep(0.5) + return False + + +def _slider_value() -> int: + """Read the current slider value via the fixture's getFieldValues().""" + raw = _safari_js("getFieldValues()") + try: + v = json.loads(raw).get("range", "") + except (json.JSONDecodeError, AttributeError): + return -1 + try: + return int(v) + except (TypeError, ValueError): + return -1 + + +def _slider_screen_rect() -> dict: + """Return the slider's screen-absolute bounding box in screen points. + + Combines viewport-local `getBoundingClientRect()` with + `window.screenLeft / screenTop` (origin of the viewport in screen + coordinates). Both are CSS pixels, which equal macOS screen points + at the default zoom — so the result is directly usable as a target + for `WindowCoordinateSpace.screenPoint(...)` after subtracting the + window's `bounds.x/y` to get window-local points, then multiplying + by the screenshot scale factor to get image pixels. + """ + expr = ( + "JSON.stringify((function(){" + "var r=document.getElementById('f-range').getBoundingClientRect();" + "return {left:r.left,top:r.top,width:r.width,height:r.height," + "screenLeft:window.screenLeft,screenTop:window.screenTop};" + "})())" + ) + raw = _safari_js(expr) + # AppleScript wraps the JSON in extra quotes/escapes — strip them. + raw = raw.replace('\\"', '"').strip('"') + return json.loads(raw) + + +class SafariRangeSliderDragDelivery(unittest.TestCase): + """Drag the range slider in backgrounded Safari — value changes, no focus steal.""" + + _safari_pid: int + _focus_proc: subprocess.Popen + _focus_pid: int + + @classmethod + def setUpClass(cls) -> None: + _build_focus_app() + try: + os.remove(_LOSS_FILE) + except FileNotFoundError: + pass + + subprocess.run(["pkill", "-x", "Safari"], check=False) + time.sleep(1.0) + # `open -g` keeps Safari in the background. + subprocess.run(["open", "-g", "-a", "Safari", _FORM_URL], check=True) + if not _wait_for_form(): + raise RuntimeError( + "Safari did not load the form fixture in time. " + "Verify Safari → Develop → Allow JavaScript from Apple Events is on." + ) + + cls.binary = default_binary_path() + with DriverClient(cls.binary) as c: + apps = c.call_tool("list_apps")["structuredContent"]["apps"] + safari = [a for a in apps if a.get("bundle_id") == SAFARI_BUNDLE] + if not safari: + raise RuntimeError("Safari is not running after open -g") + cls._safari_pid = safari[0]["pid"] + + cls._focus_proc, cls._focus_pid = _launch_focus_app() + time.sleep(0.5) + + with DriverClient(cls.binary) as c: + active = frontmost_bundle_id(c) + assert active == FOCUS_MONITOR_BUNDLE, ( + f"Expected FocusMonitorApp frontmost at start, got {active}" + ) + losses = _read_focus_losses() + assert losses == 0, f"Expected 0 focus losses at start, got {losses}" + + @classmethod + def tearDownClass(cls) -> None: + cls._focus_proc.terminate() + try: + cls._focus_proc.wait(timeout=3) + except subprocess.TimeoutExpired: + cls._focus_proc.kill() + subprocess.run(["pkill", "-x", "Safari"], check=False) + try: + os.remove(_LOSS_FILE) + except FileNotFoundError: + pass + + def setUp(self) -> None: + # Reset the slider to its default mid-track value so the test + # is independent of any prior run that left it elsewhere. + _safari_js( + "(function(){var s=document.getElementById('f-range');" + "s.value=50;s.dispatchEvent(new Event('input'));" + "s.dispatchEvent(new Event('change'));})()" + ) + time.sleep(0.2) + self._losses_before = _read_focus_losses() + + def test_slider_drag_to_right_edge(self) -> None: + """Drag the thumb from mid-track to past the right edge — value goes to 100.""" + before = _slider_value() + self.assertEqual( + before, 50, + f"slider didn't reset to 50 (got {before}) — fixture or JS bridge off" + ) + + with DriverClient(self.binary) as c: + window_id = resolve_window_id(c, self._safari_pid) + snap = c.call_tool( + "get_window_state", + {"pid": self._safari_pid, "window_id": window_id}, + ) + sc = snap.get("structuredContent", snap) + scale = sc.get("screenshot_scale_factor", 2) + + # Window origin in screen points (top-left). + windows = c.call_tool( + "list_windows", {"pid": self._safari_pid} + )["structuredContent"]["windows"] + win = next( + w for w in windows + if w["window_id"] == window_id + ) + win_x = win["bounds"]["x"] + win_y = win["bounds"]["y"] + + rect = _slider_screen_rect() + # Slider center in screen points. + cx = rect["screenLeft"] + rect["left"] + rect["width"] / 2 + cy = rect["screenTop"] + rect["top"] + rect["height"] / 2 + # End point: 50 px past the right edge of the slider, same y. + ex = rect["screenLeft"] + rect["left"] + rect["width"] + 50 + ey = cy + # Convert screen points → window-local image pixels. + from_x = (cx - win_x) * scale + from_y = (cy - win_y) * scale + to_x = (ex - win_x) * scale + to_y = (ey - win_y) * scale + + print( + f"\n slider rect (screen pts): " + f"x={cx:.0f}..{ex:.0f}, y={cy:.0f}; " + f"window-local pixels: ({from_x:.0f},{from_y:.0f}) → ({to_x:.0f},{to_y:.0f})" + ) + + result = c.call_tool( + "drag", + { + "pid": self._safari_pid, + "window_id": window_id, + "from_x": from_x, + "from_y": from_y, + "to_x": to_x, + "to_y": to_y, + "duration_ms": 500, + "steps": 24, + }, + ) + print( + f" drag result: {result.get('content', [{}])[0].get('text', '')[:200]}" + ) + + time.sleep(0.4) + after = _slider_value() + print(f" slider value: {before} → {after}") + + # Drag landed AND clamped at the slider's max (100). + self.assertGreaterEqual( + after, 80, + f"Drag did not move the slider far enough — value is {after} (was {before})" + ) + + # Focus invariant: FocusMonitorApp stayed frontmost throughout. + with DriverClient(self.binary) as c: + active = frontmost_bundle_id(c) + losses = _read_focus_losses() + print(f" losses: {self._losses_before}→{losses}, frontmost: {active}") + self.assertEqual( + active, FOCUS_MONITOR_BUNDLE, + f"Focus stolen — frontmost is {active} (expected FocusMonitorApp)" + ) + self.assertEqual( + losses, self._losses_before, + f"Focus losses increased: {self._losses_before} → {losses}" + ) + + +if __name__ == "__main__": + unittest.main()