-
Notifications
You must be signed in to change notification settings - Fork 90
feat(macos): add per-process mouse input helper #29325
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
siddseethepalli
merged 1 commit into
siddseethepalli/app-control-skill
from
app-control/pr-8-app-mouse
May 3, 2026
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
176 changes: 176 additions & 0 deletions
176
clients/macos/vellum-assistant/AppControl/AppMouse.swift
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,176 @@ | ||
| import CoreGraphics | ||
| import Foundation | ||
| import VellumAssistantShared | ||
|
|
||
| /// Per-process mouse input helper for the app-control skill. | ||
| /// | ||
| /// All events are posted via `CGEvent.postToPid(_:)` (the modern Swift | ||
| /// spelling of `CGEventPostToPid`) so they target the specific host process | ||
| /// rather than the global event tap, keeping the user's real cursor and | ||
| /// other apps unaffected. | ||
| /// | ||
| /// Coordinates are window-relative and translated to global at post time | ||
| /// using the current `WindowBounds` reported by the daemon. | ||
| enum AppMouse { | ||
|
|
||
| // MARK: - Errors | ||
|
|
||
| enum AppMouseError: LocalizedError { | ||
| case eventCreationFailed | ||
|
|
||
| var errorDescription: String? { | ||
| switch self { | ||
| case .eventCreationFailed: return "Failed to create CGEvent" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // MARK: - Mouse buttons | ||
|
|
||
| enum MouseButton: String { | ||
| case left | ||
| case right | ||
| case middle | ||
| } | ||
|
|
||
| static func cgButton(for button: MouseButton) -> CGMouseButton { | ||
| switch button { | ||
| case .left: return .left | ||
| case .right: return .right | ||
| case .middle: return .center | ||
| } | ||
| } | ||
|
|
||
| private static func downType(for button: MouseButton) -> CGEventType { | ||
| switch button { | ||
| case .left: return .leftMouseDown | ||
| case .right: return .rightMouseDown | ||
| case .middle: return .otherMouseDown | ||
| } | ||
| } | ||
|
|
||
| private static func upType(for button: MouseButton) -> CGEventType { | ||
| switch button { | ||
| case .left: return .leftMouseUp | ||
| case .right: return .rightMouseUp | ||
| case .middle: return .otherMouseUp | ||
| } | ||
| } | ||
|
|
||
| private static func draggedType(for button: MouseButton) -> CGEventType { | ||
| switch button { | ||
| case .left: return .leftMouseDragged | ||
| case .right: return .rightMouseDragged | ||
| case .middle: return .otherMouseDragged | ||
| } | ||
| } | ||
|
|
||
| // MARK: - Pure helpers (unit-tested) | ||
|
|
||
| /// Translates a window-relative point to a global screen coordinate. | ||
| static func windowRelativeToGlobal(_ point: CGPoint, windowBounds: WindowBounds) -> CGPoint { | ||
| return CGPoint(x: windowBounds.x + point.x, y: windowBounds.y + point.y) | ||
| } | ||
|
|
||
| /// Returns `steps` evenly-spaced intermediate points strictly between | ||
| /// `from` and `to` (exclusive of both endpoints). Returns an empty | ||
| /// array when `steps <= 0`. | ||
| static func interpolate(from: CGPoint, to: CGPoint, steps: Int) -> [CGPoint] { | ||
| guard steps > 0 else { return [] } | ||
| var points: [CGPoint] = [] | ||
| points.reserveCapacity(steps) | ||
| let denom = CGFloat(steps + 1) | ||
| for i in 1...steps { | ||
| let t = CGFloat(i) / denom | ||
| points.append(CGPoint( | ||
| x: from.x + (to.x - from.x) * t, | ||
| y: from.y + (to.y - from.y) * t | ||
| )) | ||
| } | ||
| return points | ||
| } | ||
|
|
||
| // MARK: - Click | ||
|
|
||
| /// Posts a synthetic mouse click to the target process. | ||
| /// | ||
| /// `x`/`y` are window-relative; they are translated to global coordinates | ||
| /// using `windowBounds`. When `double` is `true`, two down/up cycles are | ||
| /// posted with `mouseEventClickState` set to 2 on the second cycle to | ||
| /// match how macOS native double-click events look. | ||
| static func click( | ||
| pid: pid_t, | ||
| windowBounds: WindowBounds, | ||
| x: Double, | ||
| y: Double, | ||
| button: MouseButton, | ||
| double: Bool | ||
| ) throws { | ||
| let global = windowRelativeToGlobal(CGPoint(x: x, y: y), windowBounds: windowBounds) | ||
| let cgButton = cgButton(for: button) | ||
| let down = downType(for: button) | ||
| let up = upType(for: button) | ||
|
|
||
| try postClick(pid: pid, position: global, downType: down, upType: up, cgButton: cgButton, clickState: 1) | ||
| if double { | ||
| try postClick(pid: pid, position: global, downType: down, upType: up, cgButton: cgButton, clickState: 2) | ||
| } | ||
| } | ||
|
|
||
| private static func postClick( | ||
| pid: pid_t, | ||
| position: CGPoint, | ||
| downType: CGEventType, | ||
| upType: CGEventType, | ||
| cgButton: CGMouseButton, | ||
| clickState: Int64 | ||
| ) throws { | ||
| try postMouseEvent(pid: pid, type: downType, position: position, cgButton: cgButton, clickState: clickState) | ||
| try postMouseEvent(pid: pid, type: upType, position: position, cgButton: cgButton, clickState: clickState) | ||
| } | ||
|
|
||
| private static func postMouseEvent( | ||
| pid: pid_t, | ||
| type: CGEventType, | ||
| position: CGPoint, | ||
| cgButton: CGMouseButton, | ||
| clickState: Int64? = nil | ||
| ) throws { | ||
| guard let event = CGEvent( | ||
| mouseEventSource: nil, | ||
| mouseType: type, | ||
| mouseCursorPosition: position, | ||
| mouseButton: cgButton | ||
| ) else { | ||
| throw AppMouseError.eventCreationFailed | ||
| } | ||
| if let clickState { | ||
| event.setIntegerValueField(.mouseEventClickState, value: clickState) | ||
| } | ||
| event.postToPid(pid) | ||
| } | ||
|
|
||
| // MARK: - Drag | ||
|
|
||
| /// Posts a synthetic mouse drag to the target process: mouseDown at | ||
| /// `from`, 10 interpolated mouseDragged events, then mouseUp at `to`. | ||
| static func drag( | ||
| pid: pid_t, | ||
| windowBounds: WindowBounds, | ||
| fromX: Double, | ||
| fromY: Double, | ||
| toX: Double, | ||
| toY: Double, | ||
| button: MouseButton | ||
| ) throws { | ||
| let fromGlobal = windowRelativeToGlobal(CGPoint(x: fromX, y: fromY), windowBounds: windowBounds) | ||
| let toGlobal = windowRelativeToGlobal(CGPoint(x: toX, y: toY), windowBounds: windowBounds) | ||
| let cgButton = cgButton(for: button) | ||
|
|
||
| try postMouseEvent(pid: pid, type: downType(for: button), position: fromGlobal, cgButton: cgButton) | ||
| for point in interpolate(from: fromGlobal, to: toGlobal, steps: 10) { | ||
| try postMouseEvent(pid: pid, type: draggedType(for: button), position: point, cgButton: cgButton) | ||
| } | ||
| try postMouseEvent(pid: pid, type: upType(for: button), position: toGlobal, cgButton: cgButton) | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // CGEventPostToPid cannot be unit-tested headlessly; runtime click behavior | ||
| // is verified manually. Tests here cover the pure coordinate-translation | ||
| // and interpolation helpers. | ||
|
|
||
| import CoreGraphics | ||
| import XCTest | ||
| @testable import VellumAssistantLib | ||
| @testable import VellumAssistantShared | ||
|
|
||
| final class AppMouseTests: XCTestCase { | ||
|
|
||
| // MARK: - windowRelativeToGlobal | ||
|
|
||
| func test_windowRelativeToGlobal_addsWindowOriginToPoint() { | ||
| let bounds = WindowBounds(x: 100, y: 200, width: 800, height: 600) | ||
| let global = AppMouse.windowRelativeToGlobal(CGPoint(x: 10, y: 20), windowBounds: bounds) | ||
| XCTAssertEqual(global.x, 110) | ||
| XCTAssertEqual(global.y, 220) | ||
| } | ||
|
|
||
| func test_windowRelativeToGlobal_handlesNegativeWindowOrigin() { | ||
| // Multi-monitor setups can place windows at negative origins relative | ||
| // to the primary screen. | ||
| let bounds = WindowBounds(x: -50, y: -100, width: 800, height: 600) | ||
| let global = AppMouse.windowRelativeToGlobal(CGPoint(x: 10, y: 20), windowBounds: bounds) | ||
| XCTAssertEqual(global.x, -40) | ||
| XCTAssertEqual(global.y, -80) | ||
| } | ||
|
|
||
| // MARK: - interpolate | ||
|
|
||
| func test_interpolate_returnsRequestedNumberOfPointsStrictlyBetweenEndpoints() { | ||
| let from = CGPoint(x: 0, y: 0) | ||
| let to = CGPoint(x: 100, y: 100) | ||
| let points = AppMouse.interpolate(from: from, to: to, steps: 10) | ||
|
|
||
| XCTAssertEqual(points.count, 10) | ||
|
|
||
| for point in points { | ||
| XCTAssertGreaterThan(point.x, from.x) | ||
| XCTAssertLessThan(point.x, to.x) | ||
| XCTAssertGreaterThan(point.y, from.y) | ||
| XCTAssertLessThan(point.y, to.y) | ||
| } | ||
|
|
||
| // Evenly spaced — gap between consecutive points (and between the | ||
| // endpoints and the first/last point) should be constant. | ||
| let expectedStep: CGFloat = 100.0 / CGFloat(10 + 1) | ||
| for i in 0..<points.count { | ||
| let prevX = i == 0 ? from.x : points[i - 1].x | ||
| XCTAssertEqual(points[i].x - prevX, expectedStep, accuracy: 0.0001) | ||
| } | ||
| XCTAssertEqual(to.x - points.last!.x, expectedStep, accuracy: 0.0001) | ||
| } | ||
|
|
||
| func test_interpolate_returnsEmptyForZeroOrNegativeSteps() { | ||
| let from = CGPoint(x: 0, y: 0) | ||
| let to = CGPoint(x: 100, y: 100) | ||
| XCTAssertEqual(AppMouse.interpolate(from: from, to: to, steps: 0).count, 0) | ||
| XCTAssertEqual(AppMouse.interpolate(from: from, to: to, steps: -1).count, 0) | ||
| } | ||
|
|
||
| // MARK: - MouseButton → CGMouseButton mapping | ||
|
|
||
| func test_mouseButton_mapsToCGMouseButton() { | ||
| XCTAssertEqual(AppMouse.cgButton(for: .left), .left) | ||
| XCTAssertEqual(AppMouse.cgButton(for: .right), .right) | ||
| // macOS represents the middle mouse button as `.center`. | ||
| XCTAssertEqual(AppMouse.cgButton(for: .middle), .center) | ||
| } | ||
|
|
||
| func test_mouseButton_rawValuesMatchDaemonContract() { | ||
|
siddseethepalli marked this conversation as resolved.
|
||
| XCTAssertEqual(AppMouse.MouseButton.left.rawValue, "left") | ||
| XCTAssertEqual(AppMouse.MouseButton.right.rawValue, "right") | ||
| XCTAssertEqual(AppMouse.MouseButton.middle.rawValue, "middle") | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.