Add cmux browser repl: a Playwright-shaped browser REPL for agents

CLI/CMUXCLI+TaskHelp.swift

@@ -361,6 +361,7 @@ extension CMUXCLI {
         browser url|get-url
         browser snapshot [--interactive|-i] [--cursor] [--compact] [--max-depth <n>] [--selector <css>]
         browser eval <script>
+        browser repl [--session <name>] [--workspace <id|ref>] [--eval <code>|-] [<code>]   (see: browser repl guide)
         browser wait [--selector <css>] [--text <text>] [--url-contains <text>] [--load-state <interactive|complete>] [--function <js>] [--timeout-ms <ms>]
         browser click|dblclick|hover|focus|check|uncheck|scroll-into-view <selector> [--snapshot-after]
         browser type <selector> <text> [--snapshot-after]

CLI/cmux.swift

@@ -16656,7 +16656,7 @@ struct CMUXCLI {
         var surfaceRaw = surfaceOpt
         var args = argsWithoutSurfaceFlag
 
-        let verbsWithoutSurface: Set<String> = ["open", "open-split", "new", "identify", "import", "profile", "profiles", "react-grab", "reactgrab", "devtools", "dev-tools", "focus-mode", "design-mode", "zoom", "history"]
+        let verbsWithoutSurface: Set<String> = ["open", "open-split", "new", "identify", "import", "profile", "profiles", "react-grab", "reactgrab", "devtools", "dev-tools", "focus-mode", "design-mode", "zoom", "history", "repl"]
         if surfaceRaw == nil, let first = args.first {
             if !first.hasPrefix("-") && !verbsWithoutSurface.contains(first.lowercased()) {
                 surfaceRaw = first
@@ -18254,6 +18254,11 @@ struct CMUXCLI {
             return
         }
 
+        if subcommand == "repl" {
+            try runBrowserRepl(subArgs, client: client, jsonOutput: effectiveJSONOutput)
+            return
+        }
+
         if subcommand == "viewport" {
             let params = try Self.browserViewportSetParams(subArgs, surfaceID: requireSurface())
             let payload = try client.sendV2(method: "browser.viewport.set", params: params)
@@ -21190,6 +21195,7 @@ struct CMUXCLI {
               addinitscript|addscript [--script <js> | <js>]
               addstyle [--css <css> | <css>]
               \(Self.browserViewportHelp)
+              \(Self.browserReplHelp)
               geolocation|geo <latitude> <longitude>
               offline <true|false>
               trace <start|stop> [path]
@@ -42655,7 +42661,7 @@ export default {
         print()
     }
 
-    private func resolvedVersionInfo() -> [String: String] {
+    func resolvedVersionInfo() -> [String: String] {
         var info: [String: String] = [:]
         if let main = versionInfo(from: Bundle.main.infoDictionary) {
             info.merge(main, uniquingKeysWith: { current, _ in current })

Packages/macOS/CmuxBrowser/Sources/CmuxBrowser/Control/BrowserKeyboardEvent.swift

@@ -35,7 +35,7 @@ public struct BrowserKeyboardEvent: Equatable, Sendable {
         }
     }
 
-    private init(key: String, code: String, legacyKeyCode: Int, location: Int = 0) {
+    init(key: String, code: String, legacyKeyCode: Int, location: Int = 0) {
         self.key = key
         self.code = code
         self.legacyKeyCode = legacyKeyCode

Packages/macOS/CmuxBrowser/Sources/CmuxBrowser/Download/BrowserSuggestedFilenameOverriding.swift

@@ -10,3 +10,11 @@ public protocol BrowserSuggestedFilenameOverriding: AnyObject {
     /// Blank names are ignored.
     func setSuggestedFilenameOverride(_ suggestedFilename: String?, for download: WKDownload)
 }
+
+/// A download delegate that wants scripted `data:` downloads (a page's
+/// `<a download>` links) as WebKit downloads it receives, rather than saved
+/// directly by the web view. The browser REPL uses this so a driven tab
+/// reports those downloads to its session. Read on the main thread.
+public protocol BrowserScriptedDownloadRouting: AnyObject {
+    var routesScriptedDownloadsThroughWebKit: Bool { get }
+}

Packages/macOS/CmuxBrowser/Sources/CmuxBrowser/Input/BrowserWebKitKeyDownDispatch.swift

@@ -36,10 +36,26 @@ public final class BrowserNativeInputDeliveryOwner {
 
     func withDispatch<T>(_ body: () -> T) -> T {
         dispatchDepth += 1
-        defer { dispatchDepth = max(0, dispatchDepth - 1) }
+        Self.activeDispatchCount += 1
+        defer {
+            dispatchDepth = max(0, dispatchDepth - 1)
+            Self.activeDispatchCount = max(0, Self.activeDispatchCount - 1)
+        }
         return body()
     }
 
+    /// Native key deliveries in progress in any web view.
+    private static var activeDispatchCount = 0
+
+    /// Whether any web view is delivering an automated key right now. WebKit's
+    /// resend of an unhandled key runs on a later turn, outside every delivery.
+    public static var isAnyDispatchActive: Bool { activeDispatchCount > 0 }
+
+    /// Key codes of modifiers currently held by automation.
+    public var heldModifierKeyCodes: [UInt16] {
+        Array(heldModifierKeys.keys)
+    }
+
     public func setModifier(_ modifier: BrowserKeyboardNativeModifiers, for keyCode: UInt16) {
         heldModifierKeys[keyCode] = modifier
     }
@@ -118,13 +134,27 @@ extension WKWebView {
             forBrowserNativeKey: nativeKey,
             additionalModifierFlags: activeModifiers
         )
-        return replayBrowserKeyboardSpecification(
+        let result = replayBrowserKeyboardSpecification(
             specification,
             action: action,
-            characters: nativeKey.characters
+            characters: nativeKey.characters,
+            marksBrowserAutomation: true
         )
+        // WebKit leaves Command+A/C/X/V/Z to the app's Edit menu, which the
+        // resend of an automated key no longer reaches; run the command on
+        // this web view, as the REPL does, never on the key window.
+        if result == .delivered, action != .keyUp,
+           let command = BrowserReplKeyStroke.editingCommand(code: event.code, key: event.key, flags: specification.modifierFlags),
+           Self.menuEditingCommands.contains(command) {
+            let selector = NSSelectorFromString(command)
+            if responds(to: selector) { _ = perform(selector, with: nil) }
+        }
+        return result
     }
 
+    /// Edit menu commands `cmux browser press` runs on the web view itself.
+    static let menuEditingCommands: Set<String> = ["selectAll:", "copy:", "cut:", "paste:", "undo:", "redo:"]
+
     /// Delivers an already-resolved AppKit key specification. The mobile
     /// browser stream and socket automation both use this seam so key-down
     /// re-entry handling and event construction cannot diverge.
@@ -133,25 +163,34 @@ extension WKWebView {
     ///   - specification: AppKit key-code and modifier metadata.
     ///   - action: Whether to send a press, key-down, or key-up.
     ///   - characters: Optional Unicode text to attach to the event.
+    ///   - marksBrowserAutomation: Marks the events as automation's
+    ///     (``NSEvent/isBrowserAutomationKeyEvent``) so the app drops WebKit's
+    ///     resend of one no page handled. The REPL and `cmux browser press`
+    ///     mark their keys; the mobile browser stream, a person's keys from a
+    ///     phone, does not, so its unhandled Command shortcuts still reach the
+    ///     Mac's menus.
     /// - Returns: The native delivery outcome.
     @discardableResult
     public func replayBrowserKeyboardSpecification(
         _ specification: SyntheticKeySpecification,
         action: BrowserKeyboardAction,
-        characters: String? = nil
+        characters: String? = nil,
+        marksBrowserAutomation: Bool = false
     ) -> BrowserKeyboardReplayResult {
         let timestamp = ProcessInfo.processInfo.systemUptime
         let down = SyntheticKeyEventFactory.keyEvent(
             specification: specification,
             keyDown: true,
             timestamp: timestamp,
-            characters: characters
+            characters: characters,
+            marksBrowserAutomation: marksBrowserAutomation
         )
         let up = SyntheticKeyEventFactory.keyEvent(
             specification: specification,
             keyDown: false,
             timestamp: timestamp,
-            characters: characters
+            characters: characters,
+            marksBrowserAutomation: marksBrowserAutomation
         )
 
         switch action {
@@ -215,6 +254,14 @@ extension WKWebView {
         return owner
     }
 
+    func replayBrowserNativeModifier(
+        _ key: BrowserKeyboardNativeKey,
+        keyDown: Bool
+    ) -> BrowserKeyboardReplayResult {
+        guard let modifierKey = key.modifierKey else { return .unsupported }
+        return replayBrowserModifier(key, modifierKey: modifierKey, action: keyDown ? .keyDown : .keyUp)
+    }
+
     private func replayBrowserModifier(
         _ key: BrowserKeyboardNativeKey,
         modifierKey: BrowserKeyboardNativeModifiers,
@@ -292,3 +339,28 @@ extension WKWebView {
         }
     }
 }
+
+/// Keys browser automation (the REPL, `cmux browser press`) delivers to a
+/// web view; the mobile browser stream's keys, a person's, are not marked. When no page handles such a key,
+/// WebKit sends it back through `NSApp.sendEvent` (WebViewImpl's
+/// doneWithKeyEvent), which hands it to the key window: the user's window,
+/// whose first responder (a terminal) would receive the text and whose menus
+/// would run Command shortcuts. The page has already received the key, so the
+/// app drops that resend (``isResentBrowserAutomationKeyEvent``).
+extension NSEvent {
+    /// `CGEventField.eventSourceUserData` of an automated browser key ("cmuxkeys").
+    static let browserAutomationKeyMark: Int64 = 0x636D_7578_6B65_7973
+
+    /// Whether browser automation created this key event for a web view.
+    public var isBrowserAutomationKeyEvent: Bool {
+        guard type == .keyDown || type == .keyUp || type == .flagsChanged, let cgEvent else { return false }
+        return cgEvent.getIntegerValueField(.eventSourceUserData) == Self.browserAutomationKeyMark
+    }
+
+    /// Whether this is an automated browser key reaching the app outside the
+    /// web view's own delivery: WebKit's resend of a key no page handled.
+    @MainActor
+    public var isResentBrowserAutomationKeyEvent: Bool {
+        isBrowserAutomationKeyEvent && !BrowserNativeInputDeliveryOwner.isAnyDispatchActive
+    }
+}

Packages/macOS/CmuxBrowser/Sources/CmuxBrowser/Input/SyntheticKeyEventFactory.swift

@@ -128,17 +128,25 @@ public enum SyntheticKeyEventFactory {
 
     /// Builds an `NSEvent` backed by a real `CGEvent` so WebKit text input can
     /// safely interpret it. Callers choose their own direct delivery target.
+    ///
+    /// - Parameter marksBrowserAutomation: Marks the event as a key browser
+    ///   automation delivers to a web view (``NSEvent/isBrowserAutomationKeyEvent``),
+    ///   so the app can drop WebKit's resend of it to the key window.
     public static func keyEvent(
         specification: SyntheticKeySpecification,
         keyDown: Bool,
         timestamp: TimeInterval,
-        characters: String? = nil
+        characters: String? = nil,
+        marksBrowserAutomation: Bool = false
     ) -> NSEvent? {
         guard let cgEvent = CGEvent(
             keyboardEventSource: nil,
             virtualKey: specification.keyCode,
             keyDown: keyDown
         ) else { return nil }
+        if marksBrowserAutomation {
+            cgEvent.setIntegerValueField(.eventSourceUserData, value: NSEvent.browserAutomationKeyMark)
+        }
         cgEvent.flags = cgFlags(specification.modifierFlags)
         cgEvent.timestamp = CGEventTimestamp(timestamp * 1_000_000_000)
         if let characters {

Packages/macOS/CmuxBrowser/Sources/CmuxBrowser/Repl/BrowserReplAgentUserScript.swift

@@ -0,0 +1,75 @@
+public import WebKit
+
+/// The REPL page agent as a document-start user script in every frame's
+/// agent world, so documents loaded while a session drives a tab have the
+/// agent before their own scripts run. Frames that loaded earlier get it on
+/// their first evaluation.
+///
+/// The script runs only while the agent world has the presence message
+/// handler (`presenceHandlerName`), which exists while a session is attached.
+/// A controller holds at most one copy, whichever attachment installed it,
+/// and the copy leaves the controller when the tab's last session detaches
+/// or the tab moves to another web view. Other user scripts stay.
+@MainActor
+public final class BrowserReplAgentUserScript {
+    /// Every agent script this process created, to tell them apart from the
+    /// controller's other user scripts.
+    private static let agentScripts = NSHashTable<WKUserScript>.weakObjects()
+
+    private weak var controller: WKUserContentController?
+
+    public init() {}
+
+    /// Adds the agent to `controller` unless it already has it, and removes
+    /// it from the controller this installer used before.
+    /// - Parameters:
+    ///   - source: The page agent's install source.
+    ///   - presenceHandlerName: The agent-world message handler whose
+    ///     presence lets the script run.
+    ///   - world: The agent's content world.
+    ///   - controller: The tab's user content controller.
+    public func install(
+        source: String,
+        presenceHandlerName: String,
+        world: WKContentWorld,
+        in controller: WKUserContentController
+    ) {
+        if let previous = self.controller, previous !== controller {
+            Self.remove(from: previous)
+        }
+        self.controller = controller
+        guard !controller.userScripts.contains(where: Self.agentScripts.contains) else { return }
+        let guarded = """
+        if (globalThis.webkit && webkit.messageHandlers && webkit.messageHandlers.\(presenceHandlerName)) {
+        \(source)
+        }
+        """
+        let script = WKUserScript(
+            source: guarded,
+            injectionTime: .atDocumentStart,
+            forMainFrameOnly: false,
+            in: world
+        )
+        Self.agentScripts.add(script)
+        controller.addUserScript(script)
+    }
+
+    /// Removes the agent from the controller; called when no session
+    /// remains attached to the tab.
+    public func release() {
+        if let controller { Self.remove(from: controller) }
+        controller = nil
+    }
+
+    /// WebKit removes user scripts only all at once, so this re-adds every
+    /// script that is not the agent, in order. `userScripts` is a live view
+    /// of the controller's list, so it is copied first.
+    private static func remove(from controller: WKUserContentController) {
+        let scripts = controller.userScripts.map { $0 }
+        guard scripts.contains(where: agentScripts.contains) else { return }
+        controller.removeAllUserScripts()
+        for script in scripts where !agentScripts.contains(script) {
+            controller.addUserScript(script)
+        }
+    }
+}