Add cmux record for capturing a cmux window to mp4 or gif

CLI/CMUXCLI+CommandSuggestions.swift

@@ -166,6 +166,7 @@ extension CMUXCLI {
         "previous-window",
         "read-screen",
         "read-selection",
+        "record",
         "refresh-surfaces",
         "reload-config",
         "remote-daemon-status",

CLI/CMUXCLI+Record.swift

@@ -0,0 +1,231 @@
+import Foundation
+
+extension CMUXCLI {
+    static var recordHelp: String {
+        String(localized: "cli.help.record", defaultValue: """
+        Usage: cmux record [start] [flags]
+               cmux record stop|status [--id <id>]
+               cmux record note [--id <id>] <text>
+               cmux record list
+
+        Record a cmux window to an mp4 or gif, for a pull request or a bug
+        report. Only cmux's own windows can be recorded, and no Screen
+        Recording permission is involved.
+
+        Flags for start:
+          --format mp4|gif       Output format (default: mp4)
+          --gif                  Shorthand for --format gif
+          --fps <1-30>           Frames per second (default: 12 for mp4, 8 for gif)
+          --max-seconds <n>      Stop by itself after n seconds, 0.5-120 (default: 15)
+          --scale <0.1-1>        Scale the output (default: 1 for mp4, 0.5 for gif)
+          --max-width <64-4096>  Cap the output width, keeping the aspect ratio
+          --region <x,y,w,h>     Record part of the window, in window points
+          --out <path>           Write here instead of a temporary directory
+          --label <text>         Name the file
+          --no-captions          Do not draw `record note` captions into the clip
+          --window <id|ref>      Window to record (default: the frontmost one)
+
+        Output: `<id> <state> <frames> <path>`, or the full response with --json.
+
+        A recording stops itself at --max-seconds, so a clip is never left
+        running by an agent that goes away. One recording at a time.
+
+        Example:
+          cmux record start --gif --max-seconds 8 --label sidebar-drag
+          cmux record note "dragging the workspace"
+          cmux record stop
+        """)
+    }
+
+    static var recordUsageLine: String {
+        String(
+            localized: "cli.usage.record",
+            defaultValue: "record [start] [--format mp4|gif] [--gif] [--fps <n>] [--max-seconds <n>] [--scale <n>] [--max-width <n>] [--region <x,y,w,h>] [--out <path>] [--label <text>] [--no-captions] [--window <id|ref|index>] | record stop|status [--id <id>] | record note [--id <id>] <text> | record list"
+        )
+    }
+
+    func runRecordCommand(
+        commandArgs: [String],
+        client: SocketClient,
+        jsonOutput: Bool,
+        windowOverride: String?
+    ) throws {
+        // `cmux record --gif` is `cmux record start --gif`: the subcommand is
+        // only a subcommand when it is not a flag.
+        let hasSubcommand = !(commandArgs.first?.hasPrefix("-") ?? true)
+        let subcommand = hasSubcommand ? commandArgs[0].lowercased() : "start"
+        let arguments = hasSubcommand ? Array(commandArgs.dropFirst()) : commandArgs
+
+        switch subcommand {
+        case "start":
+            try runRecordStart(
+                arguments: arguments,
+                client: client,
+                jsonOutput: jsonOutput,
+                windowOverride: windowOverride
+            )
+        case "stop", "status":
+            var (recordingID, trailing) = parseRecordOption(arguments, name: "--id")
+            if recordingID == nil, let positional = trailing.first, !positional.hasPrefix("-") {
+                recordingID = positional
+                trailing = Array(trailing.dropFirst())
+            }
+            try requireNoRecordArguments(trailing, subcommand: subcommand)
+            var params: [String: Any] = [:]
+            if let recordingID { params["id"] = recordingID }
+            printRecordStatus(
+                try client.sendV2(
+                    method: subcommand == "stop" ? "window.record.stop" : "window.record.status",
+                    params: params,
+                    responseTimeout: 60
+                ),
+                jsonOutput: jsonOutput
+            )
+        case "note":
+            let (id, parsed) = parseRecordOption(arguments, name: "--id")
+            // Only a leading `--` ends the options; one later in the text is
+            // part of the caption.
+            let trailing = parsed.first == "--" ? Array(parsed.dropFirst()) : parsed
+            let text = trailing.joined(separator: " ")
+                .trimmingCharacters(in: .whitespacesAndNewlines)
+            guard !text.isEmpty else {
+                throw CLIError(message: String(
+                    localized: "cli.record.error.noteText",
+                    defaultValue: "record note: text is required"
+                ))
+            }
+            var params: [String: Any] = ["text": text]
+            if let id { params["id"] = id }
+            printRecordStatus(
+                try client.sendV2(method: "window.record.note", params: params),
+                jsonOutput: jsonOutput
+            )
+        case "list":
+            try requireNoRecordArguments(arguments, subcommand: subcommand)
+            let response = try client.sendV2(method: "window.record.list")
+            if jsonOutput {
+                print(jsonString(response))
+                return
+            }
+            let recordings = (response["recordings"] as? [[String: Any]]) ?? []
+            for recording in recordings {
+                print(Self.recordStatusLine(recording))
+            }
+        default:
+            throw CLIError(message: String(
+                format: String(
+                    localized: "cli.record.error.unknownSubcommand",
+                    defaultValue: "record: unknown subcommand '%@' (start, stop, status, note, list)"
+                ),
+                subcommand
+            ))
+        }
+    }
+
+    private func runRecordStart(
+        arguments: [String],
+        client: SocketClient,
+        jsonOutput: Bool,
+        windowOverride: String?
+    ) throws {
+        let (format, afterFormat) = parseRecordOption(arguments, name: "--format")
+        let (fps, afterFPS) = parseRecordOption(afterFormat, name: "--fps")
+        let (maxSeconds, afterSeconds) = parseRecordOption(afterFPS, name: "--max-seconds")
+        let (scale, afterScale) = parseRecordOption(afterSeconds, name: "--scale")
+        let (maxWidth, afterWidth) = parseRecordOption(afterScale, name: "--max-width")
+        let (region, afterRegion) = parseRecordOption(afterWidth, name: "--region")
+        let (out, afterOut) = parseRecordOption(afterRegion, name: "--out")
+        let (label, afterLabel) = parseRecordOption(afterOut, name: "--label")
+        let (window, afterWindow) = parseRecordOption(afterLabel, name: "--window")
+
+        var trailing = afterWindow
+        let wantsGIF = trailing.contains("--gif")
+        let drawsCaptions = !trailing.contains("--no-captions")
+        trailing.removeAll { $0 == "--gif" || $0 == "--no-captions" }
+        try requireNoRecordArguments(trailing, subcommand: "start")
+
+        if wantsGIF, let format, format.lowercased() != "gif" {
+            throw CLIError(message: String(
+                localized: "cli.record.error.formatConflict",
+                defaultValue: "record start: --gif conflicts with --format"
+            ))
+        }
+
+        // Values stay as the caller typed them: the app owns the limits, so the
+        // CLI cannot disagree with it about what a valid frame rate is.
+        var params: [String: Any] = [:]
+        if let format { params["format"] = format } else if wantsGIF { params["format"] = "gif" }
+        if let fps { params["fps"] = fps }
+        if let maxSeconds { params["max_seconds"] = maxSeconds }
+        if let scale { params["scale"] = scale }
+        if let maxWidth { params["max_width"] = maxWidth }
+        if let region { params["region"] = region }
+        if let label { params["label"] = label }
+        if !drawsCaptions { params["captions"] = false }
+        // The app has no idea what directory the CLI was run from.
+        if let out { params["out"] = Self.absoluteRecordingPath(out) }
+        if let windowID = try normalizeWindowHandle(window ?? windowOverride, client: client) {
+            params["window"] = windowID
+        }
+
+        printRecordStatus(
+            try client.sendV2(
+                method: "window.record.start",
+                params: params,
+                responseTimeout: 45
+            ),
+            jsonOutput: jsonOutput
+        )
+    }
+
+    /// A flag where a value belongs is a typo rather than a value: plain
+    /// `parseOption` would let `--label --gif` name the clip "--gif" and quietly
+    /// drop the format. Hand both back so the caller's error names them.
+    private func parseRecordOption(_ args: [String], name: String) -> (String?, [String]) {
+        let (value, remaining) = parseOption(args, name: name)
+        guard let value, value.hasPrefix("--") else { return (value, remaining) }
+        return (nil, [name, value] + remaining)
+    }
+
+    private func requireNoRecordArguments(_ trailing: [String], subcommand: String) throws {
+        guard trailing.isEmpty else {
+            throw CLIError(message: String(
+                format: String(
+                    localized: "cli.record.error.unexpectedArguments",
+                    defaultValue: "record %@: unexpected arguments: %@"
+                ),
+                subcommand,
+                trailing.joined(separator: " ")
+            ))
+        }
+    }
+
+    private func printRecordStatus(_ response: [String: Any], jsonOutput: Bool) {
+        if jsonOutput {
+            print(jsonString(response))
+            return
+        }
+        print(Self.recordStatusLine(response))
+    }
+
+    static func recordStatusLine(_ response: [String: Any]) -> String {
+        let id = (response["id"] as? String) ?? "-"
+        let state = (response["state"] as? String) ?? "-"
+        let frames = (response["frames"] as? Int).map(String.init) ?? "-"
+        let path = (response["path"] as? String) ?? "-"
+        var line = "\(id) \(state) \(frames) \(path)"
+        if let error = response["error"] as? String, !error.isEmpty {
+            line += " (\(error))"
+        }
+        return line
+    }
+
+    static func absoluteRecordingPath(_ path: String) -> String {
+        let expanded = NSString(string: path).expandingTildeInPath
+        guard !expanded.hasPrefix("/") else { return expanded }
+        return URL(fileURLWithPath: FileManager.default.currentDirectoryPath)
+            .appendingPathComponent(expanded)
+            .standardizedFileURL
+            .path
+    }
+}

CLI/CMUXCLI+TaskHelp.swift

@@ -275,6 +275,7 @@ extension CMUXCLI {
         current-workspace [--window <id|ref|index>]
         \(Self.readSelectionUsageLine)
         \(Self.readScreenUsageLine)
+        \(Self.recordUsageLine)
         sidebar-state [--workspace <id|ref|index>] [--window <id|ref|index>]
         markdown [open] <path> [--focus <true|false>] (open markdown file in formatted viewer panel with live reload)
         diff [patch-file|-] [--source <unstaged|staged|branch|last-turn>] [--cwd <path>] [--base <ref>] [--focus <true|false>] [--no-focus] [--title <text>] [--layout <split|unified>] [--font-size <points>] (open patch input or git source in a browser split)

CLI/CMUXCLI+WindowDispatch.swift

@@ -24,6 +24,11 @@ extension CMUXCLI {
         if normalizedCommand == "read-screen" || normalizedCommand == "read-selection" || normalizedCommand == "current" {
             return false
         }
+        // Recording films whatever is on screen; activating a window first
+        // would put the recording's own side effect in the clip.
+        if normalizedCommand == "record" {
+            return false
+        }
         if normalizedCommand == "rpc",
            commandArgs.first?.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
                 == "surface.read_selection" {

CLI/cmux.swift

@@ -7586,6 +7586,14 @@ struct CMUXCLI {
                 includeContextInPlainOutput: true
             )
 
+        case "record":
+            try runRecordCommand(
+                commandArgs: commandArgs,
+                client: client,
+                jsonOutput: jsonOutput,
+                windowOverride: windowId
+            )
+
         case "read-screen":
             let selectionOnly = commandArgs.contains("--selection")
             if selectionOnly {
@@ -20436,6 +20444,8 @@ struct CMUXCLI {
             return Self.readSelectionHelp
         case "read-screen":
             return Self.readScreenHelp
+        case "record":
+            return Self.recordHelp
         case "paste":
             return Self.pasteHelp
         case "send":

Packages/macOS/CmuxControlSocket/Sources/CmuxControlSocket/Wire/ControlCommandExecutionPolicy.swift

@@ -206,6 +206,14 @@ public enum ControlCommandExecutionPolicy: Sendable, Equatable {
         // compositor before falling back to AppKit. Keep that wait on the
         // socket worker so WebKit-backed panels can render on the main actor.
         "debug.window.screenshot",
+        // Window recording samples ScreenCaptureKit on a schedule for as long
+        // as the clip lasts. The sampling loop must never own the main actor:
+        // the window it is filming has to keep drawing.
+        "window.record.start",
+        "window.record.stop",
+        "window.record.status",
+        "window.record.note",
+        "window.record.list",
         // debug.sidebar.simulate_drag intentionally runs on the socket worker
         // so its Thread.sleep between drag-state ticks doesn't block the main
         // actor (which still owns the SidebarDragState mutations via

Packages/macOS/CmuxControlSocket/Tests/CmuxControlSocketTests/ControlCommandExecutionPolicyTests.swift

@@ -124,6 +124,20 @@ struct ControlCommandExecutionPolicyTests {
         )
     }
 
+    @Test func windowRecordingRunsOnTheWorkerAndIsNotMainThreadCallable() {
+        // A recording samples the window for as long as the clip lasts, so its
+        // verbs must never be callable inline on the main thread: the window
+        // being filmed has to keep drawing while the sampler runs.
+        for method in [
+            "window.record.start", "window.record.stop", "window.record.status",
+            "window.record.note", "window.record.list",
+        ] {
+            let policy = ControlCommandExecutionPolicy(forMethod: method)
+            #expect(policy == .socketWorker(mainThreadCallable: false), "\(method)")
+            #expect(policy.runsOnSocketWorker, "\(method)")
+        }
+    }
+
     @Test func v2ResolutionReadsRunOnTheWorkerAndAreMainThreadCallable() {
         // Tranche D (issue #5757): the implicit handle-normalization reads.
         // One controlResolveOnMain hop (refresh + witness + ref minting),

Packages/macOS/CmuxFoundation/Sources/CmuxFoundation/WindowRecording/WindowRecordingCaptionTrack.swift

@@ -0,0 +1,78 @@
+internal import Foundation
+
+/// The captions burned into a recording's frames.
+///
+/// An agent that records a tour wants the clip to say what it did, not just
+/// show a cursor moving: `cmux record note "open Settings"` appends one here,
+/// and the recorder draws whichever note is current when it writes a frame.
+/// Automation commands push their own notes through the same sink, which is
+/// what `--overlay-actions` turns on.
+public struct WindowRecordingCaptionTrack: Sendable, Equatable {
+    public struct Note: Sendable, Equatable {
+        public let offsetSeconds: Double
+        public let text: String
+
+        public init(offsetSeconds: Double, text: String) {
+            self.offsetSeconds = offsetSeconds
+            self.text = text
+        }
+    }
+
+    /// How long one note stays on screen after the moment it was pushed.
+    public static let defaultVisibleSeconds: Double = 2.5
+    /// Longest caption drawn; a whole prompt would cover the window.
+    public static let maximumCharacters = 120
+
+    public let visibleSeconds: Double
+    private var notes: [Note] = []
+
+    public init(visibleSeconds: Double = WindowRecordingCaptionTrack.defaultVisibleSeconds) {
+        self.visibleSeconds = max(0.1, visibleSeconds)
+    }
+
+    public var isEmpty: Bool { notes.isEmpty }
+    public var notesInOrder: [Note] { notes }
+    public var count: Int { notes.count }
+
+    /// Records one caption. Returns false when the text carries nothing to draw.
+    ///
+    /// Notes arrive from the recorder's own clock, so they normally append; an
+    /// out-of-order offset is still inserted in order rather than dropped,
+    /// which keeps `caption(atOffsetSeconds:)` a plain search.
+    @discardableResult
+    public mutating func append(text: String, atOffsetSeconds offset: Double) -> Bool {
+        let collapsed = text
+            .components(separatedBy: .whitespacesAndNewlines)
+            .filter { !$0.isEmpty }
+            .joined(separator: " ")
+        guard !collapsed.isEmpty else { return false }
+        let clipped = collapsed.count > Self.maximumCharacters
+            ? String(collapsed.prefix(Self.maximumCharacters - 1)) + "\u{2026}"
+            : collapsed
+        let note = Note(
+            offsetSeconds: offset.isFinite ? max(0, offset) : 0,
+            text: clipped
+        )
+        if let last = notes.last, last.offsetSeconds <= note.offsetSeconds {
+            notes.append(note)
+        } else {
+            let index = notes.firstIndex { $0.offsetSeconds > note.offsetSeconds } ?? notes.endIndex
+            notes.insert(note, at: index)
+        }
+        return true
+    }
+
+    /// The caption to draw on the frame captured at `offset`, if any.
+    public func caption(atOffsetSeconds offset: Double) -> String? {
+        guard offset.isFinite else { return nil }
+        var current: Note?
+        for note in notes {
+            if note.offsetSeconds > offset { break }
+            current = note
+        }
+        guard let current, offset - current.offsetSeconds < visibleSeconds else {
+            return nil
+        }
+        return current.text
+    }
+}