Play a video or recording an agent writes inline in the chat

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneReplyImages.swift

@@ -20,6 +20,8 @@ public nonisolated enum AgentPaneReplyError: String, Error, Equatable, Sendable
     case imageTooLarge = "link.image_too_large"
     /// The transfer failed, or the bytes are not an image the pane shows.
     case imageFailed = "link.image_failed"
+    /// Not a local video or audio file the pane plays (``AgentPaneMediaGrants``).
+    case mediaRefused = "link.media_refused"
     /// A browser id the last `browser.list` did not give.
     case browserUnknown = "link.browser_unknown"
     case openFailed = "link.open_failed"

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneReplyLinks.swift

@@ -71,6 +71,8 @@ extension AgentPaneModel {
             return Self.replyResult(await openReplyPath(path))
         case .loadImage(let src):
             return await loadReplyImage(src)
+        case .loadMedia(let src):
+            return await loadReplyMedia(src)
         case .listBrowsers:
             return AgentPaneReply.success(["browsers": replyLinks.choices.list(from: replyLinks.browsers)])
         case .openIn(let url, let browserId):

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneReplyMedia.swift

@@ -0,0 +1,71 @@
+import Foundation
+
+/// Local video and audio files the pane may play. The page's CSP loads media only from the pane's
+/// own origin, so a checked file gets an unguessable `cmux-agent://pane/__media/<token>.<ext>` URL
+/// that ``AgentPaneSchemeHandler`` serves in byte ranges. The page never names a file itself: it
+/// asks `media.load` with the path from the reply, and the host checks it like a reply image.
+final class AgentPaneMediaGrants {
+    static let shared = AgentPaneMediaGrants()
+
+    /// The extensions the pane plays, with the type each is served as.
+    nonisolated static let types: [String: String] = [
+        "mp4": "video/mp4", "m4v": "video/x-m4v", "mov": "video/quicktime", "webm": "video/webm",
+        "mp3": "audio/mpeg", "m4a": "audio/mp4", "aac": "audio/aac", "wav": "audio/wav", "flac": "audio/flac",
+    ]
+    /// Largest file the pane plays (it reads only the ranges the player asks for).
+    nonisolated static let maximumBytes = 4 << 30
+    /// Most files granted at once; the oldest grant goes first.
+    static let maximumGrants = 256
+    static let pathPrefix = "/__media/"
+
+    private var files: [String: URL] = [:]
+    private var tokens: [URL: String] = [:]
+    private var order: [String] = []
+
+    /// The URL the page plays `file` (canonical, already checked) by; the same file keeps its URL.
+    func grant(_ file: URL) -> String {
+        let name: String
+        if let known = tokens[file] {
+            name = known
+        } else {
+            name = UUID().uuidString.replacingOccurrences(of: "-", with: "").lowercased() + "." + file.pathExtension.lowercased()
+            files[name] = file
+            tokens[file] = name
+            order.append(name)
+            if order.count > Self.maximumGrants, let oldest = order.first {
+                order.removeFirst()
+                if let gone = files.removeValue(forKey: oldest) { tokens[gone] = nil }
+            }
+        }
+        return "\(AgentPaneSource.bundledScheme)://\(AgentPaneSource.bundledHost)\(Self.pathPrefix)\(name)"
+    }
+
+    /// The granted file a media URL names, nil for any other URL.
+    func file(for url: URL) -> URL? {
+        guard url.scheme?.lowercased() == AgentPaneSource.bundledScheme,
+              url.host?.lowercased() == AgentPaneSource.bundledHost,
+              url.path.hasPrefix(Self.pathPrefix) else { return nil }
+        return files[String(url.path.dropFirst(Self.pathPrefix.count))]
+    }
+}
+
+extension AgentPaneModel {
+    /// A video or audio file inside the roots, as a URL the pane plays. Web media is refused
+    /// (the pane fetches nothing itself).
+    func loadReplyMedia(_ src: String) async -> [String: Any] {
+        if let scheme = URL(string: src)?.scheme?.lowercased(), scheme != "file" { return Self.replyFailure(.mediaRefused) }
+        guard let resolved = replyPaths().resolve(src) else { return Self.replyFailure(.pathInvalid) }
+        switch resolved.place {
+        case .root: break
+        case .denied: return Self.replyFailure(.pathDenied)
+        case .outside: return Self.replyFailure(.pathOutsideRoots)
+        case .missing: return Self.replyFailure(.pathInvalid)
+        }
+        let file = URL(fileURLWithPath: resolved.path)
+        guard AgentPaneMediaGrants.types[file.pathExtension.lowercased()] != nil,
+              let values = try? file.resourceValues(forKeys: [.isRegularFileKey, .fileSizeKey]), values.isRegularFile == true,
+              (values.fileSize ?? Int.max) <= AgentPaneMediaGrants.maximumBytes
+        else { return Self.replyFailure(.mediaRefused) }
+        return AgentPaneReply.success(["src": AgentPaneMediaGrants.shared.grant(file)])
+    }
+}

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneReplyRequest.swift

@@ -10,24 +10,28 @@ public import Foundation
 /// - `link.openPath {path}`: open a path chip. Spends a user gesture.
 /// - `image.load {src}`: a reply image, a file inside the roots or an https URL (host fetch).
 ///   A web image spends a gesture unless `agentPane.images.remote` is `always`.
+/// - `media.load {src}`: a video or audio file inside the roots, as a URL the pane plays
+///   (``AgentPaneMediaGrants``). Passive.
 /// - `browser.list {}`: the installed browsers, by opaque id. Passive.
 /// - `browser.openIn {url, browserId}`: open a validated http(s) URL in one of them. Spends a
 ///   user gesture.
 public nonisolated enum AgentPaneReplyRequest: Equatable, Sendable {
     case inspect(paths: [String], urls: [String])
     case openPath(String)
     case loadImage(String)
+    case loadMedia(String)
     case listBrowsers
     case openIn(url: URL, browserId: String)
 
     /// The methods, for ``AgentPageOps`` and ``AgentPaneRequest``.
-    public static let methods: Set<String> = ["link.inspect", "link.openPath", "image.load", "browser.list", "browser.openIn"]
+    public static let methods: Set<String> = ["link.inspect", "link.openPath", "image.load", "media.load", "browser.list", "browser.openIn"]
 
     /// Each method's exact param keys (all required; `link.inspect` takes either list).
     static let knownParams: [String: Set<String>] = [
         "link.inspect": ["paths", "urls"],
         "link.openPath": ["path"],
         "image.load": ["src"],
+        "media.load": ["src"],
         "browser.list": [],
         "browser.openIn": ["url", "browserId"],
     ]
@@ -40,7 +44,7 @@ public nonisolated enum AgentPaneReplyRequest: Equatable, Sendable {
     /// Whether the request only reads (it does not count as the page being touched).
     var isPassive: Bool {
         switch self {
-        case .inspect, .listBrowsers, .loadImage: true
+        case .inspect, .listBrowsers, .loadImage, .loadMedia: true
         case .openPath, .openIn: false
         }
     }
@@ -74,6 +78,9 @@ public nonisolated enum AgentPaneReplyRequest: Equatable, Sendable {
         case "image.load":
             guard let src = text("src", Self.maximumURL) else { return nil }
             self = .loadImage(src)
+        case "media.load":
+            guard let src = text("src", Self.maximumURL) else { return nil }
+            self = .loadMedia(src)
         case "browser.list":
             self = .listBrowsers
         case "browser.openIn":

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneSchemeHandler+Media.swift

@@ -0,0 +1,98 @@
+import Foundation
+import WebKit
+
+extension AgentPaneSchemeHandler {
+    /// Bytes one media response carries at most; the player asks for the next range.
+    nonisolated static let mediaChunk = 8 << 20
+
+    /// One answer to a media request: 200 for the whole file, 206 for a range, 416 past the end.
+    nonisolated struct MediaSlice: Sendable {
+        var status: Int
+        var headers: [String: String]
+        var data: Data
+    }
+
+    /// The granted file a `cmux-agent://pane/__media/` URL names (``AgentPaneMediaGrants``).
+    static func mediaFile(for url: URL) -> URL? {
+        AgentPaneMediaGrants.shared.file(for: url)
+    }
+
+    /// The part of `file` an HTTP `Range` header asks for (`bytes=a-b`, `bytes=a-` or `bytes=-n`),
+    /// at most ``mediaChunk`` bytes. Nil when the file cannot be read or is no longer the regular
+    /// file that was granted (a link swapped in after the grant).
+    nonisolated static func mediaSlice(of file: URL, range: String?) -> MediaSlice? {
+        guard let values = try? file.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey, .fileSizeKey]),
+              values.isRegularFile == true, values.isSymbolicLink != true, let size = values.fileSize
+        else { return nil }
+        var headers = [
+            "Content-Type": AgentPaneMediaGrants.types[file.pathExtension.lowercased()] ?? "application/octet-stream",
+            "Accept-Ranges": "bytes",
+            "Cache-Control": "no-store",
+            "X-Content-Type-Options": "nosniff",
+            "Content-Security-Policy": resourcePolicy,
+        ]
+        var lower = 0, upper = size - 1
+        if let range {
+            guard let (start, end) = byteRange(range, size: size), start < size else {
+                headers["Content-Range"] = "bytes */\(size)"
+                headers["Content-Length"] = "0"
+                return MediaSlice(status: 416, headers: headers, data: Data())
+            }
+            lower = start
+            upper = min(end, size - 1)
+        }
+        upper = min(upper, lower + mediaChunk - 1)
+        let partial = range != nil || upper < size - 1
+        guard let handle = try? FileHandle(forReadingFrom: file) else { return nil }
+        defer { try? handle.close() }
+        guard (try? handle.seek(toOffset: UInt64(lower))) != nil,
+              let data = size == 0 ? Data() : try? handle.read(upToCount: upper - lower + 1) else { return nil }
+        headers["Content-Length"] = String(data.count)
+        if partial { headers["Content-Range"] = "bytes \(lower)-\(lower + data.count - 1)/\(size)" }
+        return MediaSlice(status: partial ? 206 : 200, headers: headers, data: data)
+    }
+
+    /// The first and last byte a `Range` header names, nil for a header this does not read.
+    nonisolated static func byteRange(_ header: String, size: Int) -> (Int, Int)? {
+        let spec = header.trimmingCharacters(in: .whitespaces)
+        guard spec.lowercased().hasPrefix("bytes="), !spec.contains(",") else { return nil }
+        let parts = spec.dropFirst(6).split(separator: "-", maxSplits: 1, omittingEmptySubsequences: false)
+        guard parts.count == 2 else { return nil }
+        let first = parts[0].trimmingCharacters(in: .whitespaces), last = parts[1].trimmingCharacters(in: .whitespaces)
+        if first.isEmpty {
+            guard let count = Int(last), count > 0 else { return nil }
+            return (max(0, size - count), size - 1)
+        }
+        guard let start = Int(first), start >= 0 else { return nil }
+        if last.isEmpty { return (start, size - 1) }
+        guard let end = Int(last), end >= start else { return nil }
+        return (start, end)
+    }
+
+    /// Reads the slice off the main thread.
+    @concurrent nonisolated static func readSlice(of file: URL, range: String?) async -> MediaSlice? {
+        // concurrency-allow: @concurrent, so this read never runs on the main actor
+        mediaSlice(of: file, range: range)
+    }
+
+    /// Answers `task` for the granted `file`; a stopped task is never answered.
+    func serveMedia(_ file: URL, url: URL, task: any WKURLSchemeTask) {
+        let id = ObjectIdentifier(task)
+        active.insert(id)
+        let range = task.request.value(forHTTPHeaderField: "Range")
+        // task-owner: one range read per media request; a stopped task is never answered
+        Task { @MainActor [weak self] in
+            let slice = await Self.readSlice(of: file, range: range)
+            guard let self, self.active.remove(id) != nil else { return }
+            guard let slice,
+                  let response = HTTPURLResponse(url: url, statusCode: slice.status, httpVersion: "HTTP/1.1", headerFields: slice.headers)
+            else {
+                task.didFailWithError(URLError(.fileDoesNotExist))
+                return
+            }
+            task.didReceive(response)
+            task.didReceive(slice.data)
+            task.didFinish()
+        }
+    }
+}

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentPaneSchemeHandler.swift

@@ -9,6 +9,8 @@ import WebKit
 ///
 /// Only GET requests for files directly beside the page are answered; any
 /// other host, an escaping path or a missing file fails the request.
+/// Media the host granted the page (``AgentPaneMediaGrants``) is served from
+/// `__media/` in byte ranges.
 final class AgentPaneSchemeHandler: NSObject, WKURLSchemeHandler {
     private let root: URL
 
@@ -17,9 +19,12 @@ final class AgentPaneSchemeHandler: NSObject, WKURLSchemeHandler {
     }
 
     /// Tasks started and not yet answered or stopped; a stopped task must not be answered.
-    private var active: Set<ObjectIdentifier> = []
+    var active: Set<ObjectIdentifier> = []
 
     func webView(_ webView: WKWebView, start task: any WKURLSchemeTask) {
+        if task.request.httpMethod.map({ $0 == "GET" }) ?? true, let url = task.request.url, let media = Self.mediaFile(for: url) {
+            return serveMedia(media, url: url, task: task)
+        }
         guard task.request.httpMethod.map({ $0 == "GET" }) ?? true,
               let url = task.request.url,
               let file = Self.fileURL(for: url, root: root)

Packages/macOS/CmuxNext/Tests/CmuxNextAgentPaneTests/AgentPaneReplyMediaTests.swift

@@ -0,0 +1,69 @@
+import CmuxNextSettings
+import Foundation
+import Testing
+@testable import CmuxNextAgentPane
+
+/// Local video and audio in replies and tool output: the host checks the path like a reply image,
+/// then lets the page play the file by an unguessable `cmux-agent://pane/__media/` URL that the
+/// pane's scheme handler serves in byte ranges. The page never names a file itself.
+@MainActor
+@Suite struct AgentPaneReplyMediaTests {
+    static func load(_ model: AgentPaneModel, _ src: String) async -> [String: Any] {
+        await model.respond(to: AgentPaneRequest(body: ["method": "media.load", "params": ["src": src]] as [String: Any]))
+    }
+
+    @Test func aVideoInsideTheRootsGetsAPlayableURLAndNothingElseDoes() async throws {
+        let base = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "reply-media-\(UUID().uuidString)")
+        let root = base.appending(path: "repo")
+        try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
+        defer { try? FileManager.default.removeItem(at: base) }
+        try Data(repeating: 7, count: 64).write(to: root.appending(path: "demo.mp4"))
+        try Data(repeating: 7, count: 64).write(to: root.appending(path: "notes.txt"))
+        try Data(repeating: 7, count: 64).write(to: base.appending(path: "outside.mp4"))
+        let model = AgentPaneReplyImageTests.model(root: root)
+
+        let src = try #require(AgentPaneReplyImageTests.src(await Self.load(model, root.appending(path: "demo.mp4").path)))
+        #expect(src.hasPrefix("cmux-agent://pane/__media/"))
+        #expect(src.hasSuffix(".mp4"))
+        #expect(!src.contains("demo"))
+        // The same file keeps its URL, so a re-render does not restart playback.
+        #expect(AgentPaneReplyImageTests.src(await Self.load(model, "demo.mp4")) == src)
+        #expect(AgentPaneSchemeHandler.mediaFile(for: try #require(URL(string: src)))?.lastPathComponent == "demo.mp4")
+
+        #expect(AgentPaneReplyLinkTests.code(await Self.load(model, root.appending(path: "notes.txt").path)) == "link.media_refused")
+        #expect(AgentPaneReplyLinkTests.code(await Self.load(model, base.appending(path: "outside.mp4").path)) == "link.path_outside_roots")
+        #expect(AgentPaneReplyLinkTests.code(await Self.load(model, "https://example.com/demo.mp4")) == "link.media_refused")
+        #expect(AgentPaneSchemeHandler.mediaFile(for: try #require(URL(string: "cmux-agent://pane/__media/0000.mp4"))) == nil)
+    }
+
+    @Test func theSchemeHandlerServesAGrantedFileInByteRanges() throws {
+        let file = URL(fileURLWithPath: NSTemporaryDirectory()).appending(path: "media-\(UUID().uuidString).mp4")
+        try Data((0 ..< 100).map { UInt8($0) }).write(to: file)
+        defer { try? FileManager.default.removeItem(at: file) }
+
+        let part = try #require(AgentPaneSchemeHandler.mediaSlice(of: file, range: "bytes=10-19"))
+        #expect(part.status == 206)
+        #expect(part.data == Data((10 ..< 20).map { UInt8($0) }))
+        #expect(part.headers["Content-Range"] == "bytes 10-19/100")
+        #expect(part.headers["Content-Type"] == "video/mp4")
+        #expect(part.headers["Accept-Ranges"] == "bytes")
+        #expect(part.headers["Content-Length"] == "10")
+
+        let tail = try #require(AgentPaneSchemeHandler.mediaSlice(of: file, range: "bytes=90-"))
+        #expect(tail.headers["Content-Range"] == "bytes 90-99/100")
+        let whole = try #require(AgentPaneSchemeHandler.mediaSlice(of: file, range: nil))
+        #expect(whole.status == 200)
+        #expect(whole.data.count == 100)
+        // A range past the end is unsatisfiable.
+        #expect(AgentPaneSchemeHandler.mediaSlice(of: file, range: "bytes=200-")?.status == 416)
+    }
+
+    @Test func theParamsContractTakesOneSource() {
+        func request(_ params: [String: Any]) -> AgentPaneRequest {
+            AgentPaneRequest(body: ["method": "media.load", "params": params] as [String: Any])
+        }
+        #expect(request(["src": "/a.mp4"]) == .reply(.loadMedia("/a.mp4")))
+        #expect(request(["src": "/a.mp4", "autoplay": true]) == .unsupported("media.load"))
+        #expect(request(["src": ""]) == .unsupported("media.load"))
+    }
+}

scripts/cmux-next/build-agent-pane-web.sh

@@ -72,7 +72,7 @@ cat "$SRC/acpmux/turnChanges/turnChanges.css" >> "$WORK/styles.css"
 # host's native transport (AgentPaneTransport) carries acpmux. No remote loads. Frames show only loopback web
 # pages (a turn's preview card; URL+AgentPanePreview.swift keeps the same hosts). The page host sends the same
 # script-src (PageDescriptor.agent, test/fixtures/agent-page-csp.txt).
-CSP="default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; img-src data:; font-src data:; connect-src 'none'; frame-src http://localhost:* http://127.0.0.1:* https://localhost:* https://127.0.0.1:*"
+CSP="default-src 'none'; script-src 'self'; style-src 'unsafe-inline'; img-src data:; media-src 'self'; font-src data:; connect-src 'none'; frame-src http://localhost:* http://127.0.0.1:* https://localhost:* https://127.0.0.1:*"
 
 {
   printf '<!doctype html>\n<html lang="en">\n<head>\n'