cmux mcp: video_frames and video_info, so an agent looks into a video without ffmpeg

Packages/macOS/CmuxNext/Sources/CmuxNextAgentPane/AgentVideoFrames.swift

@@ -0,0 +1,102 @@
+import AVFoundation
+import CoreGraphics
+// Public: `read` takes a `URL` (InternalImportsByDefault).
+public import Foundation
+
+/// Frames of a video file at the times an agent asks for, for the `video_info` and
+/// `video_frames` agent tools (`cmux mcp`, cmux-tui `video_tools.rs`), through the app method
+/// `video.frames` (CmuxNextApp `VideoControl`). Leo, 2026-11-01, after an agent had to
+/// hand-write ffmpeg to see a 16 s recording: "i feel like cmux-next should give u better tools
+/// to deal with this kind of thing". The server picks the times, the size and the budget and
+/// finds the scene changes; this only decodes, with the composer's decoder settings
+/// (``AgentPaneVideoInfo``): only the frames asked for, never the whole file, and a reference
+/// movie's media in other files is refused.
+public nonisolated enum AgentVideoFrames {
+    /// How a frame comes back: a JPEG (``AgentPaneVideoInfo/jpeg(_:)``), or `side` by `side`
+    /// 8-bit grey pixels, the samples the server compares to find scene changes.
+    public enum Format: String, Sendable {
+        case jpeg
+        case luma
+    }
+
+    public struct Frame: Equatable, Sendable {
+        /// Where in the video the frame is, in milliseconds.
+        public var atMs: Int
+        /// Its bytes, base64.
+        public var data: String
+    }
+
+    public struct Report: Equatable, Sendable {
+        public var durationMs: Int
+        /// The size the video shows at, after its track's rotation.
+        public var width: Int
+        public var height: Int
+        /// The video track's nominal frame rate (0 when the file does not say).
+        public var fps: Double
+        public var hasAudio: Bool
+        public var frames: [Frame] = []
+        /// Frames asked for and left out because the answer reached its byte budget.
+        public var dropped = 0
+    }
+
+    /// The most times one request decodes (the server's scene samples).
+    public static let maximumTimes = 64
+    /// The sides a frame may be asked at.
+    public static let sides = 8...2048
+
+    /// The video at `file` and its frames at `times` (seconds), each at most `side` pixels on its
+    /// longest side, within `tolerance` seconds of the time asked; frames stop once their base64
+    /// passes `maxBytes`. Nil when the file is not a video AVFoundation can show.
+    @concurrent public static func read(_ file: URL, times: [Double], side: Int, format: Format,
+                                        tolerance: Double, maxBytes: Int) async -> Report? {
+        guard AgentPaneVideoInfo.isRegularFile(file) else { return nil }
+        let asset = AVURLAsset(url: file, options: [AVURLAssetReferenceRestrictionsKey: AVAssetReferenceRestrictions.forbidAll.rawValue])
+        guard let duration = try? await asset.load(.duration), let track = try? await asset.loadTracks(withMediaType: .video).first,
+              let geometry = try? await track.load(.naturalSize, .preferredTransform) else { return nil }
+        let seconds = duration.seconds
+        let shown = CGRect(origin: .zero, size: geometry.0).applying(geometry.1).size
+        let width = Int(abs(shown.width).rounded()), height = Int(abs(shown.height).rounded())
+        guard seconds.isFinite, seconds > 0, width > 0, height > 0 else { return nil }
+        let fps = Double((try? await track.load(.nominalFrameRate)) ?? 0)
+        let hasAudio = !((try? await asset.loadTracks(withMediaType: .audio)) ?? []).isEmpty
+        var report = Report(durationMs: Int((seconds * 1000).rounded()), width: width, height: height,
+                            fps: fps.isFinite ? fps : 0, hasAudio: hasAudio)
+        let times = Array(times.prefix(maximumTimes))
+        guard !times.isEmpty else { return report }
+        let side = min(max(side, sides.lowerBound), sides.upperBound)
+        let generator = AVAssetImageGenerator(asset: asset)
+        generator.appliesPreferredTrackTransform = true
+        // A grey sample is drawn small from a small decode; a JPEG is decoded at its size.
+        let decodeSide = format == .jpeg ? side : max(side * 8, 64)
+        generator.maximumSize = CGSize(width: decodeSide, height: decodeSide)
+        let slack = CMTime(seconds: tolerance.isFinite ? max(0, tolerance) : 0, preferredTimescale: 600)
+        generator.requestedTimeToleranceBefore = slack
+        generator.requestedTimeToleranceAfter = slack
+        var used = 0
+        for (index, time) in times.enumerated() {
+            let asked = min(max(time.isFinite ? time : 0, 0), seconds)
+            guard let frame = try? await generator.image(at: CMTime(seconds: asked, preferredTimescale: 600)),
+                  let data = format == .jpeg ? AgentPaneVideoInfo.jpeg(frame.image) : luma(frame.image, side: side) else { continue }
+            let encoded = data.base64EncodedString()
+            guard used + encoded.utf8.count <= maxBytes else {
+                report.dropped = times.count - index
+                break
+            }
+            used += encoded.utf8.count
+            let at = frame.actualTime.seconds.isFinite ? frame.actualTime.seconds : asked
+            report.frames.append(Frame(atMs: Int((min(max(at, 0), seconds) * 1000).rounded()), data: encoded))
+        }
+        return report
+    }
+
+    /// `image` drawn into `side` by `side` 8-bit grey pixels (its aspect is not kept: the samples
+    /// are only compared with each other).
+    static func luma(_ image: CGImage, side: Int) -> Data? {
+        guard let context = CGContext(data: nil, width: side, height: side, bitsPerComponent: 8, bytesPerRow: side,
+                                      space: CGColorSpaceCreateDeviceGray(), bitmapInfo: CGImageAlphaInfo.none.rawValue) else { return nil }
+        context.interpolationQuality = .medium
+        context.draw(image, in: CGRect(x: 0, y: 0, width: side, height: side))
+        guard let pixels = context.data else { return nil }
+        return Data(bytes: pixels, count: side * side)
+    }
+}

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Control/AppControl.swift

@@ -86,6 +86,8 @@ final class AppControl {
         service.router.register(FeedControl.methods(services: services))
         service.router.register(ServerReachControl.methods(services: services))
         service.router.register(KeybindingControl.methods(services: services))
+        // `video.frames`: a video's numbers and frames for the agent tools `video_info` and `video_frames`.
+        service.router.register(VideoControl.methods())
         // `agent.acpmux`: the acpmux this app's chat tabs use (`cmux agents chat new`, #23026).
         let build = services.environment.build
         service.router.register([.agentAcpmux {

Packages/macOS/CmuxNext/Sources/CmuxNextApp/Control/VideoControl.swift

@@ -0,0 +1,66 @@
+import CmuxNextAgentPane
+import CmuxNextControl
+import CmuxNextSettings
+import Darwin
+import Foundation
+
+/// `video.frames` on the app control socket: a video file's length, size, frame rate, audio
+/// and its frames at the times asked (``AgentVideoFrames``), for the `video_info` and
+/// `video_frames` agent tools (cmux-tui `cli/mcp/video_tools.rs`), so an agent looks into a
+/// video without ffmpeg.
+///
+/// Params: `path` (absolute), `dev` and `ino` (the file as the caller sees it, decimal
+/// strings), `times` (seconds; empty for only the numbers), `side` (longest side, default
+/// 1280), `format` (`jpeg` or `luma`), `tolerance` (seconds), `maxBytes` (the frames' base64
+/// budget). A path whose file is not the caller's (a chat whose server runs on another machine
+/// names its own files) is `not_found`: this Mac's file of that name is never read for it.
+nonisolated enum VideoControl {
+    /// Up to 64 small frames or 16 large ones from a long file.
+    static let deadline: Duration = .seconds(60)
+    /// The largest answer: under the CLI's 16 MiB response cap.
+    static let maximumBytes = 12 << 20
+
+    static func methods() -> [ControlMethod] {
+        [.async("video.frames") { call in try await Self.run(call.params) }.withDeadline(.fixed(deadline))]
+    }
+
+    static func run(_ params: [String: JSONValue]) async throws -> JSONValue {
+        guard let path = params["path"]?.stringValue, path.hasPrefix("/") else {
+            throw ControlError.invalidParams("video.frames needs path: an absolute file path")
+        }
+        guard let dev = params["dev"]?.stringValue, let ino = params["ino"]?.stringValue else {
+            throw ControlError.invalidParams("video.frames needs dev and ino: the file as the caller sees it")
+        }
+        var info = stat()
+        guard stat(path, &info) == 0, String(UInt64(bitPattern: Int64(info.st_dev))) == dev, String(info.st_ino) == ino else {
+            throw ControlError(code: "not_found", message: "\(path) is not a file on the Mac the cmux app runs on; a chat on another machine reads its videos with that machine's own tools")
+        }
+        let times: [Double] = try (params["times"]?.arrayValue ?? []).map { value in
+            guard let seconds = value.doubleValue, seconds.isFinite, seconds >= 0 else {
+                throw ControlError.invalidParams("video.frames times are seconds, 0 or more")
+            }
+            return seconds
+        }
+        guard times.count <= AgentVideoFrames.maximumTimes else {
+            throw ControlError.invalidParams("video.frames takes at most \(AgentVideoFrames.maximumTimes) times")
+        }
+        let format: AgentVideoFrames.Format = try params["format"]?.stringValue.map { raw in
+            guard let parsed = AgentVideoFrames.Format(rawValue: raw) else { throw ControlError.invalidParams("video.frames format is jpeg or luma") }
+            return parsed
+        } ?? .jpeg
+        let side = params["side"]?.intValue ?? 1280
+        guard AgentVideoFrames.sides.contains(side) else {
+            throw ControlError.invalidParams("video.frames side is \(AgentVideoFrames.sides.lowerBound) to \(AgentVideoFrames.sides.upperBound)")
+        }
+        let budget = min(max(params["maxBytes"]?.intValue ?? maximumBytes, 0), maximumBytes)
+        guard let report = await AgentVideoFrames.read(URL(fileURLWithPath: path), times: times, side: side, format: format,
+                                                       tolerance: params["tolerance"]?.doubleValue ?? 0.5, maxBytes: budget) else {
+            throw ControlError(code: "unsupported", message: "\(path) is not a video this Mac can decode (no video track, no length, or no frame)")
+        }
+        return .object([
+            "durationMs": JSONValue(report.durationMs), "width": JSONValue(report.width), "height": JSONValue(report.height),
+            "fps": .number(report.fps), "hasAudio": .bool(report.hasAudio), "dropped": JSONValue(report.dropped),
+            "frames": .array(report.frames.map { frame in JSONValue.object(["atMs": JSONValue(frame.atMs), "data": .string(frame.data)]) }),
+        ])
+    }
+}

Packages/macOS/CmuxNext/Tests/CmuxNextAgentPaneTests/AgentVideoFramesTests.swift

@@ -0,0 +1,123 @@
+import AVFoundation
+@testable import CmuxNextAgentPane
+import CoreGraphics
+import CoreVideo
+import Foundation
+import ImageIO
+import Testing
+
+/// Leo, 2026-11-01: "i feel like cmux-next should give u better tools to deal with this kind of
+/// thing". The `video_frames` agent tool's decoder (``AgentVideoFrames``, `video.frames`): the
+/// numbers, frames at the asked times and size, grey samples for scene changes, and a byte budget.
+@Suite(.serialized)
+struct AgentVideoFramesTests {
+    /// A 2 s H.264 movie of `width`×`height` at 5 frames a second: red for its first second,
+    /// blue for the next.
+    private static func movie(width: Int, height: Int) async throws -> URL {
+        let url = FileManager.default.temporaryDirectory.appendingPathComponent("video-frames-\(UUID().uuidString).mov")
+        let writer = try AVAssetWriter(outputURL: url, fileType: .mov)
+        let input = AVAssetWriterInput(mediaType: .video, outputSettings: [
+            AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: width, AVVideoHeightKey: height,
+        ])
+        input.expectsMediaDataInRealTime = false
+        let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: [
+            kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA,
+            kCVPixelBufferWidthKey as String: width, kCVPixelBufferHeightKey as String: height,
+        ])
+        writer.add(input)
+        #expect(writer.startWriting(), "\(String(describing: writer.error))")
+        writer.startSession(atSourceTime: .zero)
+        for index in 0..<10 {
+            while !input.isReadyForMoreMediaData { await Task.yield() }
+            var buffer: CVPixelBuffer?
+            CVPixelBufferCreate(nil, width, height, kCVPixelFormatType_32BGRA, nil, &buffer)
+            let pixels = try #require(buffer)
+            CVPixelBufferLockBaseAddress(pixels, [])
+            let base = try #require(CVPixelBufferGetBaseAddress(pixels)).assumingMemoryBound(to: UInt8.self)
+            let red = index < 5
+            for row in 0..<height {
+                let line = base + row * CVPixelBufferGetBytesPerRow(pixels)
+                for column in 0..<width {
+                    line[column * 4] = red ? 0 : 255
+                    line[column * 4 + 1] = 0
+                    line[column * 4 + 2] = red ? 255 : 0
+                    line[column * 4 + 3] = 255
+                }
+            }
+            CVPixelBufferUnlockBaseAddress(pixels, [])
+            #expect(adaptor.append(pixels, withPresentationTime: CMTime(value: CMTimeValue(index), timescale: 5)))
+        }
+        input.markAsFinished()
+        await writer.finishWriting()
+        #expect(writer.status == .completed, "\(String(describing: writer.error))")
+        return url
+    }
+
+    private static func image(_ base64: String) throws -> CGImage {
+        let jpeg = try #require(Data(base64Encoded: base64))
+        let source = try #require(CGImageSourceCreateWithData(jpeg as CFData, nil))
+        return try #require(CGImageSourceCreateImageAtIndex(source, 0, nil))
+    }
+
+    private static func isRed(_ image: CGImage) throws -> Bool {
+        var pixel = [UInt8](repeating: 0, count: 4)
+        let context = try #require(CGContext(data: &pixel, width: 1, height: 1, bitsPerComponent: 8, bytesPerRow: 4,
+                                             space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue))
+        context.draw(image, in: CGRect(x: -image.width / 2, y: -image.height / 2, width: image.width, height: image.height))
+        return pixel[0] > pixel[2]
+    }
+
+    @Test func noTimesGiveTheNumbersOnly() async throws {
+        let clip = try await Self.movie(width: 640, height: 360)
+        defer { try? FileManager.default.removeItem(at: clip) }
+        let report = try #require(await AgentVideoFrames.read(clip, times: [], side: 1280, format: .jpeg, tolerance: 0, maxBytes: 1 << 20))
+        #expect(abs(report.durationMs - 2000) <= 250, "\(report.durationMs)")
+        #expect(report.width == 640 && report.height == 360)
+        #expect(abs(report.fps - 5) < 1, "\(report.fps)")
+        #expect(!report.hasAudio)
+        #expect(report.frames.isEmpty && report.dropped == 0)
+    }
+
+    @Test func framesAreAtTheAskedTimesAndSize() async throws {
+        let clip = try await Self.movie(width: 640, height: 360)
+        defer { try? FileManager.default.removeItem(at: clip) }
+        let report = try #require(await AgentVideoFrames.read(clip, times: [0.2, 1.6], side: 320, format: .jpeg, tolerance: 0, maxBytes: 1 << 20))
+        #expect(report.frames.map(\.atMs) == [200, 1600])
+        let frames = try report.frames.map { try Self.image($0.data) }
+        #expect(frames.allSatisfy { $0.width == 320 && $0.height == 180 })
+        #expect(try Self.isRed(frames[0]))
+        #expect(try !Self.isRed(frames[1]))
+        // A frame is never made larger than the video.
+        let large = try #require(await AgentVideoFrames.read(clip, times: [0.2], side: 2048, format: .jpeg, tolerance: 0, maxBytes: 1 << 20))
+        #expect(try Self.image(large.frames[0].data).width == 640)
+    }
+
+    @Test func greySamplesAreSideBySideBytesThatChangeWithThePicture() async throws {
+        let clip = try await Self.movie(width: 640, height: 360)
+        defer { try? FileManager.default.removeItem(at: clip) }
+        let report = try #require(await AgentVideoFrames.read(clip, times: [0.2, 0.6, 1.4], side: 8, format: .luma, tolerance: 0, maxBytes: 1 << 20))
+        let samples = try report.frames.map { try #require(Data(base64Encoded: $0.data)) }
+        #expect(samples.allSatisfy { $0.count == 64 })
+        let change = { (a: Data, b: Data) in zip(a, b).map { abs(Int($0) - Int($1)) }.reduce(0, +) / a.count }
+        #expect(change(samples[0], samples[1]) < 2, "red to red")
+        #expect(change(samples[1], samples[2]) > 10, "red to blue")
+    }
+
+    @Test func framesStopAtTheByteBudgetAndSayHowManyAreLeftOut() async throws {
+        let clip = try await Self.movie(width: 640, height: 360)
+        defer { try? FileManager.default.removeItem(at: clip) }
+        let one = try #require(await AgentVideoFrames.read(clip, times: [0.2], side: 640, format: .jpeg, tolerance: 0, maxBytes: 1 << 20))
+        let size = one.frames[0].data.utf8.count
+        let report = try #require(await AgentVideoFrames.read(clip, times: [0.2, 0.4, 0.6, 0.8], side: 640, format: .jpeg, tolerance: 0,
+                                                              maxBytes: size * 2 + size / 2))
+        #expect(report.frames.count == 2)
+        #expect(report.dropped == 2)
+    }
+
+    @Test func aFileThatIsNotAVideoGivesNothing() async throws {
+        let file = FileManager.default.temporaryDirectory.appendingPathComponent("not-a-video-\(UUID().uuidString).mp4")
+        try Data("hello".utf8).write(to: file)
+        defer { try? FileManager.default.removeItem(at: file) }
+        #expect(await AgentVideoFrames.read(file, times: [0], side: 320, format: .jpeg, tolerance: 0, maxBytes: 1 << 20) == nil)
+    }
+}

cmux-tui/crates/cmux-tui/src/cli/mcp.rs

@@ -41,6 +41,7 @@ mod transport;
 mod upstream;
 mod upstream_listing;
 mod v2_tools;
+mod video_tools;
 mod watch;
 
 use std::collections::BTreeSet;
@@ -722,6 +723,7 @@ impl<B: Backend> Server<B> {
         tools.extend(terminal_talk_tools::descriptors());
         tools.push(render_tool::descriptor());
         tools.push(open_tool::descriptor());
+        tools.extend(video_tools::descriptors());
         tools
             .extend(settings_tools::TOOLS.iter().map(settings_tools::SettingTool::descriptor_json));
         tools.extend(
@@ -935,6 +937,9 @@ impl<B: Backend> Server<B> {
             let progress = agent_wait_tool::Progress::requested(params, self.notes.as_ref());
             return Ok(terminal_talk_tools::call_send(&self.backend, &arguments, progress));
         }
+        if let Some(result) = self.call_video(name, &arguments) {
+            return Ok(result);
+        }
         if name == open_tool::NAME {
             if !self.actions_loaded {
                 self.refresh_actions();