Agent pane: a look at a video's frames draws as a filmstrip card

webviews/src/agent-session/acpmux/conversation/McpCallRows.tsx

@@ -26,6 +26,8 @@ import { RawToggle } from "./SentMessageLine";
 import { isFailed, isRunning } from "./toolGroups";
 import { ToolOutput } from "./ToolOutputBody";
 import { ToolImages } from "./ToolRow";
+import { VideoFramesCard } from "./VideoFramesCard";
+import { videoFrames } from "./videoFrames";
 import { useRowDisclosure, useTranscriptDisclosure } from "./transcriptDisclosure";
 import { useAnchoredCollapse } from "./useAnchoredCollapse";
 import { ToolLabel } from "./ToolVerb";
@@ -119,12 +121,18 @@ function McpCallRow({ item, grouped = false }: { item: AcpmuxActivity; grouped?:
   // The machine name stays at hand as the opened call's first line (no tooltip: the line's text
   // already shows, test/tooltip-rules.ts).
   const raw = mcpRawName(tool, call);
+  // A look at a video's frames reads as one ("Looked at 8 video frames") over its filmstrip, which
+  // names the video (videoFrames.ts); opened, the call's response is the tool's note without the frames.
+  const video = useMemo(() => videoFrames(tool), [tool]);
   const label = useMemo(
-    () => cmuxTitle(t, [call], [tool], names) ?? mcpLabel(t, [call], names),
-    [t, call, tool, names],
+    () =>
+      video
+        ? t("tools.videoFrames", { n: video.count })
+        : (cmuxTitle(t, [call], [tool], names) ?? mcpLabel(t, [call], names)),
+    [t, video, call, tool, names],
   );
   const exec = isCmuxExec(call);
-  const images = useMemo(() => toolImages(tool), [tool]);
+  const images = useMemo(() => (video ? [] : toolImages(tool)), [video, tool]);
   const running = isRunning(tool);
   // An MCP server's error is often JSON: its message, in one line (toolFailure.ts).
   const failure = useMemo(() => (tool.status === "failed" ? toolFailure(tool) : undefined), [tool]);
@@ -153,6 +161,7 @@ function McpCallRow({ item, grouped = false }: { item: AcpmuxActivity; grouped?:
         />
       </button>
       <ToolFailureCause failure={failure} />
+      {video && <VideoFramesCard video={video} />}
       {images.length > 0 && <ToolImages sources={images} />}
       {fold.present && (
         <div {...fold.bodyProps}>
@@ -163,7 +172,7 @@ function McpCallRow({ item, grouped = false }: { item: AcpmuxActivity; grouped?:
               id={detailsId}
               call={call}
               raw={raw}
-              output={tool.output}
+              output={video ? video.text : tool.output}
               tool={tool}
               strip={stripsEnvelopeImages(tool)}
               reveal={reveal}

webviews/src/agent-session/acpmux/conversation/VideoFramesCard.gallery.tsx

@@ -0,0 +1,149 @@
+// l10n-allow-file: gallery fixtures (a sample video path, its note and painted frames), not shipped UI.
+// A `video_frames` call, folded from the ACP updates acpmux sends (mergeToolItem, as the pane folds
+// them): Claude Code's `mcp__cmux__video_frames` with the video's path, then its completion, one
+// image block per frame and a text block naming the video's length, size and each frame's time.
+// The row draws a filmstrip card instead of the call's raw response (Leo, 2026-11-01: "i feel like
+// cmux-next should give u better tools to deal with this kind of thing"). `strip` is the
+// transcript's; `grid` (a contact sheet) and `poster` (the first frame large, the rest under it)
+// are its alternatives.
+import { useMemo } from "react";
+import { componentEntry } from "../../../gallery/format";
+import { mergeToolItem, textFromContent } from "../direct";
+import { ImageViewerContext } from "./imageViewerContext";
+import type { McpCallRows as McpCallRowsComponent } from "./McpCallRows";
+import type { VideoFramesLayout, VideoFramesLayoutContext as LayoutContext } from "./VideoFramesCard";
+
+type Props = { layout: VideoFramesLayout; frames: number; named: boolean };
+
+const path = "/Users/demo/Library/Application Support/cmux/Chat Uploads/screen-recording-sidebar.mp4";
+const durationMs = 16_400;
+
+/// A screen-recording-like frame: a window whose sidebar selection and progress move with time.
+function paint(index: number, count: number): string {
+  const [width, height] = [640, 360];
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const ink = canvas.getContext("2d")!;
+  const progress = count > 1 ? index / (count - 1) : 0;
+  ink.fillStyle = `hsl(${220 + progress * 40} 22% 95%)`;
+  ink.fillRect(0, 0, width, height);
+  ink.fillStyle = "hsl(220 14% 86%)";
+  ink.fillRect(0, 0, width, 28);
+  ink.fillRect(0, 0, 150, height);
+  ink.fillStyle = "hsl(220 60% 56%)";
+  ink.fillRect(8, 44 + Math.round(progress * 6) * 34, 134, 24);
+  ink.fillStyle = "hsl(220 12% 70%)";
+  for (let line = 0; line < 7; line += 1) ink.fillRect(176, 56 + line * 30, 180 + ((line * 61 + index * 23) % 240), 10);
+  ink.fillStyle = "hsl(150 45% 46%)";
+  ink.fillRect(176, height - 40, Math.max(12, (width - 200) * progress), 10);
+  return canvas.toDataURL("image/jpeg", 0.8);
+}
+
+function frameTime(ms: number): string {
+  const tenths = Math.round(ms / 100);
+  const m = Math.floor(tenths / 600);
+  const s = ((tenths % 600) / 10).toFixed(1).padStart(4, "0");
+  return `${m}:${s}`;
+}
+
+/// The call as the pane folds it from its ACP updates: the call, then its completion.
+function framesItem(count: number, named: boolean) {
+  const id = "gallery-video-frames";
+  const call = mergeToolItem(
+    undefined,
+    {
+      sessionUpdate: "tool_call",
+      toolCallId: id,
+      title: "mcp__cmux__video_frames",
+      kind: "other",
+      status: "in_progress",
+      rawInput: named ? { path, count } : { count },
+    },
+    id,
+    "",
+  );
+  const times = Array.from({ length: count }, (_, index) => Math.round((index * durationMs) / count));
+  // Named: the cmux tool's own note (cmux-tui video_tools.rs `summary`). Unnamed: another tool's.
+  const note = named
+    ? [
+        `${path}: ${(durationMs / 1000).toFixed(2)} s, 1920x1080, 30 fps, no audio. ${count} frames below, in order, at most 768 px on the longest side:`,
+        ...times.map((ms, index) => `#${index + 1} at ${(ms / 1000).toFixed(2)} s`),
+      ]
+    : [`Duration: ${frameTime(durationMs)}, 1920x1080`, ...times.map((ms, index) => `${index + 1}. ${frameTime(ms)}`)];
+  const text = note.join("\n");
+  const images = times.map((_, index) => {
+    const data = paint(index, count);
+    return {
+      type: "content",
+      content: { type: "image", mimeType: "image/jpeg", data: data.slice(data.indexOf(",") + 1) },
+    };
+  });
+  const content = [{ type: "content", content: { type: "text", text } }, ...images];
+  // The pane passes the update's text as the call's output (direct.ts), its images as its images.
+  return mergeToolItem(
+    call,
+    { sessionUpdate: "tool_call_update", toolCallId: id, status: "completed", content },
+    id,
+    textFromContent(content),
+  );
+}
+
+/// The row as the pane draws it.
+function galleryVideoFrames(McpCallRows: typeof McpCallRowsComponent, Layout: typeof LayoutContext) {
+  return function GalleryVideoFrames({ layout, frames, named }: Props) {
+    const item = useMemo(() => framesItem(frames, named), [frames, named]);
+    return (
+      <ImageViewerContext.Provider value={() => {}}>
+        <Layout.Provider value={layout}>
+          <div style={{ padding: 16 }}>
+            <McpCallRows items={[item]} disclosureKey="gallery:video-frames" />
+          </div>
+        </Layout.Provider>
+      </ImageViewerContext.Provider>
+    );
+  };
+}
+
+export default componentEntry<Props>({
+  id: "agent-pane.video-frames",
+  title: "Video frames an agent looked at",
+  area: "Agent pane",
+  pane: true,
+  height: 420,
+  covers: [
+    "agent-session/acpmux/conversation/VideoFramesCard.tsx#VideoFramesCard",
+    "agent-session/acpmux/conversation/VideoFramesCard.tsx#VideoFramesLayoutContext",
+  ],
+  load: async () => {
+    const [{ McpCallRows }, { VideoFramesLayoutContext }] = await Promise.all([
+      import("./McpCallRows"),
+      import("./VideoFramesCard"),
+    ]);
+    return galleryVideoFrames(McpCallRows, VideoFramesLayoutContext);
+  },
+  styles: () => Promise.all([import("../styles.css"), import("./conversation.css"), import("../chips/chips.css")]),
+  widths: { narrow: 420, normal: 720 },
+  variants: {
+    strip: {
+      note: "The transcript's: the video's name (plays it) and length, then a filmstrip of its frames, each with its time; a frame opens large.",
+      props: { layout: "strip", frames: 8, named: true },
+    },
+    grid: {
+      note: "Alternative: the frames as a contact sheet that wraps to the pane's width.",
+      props: { layout: "grid", frames: 8, named: true },
+    },
+    poster: {
+      note: "Alternative: the first frame large, the rest in a strip under it, the name below.",
+      props: { layout: "poster", frames: 8, named: true },
+    },
+    "strip-many": {
+      note: "Sixteen frames: the strip scrolls sideways at the row's width instead of growing tall.",
+      props: { layout: "strip", frames: 16, named: true },
+    },
+    "strip-unnamed": {
+      note: "A call that named no file: the card draws its length and frames, with no name to play.",
+      props: { layout: "strip", frames: 4, named: false },
+    },
+  },
+});

webviews/src/agent-session/acpmux/conversation/VideoFramesCard.tsx

@@ -0,0 +1,125 @@
+// The frames an agent looked at in a video (videoFrames.ts), under the tool row that read them: a
+// filmstrip of the frames, each with its time, and the video's name and length above it (Leo,
+// 2026-11-01: "i feel like cmux-next should give u better tools to deal with this kind of thing").
+// A frame opens large in the pane's image viewer; the name plays the video in Quick Look, the same
+// viewer an attached video's tile opens (`video.quickLook`, which the host checks like any path).
+// Every frame draws in a box of the video's shape before it decodes, so the transcript never moves.
+// `layout` keeps the gallery's alternatives (VideoFramesCard.gallery.tsx); the transcript draws `strip`.
+import { createContext, useContext, type CSSProperties } from "react";
+import { callChipHost } from "../chips/host";
+import { formatDuration } from "../chips/ReplyMedia";
+import { useT } from "../i18n";
+import { ImageViewerContext } from "./imageViewerContext";
+import { PlayTriangle } from "./icons";
+import { useTranscriptReadOnly } from "./readOnly";
+import { frameLabel, needsTenths, type VideoFrame, type VideoFrames } from "./videoFrames";
+
+export type VideoFramesLayout = "strip" | "grid" | "poster";
+
+/// The layout the transcript's cards take: `strip`, but for the gallery's comparison.
+export const VideoFramesLayoutContext = createContext<VideoFramesLayout>("strip");
+
+/// The shape a frame box takes: the video's, held between 9:16 and 21:9.
+function aspect(video: VideoFrames): number {
+  const ratio = video.width && video.height ? video.width / video.height : 16 / 9;
+  return Math.min(21 / 9, Math.max(9 / 16, ratio));
+}
+
+export function VideoFramesCard({ video }: { video: VideoFrames }) {
+  const t = useT();
+  const layout = useContext(VideoFramesLayoutContext);
+  const readOnly = useTranscriptReadOnly();
+  const openImage = useContext(ImageViewerContext);
+  const tenths = needsTenths(video.frames);
+  const length = video.durationMs ? formatDuration(video.durationMs / 1000) : undefined;
+  const name = video.name ?? "";
+  const playable = Boolean(video.path) && !readOnly;
+  const style = { "--cv-video-frame-aspect": String(aspect(video)) } as CSSProperties;
+  const time = (frame: VideoFrame) => (frame.atMs === undefined ? undefined : frameLabel(frame.atMs, tenths));
+  const alt = (frame: VideoFrame) => {
+    const at = time(frame);
+    return at && name ? t("media.videoFrameAt", { name, time: at }) : name;
+  };
+  const frameButton = (frame: VideoFrame, index: number, lead = false) => {
+    const at = time(frame);
+    const image = <img src={frame.src} alt={alt(frame)} loading="lazy" decoding="async" draggable={false} />;
+    const badge = at && (
+      <span className="cv-video-frames__time" aria-hidden="true">
+        {at}
+      </span>
+    );
+    const className = `cv-video-frames__frame${lead ? " is-lead" : ""}`;
+    if (!openImage)
+      return (
+        <span key={index} className={className}>
+          {image}
+          {badge}
+        </span>
+      );
+    return (
+      <button
+        key={index}
+        type="button"
+        className={className}
+        data-open-image
+        aria-label={alt(frame) || t("image.view")}
+        onClick={() => openImage(frame.src, alt(frame))}
+      >
+        {image}
+        {badge}
+      </button>
+    );
+  };
+  const head = (
+    <div className="cv-video-frames__head">
+      {playable ? (
+        <button
+          type="button"
+          className="cv-video-frames__name is-playable"
+          /* tooltip-allow: the full path behind the shown file name */ title={video.path}
+          aria-label={length ? t("composer.playVideo", { name, length }) : t("media.play")}
+          onClick={() => void callChipHost("video.quickLook", { src: video.path! })}
+        >
+          <PlayTriangle size={10} className="cv-video-frames__play" />
+          <span className="cv-video-frames__label">{name}</span>
+        </button>
+      ) : (
+        name && (
+          <span
+            className="cv-video-frames__name"
+            /* tooltip-allow: the full path behind the shown file name */ title={video.path}
+          >
+            <span className="cv-video-frames__label">{name}</span>
+          </span>
+        )
+      )}
+      {(length || (video.width && video.height)) && (
+        <span className="cv-video-frames__meta">
+          {length}
+          {length && video.width && video.height ? " · " : ""}
+          {video.width && video.height ? `${video.width}×${video.height}` : ""}
+        </span>
+      )}
+    </div>
+  );
+  if (layout === "poster") {
+    const [lead, ...rest] = video.frames;
+    return (
+      <div className="cv-video-frames is-poster" style={style}>
+        <div className="cv-video-frames__poster">{lead && frameButton(lead, 0, true)}</div>
+        {rest.length > 0 && (
+          <div className="cv-video-frames__strip">{rest.map((frame, index) => frameButton(frame, index + 1))}</div>
+        )}
+        {head}
+      </div>
+    );
+  }
+  return (
+    <div className={`cv-video-frames is-${layout}`} style={style}>
+      {head}
+      <div className={layout === "grid" ? "cv-video-frames__grid" : "cv-video-frames__strip"}>
+        {video.frames.map((frame, index) => frameButton(frame, index))}
+      </div>
+    </div>
+  );
+}