feat(videos): add Grok video bridge for non-OpenAI models

docs-site/src/content/docs/guides/video-bridge.md

@@ -0,0 +1,85 @@
+---
+title: Video Bridge
+description: Generate videos with Grok Imagine Video through a non-OpenAI model.
+---
+
+## Overview
+
+The Video Bridge lets you use xAI's Grok Imagine Video generation through any non-OpenAI model
+routed by opencodex. When enabled, a synthetic `video_gen` tool is injected into the conversation.
+The model calls it like any function tool; opencodex intercepts the call, submits a video generation
+job to xAI, polls until completion, and downloads the result.
+
+## Prerequisites
+
+- An `xai` provider entry with an **API key** (`ocx login xai` alone is not sufficient — the video bridge requires key auth, not OAuth)
+- A non-OpenAI model as your routed provider (e.g. Anthropic Claude, Google Gemini)
+- opencodex configured to route through the non-OpenAI provider
+
+> **⚠ Provider key required:** The video bridge only activates when the `xai` provider uses
+> API key auth. Add this to your config:
+>
+> ```json
+> {
+>   "providers": {
+>     "xai": { "adapter": "openai-chat", "apiKey": "xai-…", "authMode": "key" }
+>   }
+> }
+> ```
+>
+> If you onboarded via `ocx login xai` (OAuth), the provider stays in `authMode: "oauth"`
+> and the bridge silently won't activate. Set `XAI_API_KEY` in the environment **or**
+> hard-code the key as shown above.
+
+## Configuration
+
+Add `videoBridgeEnabled: true` to your `images` config:
+
+```json
+{
+  "images": {
+    "bridgeEnabled": true,
+    "videoBridgeEnabled": true,
+    "videoBridgeModel": "grok-imagine-video",
+    "videoMaxRounds": 2,
+    "videoTimeoutMs": 300000
+  }
+}
+```
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `videoBridgeEnabled` | `false` | Master switch. Must be explicitly enabled. |
+| `videoBridgeModel` | `"grok-imagine-video"` | xAI video model id. |
+| `videoMaxRounds` | `2` | Max video-gen rounds before forced final answer. |
+| `videoTimeoutMs` | `300000` (5 min) | Per-video timeout including polling. |
+
+## How It Works
+
+1. opencodex detects a non-OpenAI routed model with `videoBridgeEnabled: true`
+2. A synthetic `video_gen` function tool is injected into the conversation
+3. When the model calls `video_gen`, opencodex submits a job to xAI's `/videos/generations`
+4. The bridge polls the job status every 5-15 seconds, sending heartbeat messages to keep the stream alive
+5. When the video is ready, it's downloaded to the artifacts directory
+6. The local file path is returned to the model as a tool result
+
+## Supported Parameters
+
+The `video_gen` tool accepts:
+
+| Parameter | Type | Range | Description |
+|-----------|------|-------|-------------|
+| `prompt` | string | required | Detailed video generation prompt |
+| `duration` | integer | 1-15 | Video length in seconds |
+| `resolution` | string | `"480p"`, `"720p"` | Video resolution |
+| `aspect_ratio` | string | 7 ratios | `16:9`, `9:16`, `1:1`, `4:3`, `3:4`, `3:2`, `2:3` |
+
+## Limitations
+
+- **xAI only**: Video generation is only available through xAI's Grok Imagine Video API
+- **Asynchronous**: Video generation takes 30-120 seconds
+- **Cost**: Video generation is a paid xAI feature (~$0.05/sec @480p, ~$0.07/sec @720p)
+- **One video per call**: Each `video_gen` call produces one video
+- **Coexists with Image Bridge**: Both bridges can be enabled simultaneously
+- **Web search priority**: When a web search sidecar is active for a turn (non-`runTurn` adapter), the video bridge is skipped — the two cannot run concurrently. A `console.warn` is emitted so you can detect this in logs.
+- **Timeout covers submit + poll**: The `videoTimeoutMs` budget starts before job submission, so the submit call (60 s) and subsequent polling share the same deadline.

src/images/artifacts.ts

@@ -1,5 +1,5 @@
 import { readdirSync, readFileSync, statSync, unlinkSync, existsSync } from "node:fs";
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, writeFile, open, unlink } from "node:fs/promises";
 import type { IncomingMessage } from "node:http";
 import https from "node:https";
 import type { RequestOptions } from "node:https";
@@ -475,3 +475,147 @@ export async function downloadImageToArtifact(
   // Retention is post-batch via pruneArtifacts (see fulfill.ts).
   return writeArtifactUnique(dir, "dl-", bytes, ext);
 }
+
+const MAX_VIDEO_DOWNLOAD_BYTES = 200 * 1024 * 1024; // 200 MiB
+/** Aggregate per-turn video download cap (600 MiB = 3 × max single download). */
+const MAX_VIDEO_BYTES_PER_TURN = MAX_VIDEO_DOWNLOAD_BYTES * 3;
+
+export interface VideoBudget {
+  spent: number;
+  /** Ceiling on total bytes across all downloads this turn. */
+  cap: number;
+}
+
+export function createVideoBudget(): VideoBudget {
+  return { spent: 0, cap: MAX_VIDEO_BYTES_PER_TURN };
+}
+
+/** Charge bytes to the budget; returns false if the ceiling would be exceeded. */
+export function chargeVideoBudget(budget: VideoBudget, bytes: number): boolean {
+  if (budget.spent + bytes > budget.cap) return false;
+  budget.spent += bytes;
+  return true;
+}
+
+export function guessVideoExtFromMagic(bytes: Uint8Array): string {
+  if (bytes.byteLength < 12) throw new Error("video data too short for magic byte sniffing");
+  const sig = Buffer.from(bytes.slice(0, 12)).toString("latin1");
+  // MP4/QuickTime/MOV: bytes 4-7 == "ftyp" (ISO BMFF)
+  if (sig.slice(4, 8) === "ftyp") return "mp4";
+  // WebM/Matroska: \x1a\x45\xdf\xa3
+  if (sig.startsWith("\x1a\x45\xdf\xa3")) return "webm";
+  throw new Error("unrecognized video format — magic bytes do not match MP4 or WebM");
+}
+
+/**
+ * Download a video from a URL to an artifact file with a 200 MiB hard cap, streaming the body
+ * to disk to avoid buffering. SSRF protection reuses the same destination policy + pinned HTTPS
+ * as image downloads. Format is sniffed from magic bytes.
+ */
+export async function downloadVideoToArtifact(
+  url: string,
+  budget?: VideoBudget,
+  signal?: AbortSignal,
+): Promise<string> {
+  // For data: URLs, handle inline (unlikely for video but keep parity)
+  if (url.startsWith("data:")) {
+    const commaIdx = url.indexOf(",");
+    const meta = url.slice(0, commaIdx);
+    const data = url.slice(commaIdx + 1);
+    const isBase64 = meta.includes(";base64");
+    if (!isBase64) throw new Error("non-base64 data URI for video is not supported");
+    const buf = Buffer.from(data, "base64");
+    if (budget && !chargeVideoBudget(budget, buf.byteLength)) {
+      throw new Error("video data URI exceeds per-turn download budget");
+    }
+    if (buf.byteLength > MAX_VIDEO_DOWNLOAD_BYTES) throw new Error("video data URI exceeds size cap");
+    const ext = guessVideoExtFromMagic(buf);
+    const dir = getArtifactsDir();
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    const name = `vid-${timestampPrefix()}-${crypto.randomUUID()}.${ext}`;
+    const dest = join(dir, name);
+    await writeFile(dest, buf, { mode: 0o600 });
+    return dest;
+  }
+
+  // SSRF protection: same validation as downloadImageToArtifact
+  let parsedUrl: URL;
+  try { parsedUrl = new URL(url); } catch { throw new Error("video URL is not valid"); }
+  if (parsedUrl.protocol !== "https:") {
+    throw new Error(`video URL must use HTTPS, got ${parsedUrl.protocol}`);
+  }
+  const assessment = assessUrlDestination(url);
+  if (assessment && assessment.kind !== "public" && assessment.kind !== "hostname") {
+    throw new Error(`video URL targets ${assessment.detail}`);
+  }
+  const resolved = await resolvePublicAddresses(url);
+  const pinned = pickPinnedAddress(resolved.addresses);
+  const resp = await pinnedHttpsGet(url, pinned, signal, { maxBytes: MAX_VIDEO_DOWNLOAD_BYTES });
+  if (!resp.ok) {
+    try { await resp.body?.cancel(); } catch { /* ignore */ }
+    throw new Error("video download failed: " + resp.status);
+  }
+
+  const dir = getArtifactsDir();
+  await mkdir(dir, { recursive: true, mode: 0o700 });
+
+  const reader = resp.body?.getReader();
+  if (!reader) throw new Error("video download returned no body");
+
+  // Buffer at least 12 bytes for magic-byte sniffing before opening the file.
+  // A single read() can return fewer bytes; accumulate until we have enough.
+  let dest: string | undefined;
+  let fh: { close(): Promise<void>; writeFile(data: Uint8Array): Promise<void> } | undefined;
+  try {
+    const sniffChunks: Uint8Array[] = [];
+    let sniffLen = 0;
+    while (sniffLen < 12) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      if (!value) continue;
+      sniffChunks.push(value);
+      sniffLen += value.byteLength;
+    }
+    if (sniffLen === 0) {
+      throw new Error("video download returned empty body");
+    }
+    const sniffBuf = Buffer.concat(sniffChunks);
+    const ext = guessVideoExtFromMagic(new Uint8Array(sniffBuf));
+    const name = `vid-${timestampPrefix()}-${crypto.randomUUID()}.${ext}`;
+    dest = join(dir, name);
+    fh = await open(dest, "w", 0o600);
+    // Write all buffered chunks and set up accounting
+    let totalBytes = sniffBuf.byteLength;
+    if (budget && !chargeVideoBudget(budget, totalBytes)) {
+      throw new Error("video download exceeds per-turn budget");
+    }
+    if (totalBytes > MAX_VIDEO_DOWNLOAD_BYTES) {
+      throw new Error("video download exceeds size cap");
+    }
+    await fh.writeFile(new Uint8Array(sniffBuf));
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      totalBytes += value.byteLength;
+      if (budget && !chargeVideoBudget(budget, value.byteLength)) {
+        throw new Error("video download exceeds per-turn budget");
+      }
+      if (totalBytes > MAX_VIDEO_DOWNLOAD_BYTES) {
+        throw new Error("video download exceeds size cap");
+      }
+      await fh.writeFile(value);
+    }
+    await fh.close();
+    try { await reader.cancel(); } catch { /* ignore */ }
+    reader.releaseLock();
+    return dest;
+  } catch (err) {
+    try { await reader.cancel(); } catch { /* ignore */ }
+    reader.releaseLock();
+    if (fh) {
+      try { await fh.close(); } catch { /* ignore */ }
+    }
+    if (dest) await unlink(dest).catch(() => {});
+    throw err;
+  }
+}

src/images/fulfill-video.ts

@@ -0,0 +1,153 @@
+import { pathToFileURL } from "node:url";
+import type { VideoBridgePlan, VideoCallResult } from "./types";
+import { submitVideoJob, pollVideoJob } from "./xai-video-client";
+import { downloadVideoToArtifact, createVideoBudget, type VideoBudget } from "./artifacts";
+
+/** Parsed arguments from the model's video_gen tool call. */
+export interface ParsedVideoArgs {
+  ok: true;
+  prompt: string;
+  duration?: number;
+  resolution?: string;
+  aspectRatio?: string;
+}
+
+const INITIAL_POLL_INTERVAL_MS = 5_000;
+const MAX_POLL_INTERVAL_MS = 15_000;
+const POLL_BACKOFF = 1.5;
+const DEFAULT_VIDEO_TIMEOUT_MS = 300_000; // 5 min
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.reject(new Error("aborted"));
+  return new Promise((resolve, reject) => {
+    const onAbort = (): void => { clearTimeout(timer); reject(new Error("aborted")); };
+    const timer = setTimeout(() => { signal?.removeEventListener("abort", onAbort); resolve(); }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+/**
+ * Parse the JSON arguments string from a video_gen tool call. Mirrors fulfillImageCall's
+ * defensive parsing: invalid JSON or missing prompt returns an error result.
+ */
+export function parseVideoCallArgs(raw: string):
+  | ParsedVideoArgs
+  | { ok: false; error: string } {
+  let args: unknown;
+  try {
+    args = JSON.parse(raw || "{}");
+  } catch {
+    return { ok: false, error: "invalid arguments JSON" };
+  }
+  if (typeof args !== "object" || args === null) {
+    return { ok: false, error: "invalid arguments JSON" };
+  }
+  const obj = args as Record<string, unknown>;
+
+  const prompt =
+    typeof obj.prompt === "string" ? obj.prompt : typeof obj.input === "string" ? obj.input : "";
+  if (!prompt) {
+    return { ok: false, error: "missing prompt" };
+  }
+
+  const result: ParsedVideoArgs = { ok: true, prompt };
+
+  // duration: clamp to 1-15
+  if (typeof obj.duration === "number") {
+    result.duration = Math.max(1, Math.min(15, Math.floor(obj.duration)));
+  }
+
+  if (typeof obj.resolution === "string" && (obj.resolution === "480p" || obj.resolution === "720p")) {
+    result.resolution = obj.resolution;
+  }
+  if (typeof obj.aspect_ratio === "string") {
+    const VALID_RATIOS = ["16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3"];
+    if (VALID_RATIOS.includes(obj.aspect_ratio)) result.aspectRatio = obj.aspect_ratio;
+  }
+
+  return result;
+}
+
+/**
+ * Poll a video generation job, yielding heartbeats with elapsed-time messages so the
+ * caller can forward them to the SSE stream. Returns the final result when the job
+ * completes or fails/times out.
+ */
+export async function* pollVideoWithHeartbeats(
+  requestId: string,
+  auth: { baseUrl: string; token: string },
+  signal: AbortSignal,
+  timeoutMs: number = DEFAULT_VIDEO_TIMEOUT_MS,
+): AsyncGenerator<
+  { type: "heartbeat"; message: string },
+  { ok: true; videoUrl: string } | { ok: false; error: string }
+> {
+  const start = Date.now();
+  let interval = INITIAL_POLL_INTERVAL_MS;
+
+  for (;;) {
+    if (Date.now() - start >= timeoutMs) {
+      return { ok: false, error: `video generation timed out after ${Math.floor(timeoutMs / 1000)}s` };
+    }
+
+    try {
+      const poll = await pollVideoJob(requestId, auth, signal);
+      if (poll.status === "done") {
+        if (poll.videoUrl) {
+          return { ok: true, videoUrl: poll.videoUrl };
+        }
+        return { ok: false, error: "video generation completed but no video URL was returned" };
+      }
+      if (poll.status === "failed") {
+        return { ok: false, error: "video generation failed" };
+      }
+      if (poll.status === "expired") {
+        return { ok: false, error: "video generation expired" };
+      }
+      // still processing — continue with backoff
+    } catch (e) {
+      if (signal.aborted) {
+        return { ok: false, error: Date.now() - start >= timeoutMs * 0.95
+          ? `video generation timed out after ${Math.floor(timeoutMs / 1000)}s`
+          : "client closed request during video generation" };
+      }
+      const status = (e as Error & { status?: number }).status;
+      if (status && status >= 400 && status < 500 && status !== 429) {
+        return { ok: false, error: `video poll failed permanently: HTTP ${status}` };
+      }
+      console.warn(`[videos] poll error (will retry): ${e instanceof Error ? e.message : String(e)}`);
+    }
+
+    const elapsed = Math.floor((Date.now() - start) / 1000);
+    yield { type: "heartbeat", message: `Generating video... ${elapsed}s` };
+
+    try {
+      await sleep(interval, signal);
+    } catch {
+      if (Date.now() - start >= timeoutMs * 0.95) {
+        return { ok: false, error: `video generation timed out after ${Math.floor(timeoutMs / 1000)}s` };
+      }
+      return { ok: false, error: "client closed request during video generation" };
+    }
+
+    interval = Math.min(MAX_POLL_INTERVAL_MS, Math.floor(interval * POLL_BACKOFF));
+  }
+}
+
+/**
+ * Build a success VideoCallResult after the video has been downloaded to disk.
+ */
+export function buildVideoResult(path: string, prompt: string, model: string): VideoCallResult {
+  return {
+    ok: true,
+    model,
+    prompt,
+    path,
+    files: [path],
+    count: 1,
+    markdown: `[video](${pathToFileURL(path).href})`,
+  };
+}
+
+export type { VideoBudget };
+export { createVideoBudget };