feat: hand off video turns to local watch-skill

server/contracts.ts

@@ -109,6 +109,8 @@ export interface SendTurnInput {
      * through the harness so this bot can message other bots. The harness
      * owns turns, permissions, and recursion limits; the proxy only forwards. */
     agents?: { command: string; args: string[]; env: Record<string, string> };
+    /** Local watch-skill MCP server, added automatically for a video turn. */
+    watchSkill?: { command: string; args: string[]; env: Record<string, string> };
   };
   cwd?: string;
 }

server/drivers/acp/acp.test.ts

@@ -108,6 +108,23 @@ posixOnly("ACP turns (fake CLI)", () => {
     expect(seen.env.XAI_API_KEY).toBeUndefined();
   });
 
+  it("forwards watch-skill environment values in ACP MCP format", async () => {
+    await create();
+    const dump = join(scratch, "watch-skill.json");
+    process.env.FAKE_ACP_DUMP = dump;
+
+    await instance.adapter.sendTurn({
+      threadId: "t-watch-skill-env",
+      text: "go",
+      integrations: { watchSkill: { command: "watch-skill", args: ["serve"], env: { WATCH_SKILL_HOME: "/tmp/index" } } },
+    });
+    await recorder.until((e) => e.type === "turn.completed");
+
+    const seen = JSON.parse(readFileSync(dump, "utf8"));
+    expect(seen.mcpServers.find((server: { name: string }) => server.name === "watch_skill").env)
+      .toEqual([{ name: "WATCH_SKILL_HOME", value: "/tmp/index" }]);
+  });
+
   it("surfaces a permission ask as request.opened and completes once allowed", async () => {
     await create(GrokAgentDriver, "permission");
     await instance.adapter.sendTurn({ threadId: "t-perm", text: "go" });

server/drivers/acp/core.ts

@@ -140,6 +140,15 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver<AcpConfig>
             env: Object.entries(agents.env).map(([name, value]) => ({ name, value: String(value) })),
           });
         }
+        const watchSkill = turn.integrations?.watchSkill;
+        if (watchSkill) {
+          servers.push({
+            name: "watch_skill",
+            command: watchSkill.command,
+            args: watchSkill.args,
+            env: Object.entries(watchSkill.env).map(([name, value]) => ({ name, value: String(value) })),
+          });
+        }
         return servers;
       };
 

server/drivers/claude.ts

@@ -278,6 +278,7 @@ export const ClaudeDriver: ProviderDriver<ClaudeConfig> = {
         mcpServers.agents = { ...turn.integrations.agents };
         allowed.push("mcp__agents");
       }
+      if (turn.integrations?.watchSkill) { mcpServers.watch_skill = { ...turn.integrations.watchSkill }; allowed.push("mcp__watch_skill"); }
       // permission broker: anything acceptEdits would silently deny becomes
       // an Allow/Deny card in chat, and the agent gets ask_user. Skipped in
       // bypassPermissions (fullAuto) — nothing would ever ask.

server/index.ts

@@ -17,6 +17,7 @@ import { BUILT_IN_DRIVERS } from "./drivers/builtIn.ts";
 import { EventBus } from "./harness/bus.ts";
 import { ProviderRegistry } from "./harness/registry.ts";
 import { mentionedBots, Store, type Message } from "./store.ts";
+import { hasWatchSkill, videoReferences, watchSkillIntegration } from "./watch-skill.ts";
 
 const PORT = Number(process.env.OMB_PORT || process.env.OGB_PORT || 8799);
 const STATIC_DIR = process.env.OMB_STATIC_DIR || null;
@@ -475,6 +476,14 @@ async function startTurn(
       ) {
         integrations.agents = agentsIntegration(bot.id, commsDepth);
       }
+      const videos = videoReferences(text);
+      const activeVideo = videos.at(-1) ?? bot.watchSkillVideo;
+      // The detected source is persisted with the thread's bot record. The
+      // local watch-skill server owns its durable index, so a later question
+      // with no URL can still mount the same tool and reuse that index after a
+      // harness restart.
+      if (videos.length) store.patchBot(bot.id, { watchSkillVideo: activeVideo });
+      if (activeVideo && await hasWatchSkill()) integrations.watchSkill = watchSkillIntegration();
       // @mentions in the user's message (the composer's tagging UI) become
       // an explicit delegation nudge — the agent still does the ask_bot call
       // itself, so the harness stays the single owner of turns/permissions
@@ -502,6 +511,13 @@ async function startTurn(
           (integrations.agents
             ? " You can work with the user's other bots through the agents tools — list_bots shows who's available, ask_bot sends one of them a message and returns their reply."
             : "") +
+          (integrations.watchSkill
+            ? videos.length === 1
+              ? " The user included a video reference. Use the watch_skill tools to index it once, cite timestamped evidence, and reuse its index for follow-up questions."
+              : videos.length > 1
+                ? " The user included multiple video references. Use the watch_skill tools to index each referenced video once, cite timestamped evidence, and reuse each corresponding index for follow-up questions."
+                : " Continue using the active video's existing watch_skill index; cite timestamped evidence when relevant."
+            : "") +
           (tagged.length
             ? ` The user tagged ${tagged
                 .map((t) => `@${t.name} (ask_bot bot_id ${t.id})`)

server/store.ts

@@ -96,6 +96,8 @@ export interface BotRecord {
   /** which computer the bot acts on: its cloud box, this Mac (local CUA),
    * or none. Unset = auto (box when it exists, else local when available). */
   computer?: "cloud" | "local" | "off";
+  /** Most recently referenced video for persistent watch-skill follow-ups. */
+  watchSkillVideo?: string;
   /** true after an edit/branch-switch rewound the visible conversation:
    * provider sessions still hold the abandoned branch, so the next turn
    * must start fresh (drop cursors) and replay the surviving path. */

server/testing/fake-acp-cli.ts

@@ -135,6 +135,9 @@ function handle(msg: any) {
       break;
     case "session/new": {
       const servers: McpEntry[] = Array.isArray(msg.params?.mcpServers) ? msg.params.mcpServers : [];
+      if (process.env.FAKE_ACP_DUMP) {
+        writeFileSync(process.env.FAKE_ACP_DUMP, JSON.stringify({ argv, env: process.env, mcpServers: servers }, null, 2));
+      }
       agentsMcp = servers.find((s: any) => s?.name === "agents") ?? null;
       result(msg.id, { sessionId: "fake-acp-session" });
       break;

server/watch-skill.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+import { videoReferences } from "./watch-skill.ts";
+describe("watch-skill handoff", () => {
+  it("recognizes supported video URLs but ignores normal links", () => expect(videoReferences("see https://youtu.be/demo https://example.com/a https://cdn.x/demo.webm")).toEqual(["https://youtu.be/demo", "https://cdn.x/demo.webm"]));
+  it("does not treat prose as a video reference", () => expect(videoReferences("please explain this recording")).toEqual([]));
+
+  it("accepts provider video paths but rejects ordinary supported-host pages", () => {
+    expect(videoReferences("https://youtube.com/channel/example https://vimeo.com/12345 https://twitch.tv/videos/77 https://loom.com/share/abc"))
+      .toEqual(["https://vimeo.com/12345", "https://twitch.tv/videos/77", "https://loom.com/share/abc"]);
+  });
+
+  it("keeps direct video queries while removing surrounding prose punctuation", () => {
+    expect(videoReferences("Watch (https://cdn.example/demo.webm?download=1), then reply."))
+      .toEqual(["https://cdn.example/demo.webm?download=1"]);
+  });
+});

server/watch-skill.ts

@@ -0,0 +1,40 @@
+import { execFile } from "node:child_process";
+
+const VIDEO_URL = /https?:\/\/[^\s<>()]+/gi;
+const VIDEO_FILE = /\.(mp4|mov|mkv|webm|avi|m4v)(?:[?#].*)?$/i;
+let availability: Promise<boolean> | undefined;
+
+function isHostedVideo(url: URL) {
+  const host = url.hostname.toLowerCase();
+  const path = url.pathname.replace(/\/+$/, "");
+  if (host === "youtu.be") return path.length > 1;
+  if (host.endsWith("youtube.com")) return (path === "/watch" && Boolean(url.searchParams.get("v"))) || /^\/(shorts|live|embed)\/[^/]+$/i.test(path);
+  if (host.endsWith("vimeo.com")) return /^\/(video\/)?\d+$/i.test(path);
+  if (host === "clips.twitch.tv") return path.length > 1;
+  if (host.endsWith("twitch.tv")) return /^\/videos\/\d+$/i.test(path) || /^\/[^/]+\/clip\/[^/]+$/i.test(path);
+  if (host.endsWith("loom.com")) return /^\/(share|embed)\/[^/]+$/i.test(path);
+  return false;
+}
+
+/** Conservative detection: ordinary links do not start a video workflow. */
+export function videoReferences(text: string): string[] {
+  return (text.match(VIDEO_URL) ?? []).flatMap((raw) => {
+    const candidate = raw.replace(/[.,;:'"\]\)]+$/, "");
+    try {
+      const url = new URL(candidate);
+      return isHostedVideo(url) || VIDEO_FILE.test(url.pathname) ? [candidate] : [];
+    } catch { return []; }
+  });
+}
+
+/** No install/fetch side effect. Users opt in by installing watch-skill locally. */
+export function hasWatchSkill(command = "watch-skill"): Promise<boolean> {
+  if (command === "watch-skill" && availability) return availability;
+  const probe = new Promise<boolean>((resolve) => {
+    execFile(command, ["--version"], { windowsHide: true, timeout: 2_000 }, (error) => resolve(!error));
+  });
+  if (command === "watch-skill") availability = probe;
+  return probe;
+}
+
+export function watchSkillIntegration(command = "watch-skill") { return { command, args: ["serve"], env: {} as Record<string, string> }; }