fix(engine): auto-normalize VFR video inputs to CFR before frame extraction

packages/engine/src/services/videoFrameExtractor.test.ts

@@ -1,5 +1,15 @@
-import { describe, expect, it } from "vitest";
-import { parseVideoElements, parseImageElements } from "./videoFrameExtractor.js";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { spawnSync } from "node:child_process";
+import {
+  parseVideoElements,
+  parseImageElements,
+  extractAllVideoFrames,
+  type VideoElement,
+} from "./videoFrameExtractor.js";
+import { extractVideoMetadata } from "../utils/ffprobe.js";
 
 describe("parseVideoElements", () => {
   it("parses videos without an id or data-start attribute", () => {
@@ -81,3 +91,119 @@ describe("parseImageElements", () => {
     expect(images[0]!.end).toBe(5);
   });
 });
+
+// Regression test for the VFR (variable frame rate) freeze bug.
+// Screen recordings and phone videos often have irregular timestamps.
+// When such inputs hit `extractVideoFramesRange`'s `-ss <start> -i ... -t <dur>
+// -vf fps=N` pipeline, the fps filter can emit fewer frames than requested —
+// e.g. a 4-second segment at 30fps would produce ~90 frames instead of 120.
+// FrameLookupTable.getFrameAtTime then returns null for out-of-range indices
+// and the compositor holds the last valid frame, which the user perceives as
+// the video freezing. extractAllVideoFrames normalizes VFR sources to CFR
+// before extraction to fix this.
+const FIXTURE_DIR = mkdtempSync(join(tmpdir(), "hf-vfr-test-"));
+const VFR_FIXTURE = join(FIXTURE_DIR, "vfr_screen.mp4");
+
+function runFfmpegSync(args: string[]): void {
+  const r = spawnSync("ffmpeg", args, { encoding: "utf8" });
+  if (r.status !== 0) {
+    throw new Error(`ffmpeg failed (${r.status}): ${r.stderr.slice(-400)}`);
+  }
+}
+
+beforeAll(() => {
+  // 10s testsrc2 at 60fps, ~40% of frames dropped via select filter and
+  // encoded with -vsync vfr so timestamps are irregular. Declared fps 60,
+  // actual average ~36 — well over the 10% threshold used by isVFR.
+  // The select expression drops four 1-second windows (frames 30-89,
+  // 180-239, 330-389, 480-539) to simulate static segments in a screen
+  // recording where no pixels changed.
+  runFfmpegSync([
+    "-y",
+    "-hide_banner",
+    "-loglevel",
+    "error",
+    "-f",
+    "lavfi",
+    "-i",
+    "testsrc2=s=320x180:d=10:rate=60",
+    "-vf",
+    "drawtext=text='n=%{n}':fontsize=24:fontcolor=white:x=10:y=10:box=1:boxcolor=black@0.6," +
+      "select='not(between(n,30,89))*not(between(n,180,239))*not(between(n,330,389))*not(between(n,480,539))'",
+    "-vsync",
+    "vfr",
+    "-c:v",
+    "libx264",
+    "-preset",
+    "ultrafast",
+    "-pix_fmt",
+    "yuv420p",
+    "-g",
+    "600",
+    "-keyint_min",
+    "600",
+    VFR_FIXTURE,
+  ]);
+}, 30_000);
+
+afterAll(() => {
+  if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true });
+});
+
+describe("extractAllVideoFrames on a VFR source", () => {
+  it("detects the synthesized fixture as VFR", async () => {
+    const md = await extractVideoMetadata(VFR_FIXTURE);
+    expect(md.isVFR).toBe(true);
+  });
+
+  it("produces the expected frame count for a mid-file segment", async () => {
+    const outputDir = join(FIXTURE_DIR, "out-mid-segment");
+    mkdirSync(outputDir, { recursive: true });
+
+    const video: VideoElement = {
+      id: "v1",
+      src: VFR_FIXTURE,
+      start: 0,
+      end: 4,
+      mediaStart: 3,
+      hasAudio: false,
+    };
+
+    const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
+      fps: 30,
+      outputDir,
+    });
+
+    expect(result.errors).toEqual([]);
+    expect(result.extracted).toHaveLength(1);
+    const frames = readdirSync(join(outputDir, "v1")).filter((f) => f.endsWith(".jpg"));
+    // Expect 120 frames for 4s at 30fps. Allow ±1 for rounding at the
+    // boundary; pre-fix behavior produced ~90 frames (a 25% shortfall).
+    expect(frames.length).toBeGreaterThanOrEqual(119);
+    expect(frames.length).toBeLessThanOrEqual(121);
+  }, 60_000);
+
+  it("produces the expected frame count when mediaStart=0 and the full VFR file is used", async () => {
+    const outputDir = join(FIXTURE_DIR, "out-full");
+    mkdirSync(outputDir, { recursive: true });
+
+    const video: VideoElement = {
+      id: "vfull",
+      src: VFR_FIXTURE,
+      start: 0,
+      end: 10,
+      mediaStart: 0,
+      hasAudio: false,
+    };
+
+    const result = await extractAllVideoFrames([video], FIXTURE_DIR, {
+      fps: 30,
+      outputDir,
+    });
+
+    expect(result.errors).toEqual([]);
+    const frames = readdirSync(join(outputDir, "vfull")).filter((f) => f.endsWith(".jpg"));
+    expect(frames.length).toBeGreaterThanOrEqual(299);
+    expect(frames.length).toBeLessThanOrEqual(301);
+  }, 60_000);
+});

packages/engine/src/services/videoFrameExtractor.ts

@@ -296,6 +296,62 @@ async function convertSdrToHdr(
   }
 }
 
+/**
+ * Re-encode a VFR (variable frame rate) video segment to CFR so the downstream
+ * fps filter can extract frames reliably. Screen recordings, phone videos, and
+ * some webcams emit irregular timestamps that cause two failure modes:
+ *   1. Output has fewer frames than expected (e.g. -ss 3 -t 4 produces 90
+ *      frames instead of 120 @ 30fps). FrameLookupTable.getFrameAtTime then
+ *      returns null for late timestamps and the caller freezes on the last
+ *      valid frame.
+ *   2. Large duplicate-frame runs where source PTS don't land on target
+ *      timestamps.
+ *
+ * Only the [startTime, startTime+duration] window is re-encoded, so long
+ * recordings aren't fully transcoded when only a short clip is used.
+ */
+async function convertVfrToCfr(
+  inputPath: string,
+  outputPath: string,
+  targetFps: number,
+  startTime: number,
+  duration: number,
+  signal?: AbortSignal,
+  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
+): Promise<void> {
+  const timeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
+
+  const args = [
+    "-ss",
+    String(startTime),
+    "-i",
+    inputPath,
+    "-t",
+    String(duration),
+    "-fps_mode",
+    "cfr",
+    "-r",
+    String(targetFps),
+    "-c:v",
+    "libx264",
+    "-preset",
+    "fast",
+    "-crf",
+    "18",
+    "-c:a",
+    "copy",
+    "-y",
+    outputPath,
+  ];
+
+  const result = await runFfmpeg(args, { signal, timeout });
+  if (!result.success) {
+    throw new Error(
+      `VFR→CFR conversion failed (exit ${result.exitCode}): ${result.stderr.slice(-300)}`,
+    );
+  }
+}
+
 export async function extractAllVideoFrames(
   videos: VideoElement[],
   baseDir: string,
@@ -371,6 +427,47 @@ export async function extractAllVideoFrames(
     }
   }
 
+  // Phase 2b: Re-encode VFR inputs to CFR so the fps filter in Phase 3 produces
+  // the expected frame count. Only the used segment is transcoded.
+  const vfrNormDir = join(options.outputDir, "_vfr_normalized");
+  for (let i = 0; i < resolvedVideos.length; i++) {
+    if (signal?.aborted) break;
+    const entry = resolvedVideos[i];
+    if (!entry) continue;
+    const metadata = await extractVideoMetadata(entry.videoPath);
+    if (!metadata.isVFR) continue;
+
+    let segDuration = entry.video.end - entry.video.start;
+    if (!Number.isFinite(segDuration) || segDuration <= 0) {
+      const sourceRemaining = metadata.durationSeconds - entry.video.mediaStart;
+      segDuration = sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds;
+    }
+
+    mkdirSync(vfrNormDir, { recursive: true });
+    const normalizedPath = join(vfrNormDir, `${entry.video.id}_cfr.mp4`);
+    try {
+      await convertVfrToCfr(
+        entry.videoPath,
+        normalizedPath,
+        options.fps,
+        entry.video.mediaStart,
+        segDuration,
+        signal,
+        config,
+      );
+      entry.videoPath = normalizedPath;
+      // Segment-scoped re-encode starts the new file at t=0, so downstream
+      // extraction must seek from 0, not the original mediaStart. Shallow-copy
+      // to avoid mutating the caller's VideoElement.
+      entry.video = { ...entry.video, mediaStart: 0 };
+    } catch (err) {
+      errors.push({
+        videoId: entry.video.id,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
   // Phase 3: Extract frames (parallel)
   const results = await Promise.all(
     resolvedVideos.map(async ({ video, videoPath }) => {

packages/producer/src/services/htmlCompiler.ts

@@ -1028,9 +1028,10 @@ export async function compileForRender(
           );
         }
         if (metadata.isVFR) {
-          console.warn(
-            `[Compiler] WARNING: Video "${video.id}" is variable frame rate (VFR). ` +
-              `Screen recordings and phone videos are often VFR, which causes stuttering and frame skipping in renders. Re-encode with: ${reencode}`,
+          console.info(
+            `[Compiler] Video "${video.id}" is variable frame rate (VFR); ` +
+              `the engine will normalize it to CFR before frame extraction. ` +
+              `If rendering feels slow on this video, pre-encode once with: ${reencode}`,
           );
         }
       })