fix(producer): fall back to copying extracted frames when symlink hits EPERM [P1]

packages/producer/src/services/render/shared.ts

@@ -10,7 +10,15 @@
  * backwards compatibility with existing test files and external callers.
  */
 
-import { copyFileSync, cpSync, existsSync, mkdirSync, symlinkSync, writeFileSync } from "node:fs";
+import {
+  copyFileSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
 import {
   CANVAS_DIMENSIONS,
@@ -275,6 +283,10 @@ type MaterializeFileSystem = {
   mkdirSync: (path: string, options: { recursive: true }) => unknown;
   symlinkSync: (target: string, path: string) => unknown;
   cpSync: (src: string, dest: string, options: { recursive: true }) => unknown;
+  // Optional: only the stale-entry (EEXIST) recovery path calls it, and the
+  // default fileSystem always supplies it. Test doubles that never trigger
+  // EEXIST may omit it.
+  rmSync?: (path: string, options: { recursive: true; force: true }) => unknown;
 };
 
 type MaterializeExtractedFramesOptions = {
@@ -304,6 +316,7 @@ const materializeFileSystem: MaterializeFileSystem = {
   mkdirSync,
   symlinkSync,
   cpSync,
+  rmSync,
 };
 
 /**
@@ -373,6 +386,76 @@ export function createMemorySampler(intervalMs: number = 250): MemorySampler {
  * Exported for integration tests; not part of the stable public API —
  * external callers should use `executeRenderJob` instead.
  */
+// Stage one video's extracted-frame dir into the compiled dir. Default is a
+// single symlink (cheap; the in-process renderer); `materializeSymlinks` copies
+// instead (distributed plan() needs a self-contained dir). On Windows without
+// Developer Mode/Administrator symlink creation is rejected with EPERM/EACCES,
+// which failed high/standard renders — degrade to a copy there rather than
+// throwing. Non-permission errors still propagate so real failures aren't hidden.
+// One-time guard for the symlink→copy fallback notice below.
+let warnedSymlinkFallback = false;
+
+// Create the symlink, degrading to a copy on Windows' no-symlink-privilege
+// errors (EPERM/EACCES, plus UNKNOWN — some Windows builds surface a symlink
+// privilege denial as an UNKNOWN-coded error rather than EPERM). Non-permission
+// errors propagate.
+function linkOrCopyFrameDir(fileSystem: MaterializeFileSystem, src: string, dest: string): void {
+  try {
+    fileSystem.symlinkSync(src, dest);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException | undefined)?.code;
+    if (code !== "EPERM" && code !== "EACCES" && code !== "UNKNOWN") throw err;
+    // Copying is measurably slower than symlinking, so surface the degrade once
+    // — it explains a render that suddenly got heavier and saves a support
+    // round-trip diagnosing slow frame staging on Windows.
+    if (!warnedSymlinkFallback) {
+      warnedSymlinkFallback = true;
+      defaultLogger.info(
+        `[Render] Symlinking extracted frames was rejected (${code}); copying them into the compiled dir instead. Expected on Windows without Developer Mode/Administrator.`,
+      );
+    }
+    fileSystem.cpSync(src, dest, { recursive: true });
+  }
+}
+
+function stageExtractedFrameDir(
+  fileSystem: MaterializeFileSystem,
+  src: string,
+  dest: string,
+  materializeSymlinks: boolean,
+): void {
+  try {
+    stageExtractedFrameDirOnce(fileSystem, src, dest, materializeSymlinks);
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException | undefined)?.code !== "EEXIST") throw err;
+    // A stale entry already sits at `dest` — typically a DANGLING symlink left
+    // after the extraction cache was GC'd (its target removed), or a Windows
+    // machine reusing a dir a prior Linux run populated with symlinks (the eager
+    // cpSync then collides with the existing link). The caller's existsSync()
+    // guard follows the link, so the dead link reads as absent and we reach
+    // here; the entry itself still exists, so re-staging collides with EEXIST.
+    // Clear the stale entry and re-stage. Applies to BOTH the symlink and
+    // eager-copy paths so a cross-platform dir reuse self-heals either way.
+    fileSystem.rmSync?.(dest, { recursive: true, force: true });
+    stageExtractedFrameDirOnce(fileSystem, src, dest, materializeSymlinks);
+  }
+}
+
+// One staging attempt: eager copy for the distributed self-contained dir,
+// otherwise a symlink (degrading to a copy on no-symlink-privilege errors).
+function stageExtractedFrameDirOnce(
+  fileSystem: MaterializeFileSystem,
+  src: string,
+  dest: string,
+  materializeSymlinks: boolean,
+): void {
+  if (materializeSymlinks) {
+    fileSystem.cpSync(src, dest, { recursive: true });
+    return;
+  }
+  linkOrCopyFrameDir(fileSystem, src, dest);
+}
+
 export function materializeExtractedFramesForCompiledDir(
   extracted: MaterializedExtractedFrames[],
   compiledDir: string,
@@ -390,11 +473,12 @@ export function materializeExtractedFramesForCompiledDir(
     const linkPath = pathModule.join(compiledFrameRoot, ext.videoId);
     if (!fileSystem.existsSync(linkPath)) {
       fileSystem.mkdirSync(pathModule.dirname(linkPath), { recursive: true });
-      if (options.materializeSymlinks) {
-        fileSystem.cpSync(resolvedOut, linkPath, { recursive: true });
-      } else {
-        fileSystem.symlinkSync(resolvedOut, linkPath);
-      }
+      stageExtractedFrameDir(
+        fileSystem,
+        resolvedOut,
+        linkPath,
+        options.materializeSymlinks === true,
+      );
     }
 
     const remapped = new Map<number, string>();

packages/producer/src/services/renderOrchestrator.test.ts

@@ -554,6 +554,188 @@ describe("materializeExtractedFramesForCompiledDir", () => {
     expect(extracted.framePaths.get(0)).toBe(win32.join(linkPath, "frame_000001.jpg"));
     expect(copies).toEqual([{ src: outputDir, dest: linkPath, recursive: true }]);
   });
+
+  // fallow-ignore-next-line code-duplication
+  it("falls back to copying frames when symlinkSync fails with EPERM (Windows, no Developer Mode)", () => {
+    // Windows without Developer Mode/Administrator rejects symlink creation with
+    // EPERM — high/standard-quality renders failed here while draft worked. The
+    // helper must degrade to a recursive copy instead of throwing.
+    const compiledDir = win32.resolve("C:\\compiled");
+    const outputDir = win32.resolve("D:\\cache\\abc123");
+    const framePath = win32.join(outputDir, "frame_000001.jpg");
+    const extracted = createExtractedFrames(outputDir, framePath);
+    const copies: Array<{ src: string; dest: string; recursive: boolean }> = [];
+
+    // fallow-ignore-next-line code-duplication
+    materializeExtractedFramesForCompiledDir([extracted], compiledDir, {
+      pathModule: win32,
+      fileSystem: {
+        existsSync: () => false,
+        mkdirSync: () => undefined,
+        symlinkSync: () => {
+          const err: NodeJS.ErrnoException = new Error("EPERM: operation not permitted, symlink");
+          err.code = "EPERM";
+          throw err;
+        },
+        cpSync: (src, dest, options) => {
+          copies.push({ src, dest, recursive: options.recursive });
+        },
+      },
+    });
+
+    const linkPath = win32.join(compiledDir, "__hyperframes_video_frames", "video-1");
+    expect(copies).toEqual([{ src: outputDir, dest: linkPath, recursive: true }]);
+    expect(extracted.outputDir).toBe(linkPath);
+    expect(extracted.framePaths.get(0)).toBe(win32.join(linkPath, "frame_000001.jpg"));
+  });
+
+  it("rethrows a non-permission symlink error instead of masking it with a copy", () => {
+    const compiledDir = win32.resolve("C:\\compiled");
+    const outputDir = win32.resolve("D:\\cache\\abc123");
+    const framePath = win32.join(outputDir, "frame_000001.jpg");
+    const extracted = createExtractedFrames(outputDir, framePath);
+
+    expect(() =>
+      materializeExtractedFramesForCompiledDir([extracted], compiledDir, {
+        pathModule: win32,
+        fileSystem: {
+          existsSync: () => false,
+          mkdirSync: () => undefined,
+          symlinkSync: () => {
+            const err: NodeJS.ErrnoException = new Error("ENOSPC: no space left");
+            err.code = "ENOSPC";
+            throw err;
+          },
+          cpSync: () => {
+            throw new Error("must not fall back to copy for a non-permission error");
+          },
+        },
+      }),
+    ).toThrow(/ENOSPC/);
+  });
+
+  // fallow-ignore-next-line code-duplication
+  it("clears a stale dangling entry and re-stages when symlinkSync fails with EEXIST", () => {
+    // After the extraction cache is GC'd, a symlink from a prior render dangles
+    // (its target removed). existsSync() follows the dead link so the caller's
+    // guard reads it as absent and reaches staging, but the link file itself
+    // still exists, so symlinkSync collides with EEXIST. The helper must clear
+    // the stale entry (rmSync) and re-stage, not hard-fail the render.
+    const compiledDir = win32.resolve("C:\\compiled");
+    const outputDir = win32.resolve("D:\\cache\\abc123");
+    const framePath = win32.join(outputDir, "frame_000001.jpg");
+    const extracted = createExtractedFrames(outputDir, framePath);
+    const linkPath = win32.join(compiledDir, "__hyperframes_video_frames", "video-1");
+    const removed: string[] = [];
+    const symlinks: Array<{ target: string; path: string }> = [];
+    let symlinkCalls = 0;
+
+    // fallow-ignore-next-line code-duplication
+    materializeExtractedFramesForCompiledDir([extracted], compiledDir, {
+      pathModule: win32,
+      fileSystem: {
+        existsSync: () => false,
+        mkdirSync: () => undefined,
+        symlinkSync: (target, path) => {
+          symlinkCalls += 1;
+          if (symlinkCalls === 1) {
+            const err: NodeJS.ErrnoException = new Error("EEXIST: file already exists, symlink");
+            err.code = "EEXIST";
+            throw err;
+          }
+          symlinks.push({ target, path });
+        },
+        cpSync: () => {
+          throw new Error("EEXIST recovery should re-link, not copy");
+        },
+        rmSync: (path) => {
+          removed.push(path);
+        },
+      },
+    });
+
+    expect(removed).toEqual([linkPath]);
+    expect(symlinks).toEqual([{ target: outputDir, path: linkPath }]);
+    expect(extracted.framePaths.get(0)).toBe(win32.join(linkPath, "frame_000001.jpg"));
+  });
+
+  // fallow-ignore-next-line code-duplication
+  it("falls back to copying when symlinkSync fails with UNKNOWN (some Windows privilege denials)", () => {
+    // Some Windows builds surface a no-symlink-privilege denial as an
+    // UNKNOWN-coded error rather than EPERM/EACCES — it must still degrade to a
+    // copy, not hard-fail the render.
+    const compiledDir = win32.resolve("C:\\compiled");
+    const outputDir = win32.resolve("D:\\cache\\abc123");
+    const framePath = win32.join(outputDir, "frame_000001.jpg");
+    const extracted = createExtractedFrames(outputDir, framePath);
+    const copies: Array<{ src: string; dest: string; recursive: boolean }> = [];
+
+    // fallow-ignore-next-line code-duplication
+    materializeExtractedFramesForCompiledDir([extracted], compiledDir, {
+      pathModule: win32,
+      fileSystem: {
+        existsSync: () => false,
+        mkdirSync: () => undefined,
+        symlinkSync: () => {
+          const err: NodeJS.ErrnoException = new Error("UNKNOWN: unknown error, symlink");
+          err.code = "UNKNOWN";
+          throw err;
+        },
+        cpSync: (src, dest, options) => {
+          copies.push({ src, dest, recursive: options.recursive });
+        },
+      },
+    });
+
+    const linkPath = win32.join(compiledDir, "__hyperframes_video_frames", "video-1");
+    expect(copies).toEqual([{ src: outputDir, dest: linkPath, recursive: true }]);
+    expect(extracted.framePaths.get(0)).toBe(win32.join(linkPath, "frame_000001.jpg"));
+  });
+
+  // fallow-ignore-next-line code-duplication
+  it("clears a stale entry and re-copies when the eager-copy path (materializeSymlinks) hits EEXIST", () => {
+    // #2025 routes Windows through the eager-copy branch. Reusing a dir a prior
+    // Linux run populated with a (now dangling) symlink makes cpSync collide
+    // with EEXIST — the recovery must clear the stale entry and re-copy, exactly
+    // like the symlink path does.
+    const compiledDir = win32.resolve("C:\\compiled");
+    const outputDir = win32.resolve("D:\\cache\\abc123");
+    const framePath = win32.join(outputDir, "frame_000001.jpg");
+    const extracted = createExtractedFrames(outputDir, framePath);
+    const linkPath = win32.join(compiledDir, "__hyperframes_video_frames", "video-1");
+    const removed: string[] = [];
+    const copies: Array<{ src: string; dest: string }> = [];
+    let cpCalls = 0;
+
+    // fallow-ignore-next-line code-duplication
+    materializeExtractedFramesForCompiledDir([extracted], compiledDir, {
+      pathModule: win32,
+      materializeSymlinks: true,
+      fileSystem: {
+        existsSync: () => false,
+        mkdirSync: () => undefined,
+        symlinkSync: () => {
+          throw new Error("eager-copy path must not symlink");
+        },
+        cpSync: (src, dest) => {
+          cpCalls += 1;
+          if (cpCalls === 1) {
+            const err: NodeJS.ErrnoException = new Error("EEXIST: file already exists, cp");
+            err.code = "EEXIST";
+            throw err;
+          }
+          copies.push({ src, dest });
+        },
+        rmSync: (path) => {
+          removed.push(path);
+        },
+      },
+    });
+
+    expect(removed).toEqual([linkPath]);
+    expect(copies).toEqual([{ src: outputDir, dest: linkPath }]);
+    expect(extracted.framePaths.get(0)).toBe(win32.join(linkPath, "frame_000001.jpg"));
+  });
 });
 
 describe("writeCompiledArtifacts — external assets on Windows drive-letter paths (GH #321)", () => {