fix(catalog): bind asset reads to checked project files

.github/workflows/windows-render.yml

@@ -484,6 +484,11 @@ jobs:
         shell: pwsh
         run: bunx vitest run packages/producer/src/services/hyperframeLint.file-race.test.ts --maxWorkers=2
 
+      - name: Verify catalog source reads on Windows
+        if: matrix.lane == 'studio-engine-cli'
+        shell: pwsh
+        run: bunx vitest run scripts/catalog/catalog-payload-assets.file-race.test.ts --maxWorkers=2
+
       - name: Run runtime contract test
         if: matrix.runtime_contract
         shell: pwsh

scripts/catalog-payload-assets.ts

@@ -8,8 +8,21 @@
  */
 
 import { createHash } from "node:crypto";
-import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
-import { extname, join, resolve } from "node:path";
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  readSync,
+  statSync,
+  writeFileSync,
+  realpathSync,
+  openSync,
+  fstatSync,
+  closeSync,
+  constants,
+} from "node:fs";
+import { extname, join, resolve, relative, isAbsolute, sep } from "node:path";
 
 export const MIME_TYPES: Record<string, string> = {
   ".png": "image/png",
@@ -127,6 +140,65 @@ export interface AssetResult {
   unresolved: string[];
 }
 
+function isWithin(root: string, filePath: string): boolean {
+  const rel = relative(root, filePath);
+  return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
+}
+
+function isMissingFile(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    (error.code === "ENOENT" || error.code === "ENOTDIR")
+  );
+}
+
+/** Keep oversized or concurrently growing directory assets within the read budget. */
+function readWithinBudget(fd: number, maxBytes: number): Buffer<ArrayBuffer> {
+  const chunks: Buffer<ArrayBuffer>[] = [];
+  let remaining = maxBytes + 1;
+  while (remaining > 0) {
+    const chunk = Buffer.alloc(Math.min(64 * 1024, remaining));
+    const count = readSync(fd, chunk, 0, chunk.length, null);
+    if (count === 0) break;
+    chunks.push(chunk.subarray(0, count));
+    remaining -= count;
+  }
+  return Buffer.concat(chunks);
+}
+
+/** Read one checked file from the prepared project, including internal links. */
+function readProjectFile(
+  root: string,
+  filePath: string,
+  maxBytes?: number,
+): Buffer<ArrayBuffer> | null {
+  if (!isWithin(root, filePath)) return null;
+  let source: string;
+  try {
+    source = realpathSync(filePath);
+    if (!isWithin(realpathSync(root), source)) return null;
+  } catch (error) {
+    if (isMissingFile(error)) return null;
+    throw error;
+  }
+  let fd: number;
+  try {
+    // Nonblocking mode lets fstat reject named pipes without waiting for a writer.
+    fd = openSync(source, constants.O_RDONLY | constants.O_NONBLOCK);
+  } catch (error) {
+    if (isMissingFile(error)) return null;
+    if (!statSync(source, { throwIfNoEntry: false })?.isFile()) return null;
+    throw error;
+  }
+  try {
+    if (!fstatSync(fd).isFile()) return null;
+    return maxBytes === undefined ? readFileSync(fd) : readWithinBudget(fd, maxBytes);
+  } finally {
+    closeSync(fd);
+  }
+}
+
 export interface AssetTarget {
   /** Directory shared by every item, so one font is stored once. */
   dir: string;
@@ -164,8 +236,8 @@ export function processAssets(html: string, projectDir: string, target: AssetTar
 
     // A composition reaching outside its own directory would pull an arbitrary
     // file from the build machine into a published payload.
-    const contained = source === root || source.startsWith(`${root}/`);
-    if (!contained || !existsSync(source) || !statSync(source).isFile()) {
+    const bytes = readProjectFile(root, source);
+    if (bytes === null) {
       // A name a script passed around that turned out not to be a file is just
       // a string; only a reference we are sure about counts as a broken one.
       if (strict) unresolved.push(ref);
@@ -179,7 +251,6 @@ export function processAssets(html: string, projectDir: string, target: AssetTar
       continue;
     }
 
-    const bytes = readFileSync(source);
     if (HOSTED_EXTENSIONS.has(ext)) {
       const name = `${createHash("sha256").update(bytes).digest("hex").slice(0, 16)}${ext}`;
       const dest = join(target.dir, name);
@@ -247,21 +318,30 @@ export function externalizeDataUris(
   return { html: out, externalized };
 }
 
-/** Copy a directory's publishable files into `destDir`, flattening one level. */
-function walkInto(from: string, rel: string, destDir: string, onCopy: () => void): void {
+/** Enumerate publishable paths without using pathname sizes to authorize reads. */
+function* hostedPaths(from: string, rel = ""): Generator<string> {
   for (const entry of readdirSync(from, { withFileTypes: true })) {
     if (entry.isSymbolicLink()) continue;
     const childRel = rel ? `${rel}/${entry.name}` : entry.name;
-    const childFrom = join(from, entry.name);
     if (entry.isDirectory()) {
-      walkInto(childFrom, childRel, destDir, onCopy);
-      continue;
+      yield* hostedPaths(join(from, entry.name), childRel);
+    } else if (HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) {
+      yield childRel;
     }
-    if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
-    const to = join(destDir, childRel);
-    mkdirSync(join(to, ".."), { recursive: true });
-    writeFileSync(to, readFileSync(childFrom));
-    onCopy();
+  }
+}
+
+/** Internal directory aliases share the buffers already collected at their real paths. */
+function downloadMirrorPrefix(projectDir: string): string | null {
+  try {
+    const root = realpathSync(projectDir);
+    const downloads = realpathSync(join(projectDir, "_downloads"));
+    if (!isWithin(root, downloads) || !statSync(downloads).isDirectory()) return null;
+    const rel = relative(root, downloads).split(sep).join("/");
+    return rel ? `${rel}/` : "";
+  } catch (error) {
+    if (isMissingFile(error)) return null;
+    throw error;
   }
 }
 
@@ -278,22 +358,6 @@ function walkInto(from: string, rel: string, destDir: string, onCopy: () => void
  * Only publishable types are copied; a composition needing something the host
  * drops still falls back to inlining, which is handled by the caller.
  */
-/** Publishable bytes an item would add, counted before anything is written. */
-function directoryBytes(dir: string): number {
-  let total = 0;
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    if (entry.isSymbolicLink()) continue;
-    const child = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      total += directoryBytes(child);
-      continue;
-    }
-    if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
-    total += statSync(child).size;
-  }
-  return total;
-}
-
 /**
  * What an item may add by publishing its own directory.
  *
@@ -304,45 +368,37 @@ function directoryBytes(dir: string): number {
 export const MAX_HOSTED_DIRECTORY_BYTES = 2_000_000;
 
 export function hostItemDirectory(projectDir: string, destDir: string, urlBase: string): string {
-  if (directoryBytes(projectDir) > MAX_HOSTED_DIRECTORY_BYTES) return "";
-  let copied = 0;
-
-  const walk = (from: string, rel: string): void => {
-    for (const entry of readdirSync(from, { withFileTypes: true })) {
-      // Nothing here should ever leave the prepared copy, and a symlink is the
-      // one entry that could point back out of it.
-      if (entry.isSymbolicLink()) continue;
-      const childRel = rel ? `${rel}/${entry.name}` : entry.name;
-      const childFrom = join(from, entry.name);
-      if (entry.isDirectory()) {
-        walk(childFrom, childRel);
-        continue;
-      }
-      if (!HOSTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) continue;
-      const to = join(destDir, childRel);
-      mkdirSync(join(to, ".."), { recursive: true });
-      writeFileSync(to, readFileSync(childFrom));
-      copied += 1;
-    }
+  const mirrorPrefix = downloadMirrorPrefix(projectDir);
+  const files = new Map<string, Buffer<ArrayBuffer>>();
+  let total = 0;
+  for (const path of hostedPaths(projectDir)) {
+    const bytes = readProjectFile(
+      projectDir,
+      join(projectDir, path),
+      MAX_HOSTED_DIRECTORY_BYTES - total,
+    );
+    if (bytes === null) continue;
+    total += bytes.length;
+    if (total > MAX_HOSTED_DIRECTORY_BYTES) return "";
+    files.set(path, bytes);
+  }
+
+  // Charge each source once, as before. Publish only after the entire item fits,
+  // and reuse the collected bytes for both registry and install-path layouts.
+  const publish = (path: string, bytes: Buffer<ArrayBuffer>): void => {
+    const to = join(destDir, path);
+    mkdirSync(join(to, ".."), { recursive: true });
+    writeFileSync(to, bytes);
   };
+  for (const [path, bytes] of files) publish(path, bytes);
 
-  // Both layouts are published. The prepared copy holds each asset twice, once
-  // as the registry stores it and once at its install path, and which one a
-  // composition asks for differs per item: the texture masks use the registry
-  // spelling, the caption textures the install one. Guessing wrong is a 404 at
-  // run time, so the budget below is what keeps the cost in check instead.
-  walk(projectDir, "");
-
-  // The compiler pulls remote media into `_downloads/` but rewrites references
-  // as if the document sat inside it, so `_remote_media/x.woff2` has to resolve
-  // from the item root too. Mirroring rather than moving keeps both spellings
-  // working, and the files are content-identical either way.
-  const downloads = join(projectDir, "_downloads");
-  if (existsSync(downloads) && statSync(downloads).isDirectory()) {
-    walkInto(downloads, "", destDir, () => (copied += 1));
+  // Compiler references omit `_downloads/`. Preserve both spellings and the
+  // existing mirror-wins collision order without reopening any source file.
+  for (const [path, bytes] of files) {
+    if (mirrorPrefix !== null && path.startsWith(mirrorPrefix))
+      publish(path.slice(mirrorPrefix.length), bytes);
   }
-
-  return copied > 0 ? urlBase : "";
+  return files.size > 0 ? urlBase : "";
 }
 
 /**
@@ -376,8 +432,9 @@ export function inlineMountedComposition(html: string, projectDir: string): stri
     (whole, quote: string, ref: string) => {
       if (/^(https?:|data:)/i.test(ref)) return whole;
       const source = resolve(projectDir, ref.replace(/^\.\//, "").split(/[?#]/)[0] ?? ref);
-      if (!source.startsWith(resolve(projectDir)) || !existsSync(source)) return whole;
-      const encoded = readFileSync(source).toString("base64");
+      const bytes = readProjectFile(resolve(projectDir), source);
+      if (bytes === null) return whole;
+      const encoded = bytes.toString("base64");
       return `data-composition-src=${quote}data:text/html;base64,${encoded}${quote}`;
     },
   );