From 2f22ba3ccadee7a223cb2489e0d2c8d9ff21ff99 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 2 Aug 2026 19:51:29 +0000 Subject: [PATCH 01/13] fix: bound HDR and video extraction resources --- packages/engine/src/index.ts | 2 + .../engine/src/services/systemMemory.test.ts | 25 ++ packages/engine/src/services/systemMemory.ts | 8 +- .../src/services/videoFrameExtractor.test.ts | 50 +++ .../src/services/videoFrameExtractor.ts | 54 ++- packages/producer/src/server.hdrMode.test.ts | 73 ++++ packages/producer/src/server.test.ts | 19 ++ packages/producer/src/server.ts | 11 + .../render/stages/captureHdrFrameShared.ts | 15 +- .../render/stages/captureHdrResources.test.ts | 288 +++++++++++++++- .../render/stages/captureHdrResources.ts | 319 ++++++++++++++---- .../services/render/stages/captureHdrStage.ts | 11 +- .../stages/captureStreamingStage.test.ts | 8 +- .../extractVideosStage.timelineBound.test.ts | 120 +++++++ .../render/stages/extractVideosStage.ts | 1 + 15 files changed, 898 insertions(+), 106 deletions(-) create mode 100644 packages/producer/src/server.hdrMode.test.ts create mode 100644 packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index b479920dc85..6793688dea7 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -66,6 +66,7 @@ export { normalizeVp9CpuUsed, } from "./services/vp9Options.js"; export { + getCgroupMemoryLimitMb, getSystemTotalMb, isLowMemorySystem, LOW_MEMORY_TOTAL_MB_THRESHOLD, @@ -179,6 +180,7 @@ export { parseImageElements, extractVideoFramesRange, extractAllVideoFrames, + resolveVideoExtractionDuration, resolveProjectRelativeSrc, getFrameAtTime, createFrameLookupTable, diff --git a/packages/engine/src/services/systemMemory.test.ts b/packages/engine/src/services/systemMemory.test.ts index 315f04d142a..f8c91528f45 100644 --- a/packages/engine/src/services/systemMemory.test.ts +++ b/packages/engine/src/services/systemMemory.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; import { _resetCgroupLimitCacheForTests, @@ -154,6 +155,30 @@ describe("parseCgroupLimitMb", () => { }); }); +describe("getCgroupMemoryLimitMb", () => { + it("returns only an actual cgroup limit and never host RAM", async () => { + await withSystemMemoryMocks( + { + files: { [CGROUP_V2_MEMORY_MAX_PATH]: `${24576 * BYTES_PER_MIB}` }, + hostTotalMb: 65536, + }, + ({ getCgroupMemoryLimitMb }) => { + expect(getCgroupMemoryLimitMb()).toBe(24576); + }, + ); + + await withSystemMemoryMocks( + { + files: { [CGROUP_V2_MEMORY_MAX_PATH]: "max" }, + hostTotalMb: 65536, + }, + ({ getCgroupMemoryLimitMb }) => { + expect(getCgroupMemoryLimitMb()).toBeNull(); + }, + ); + }); +}); + describe("getSystemTotalMb", () => { it("caches cgroup probes until the test reset hook clears the cache", async () => { const readCalls: string[] = []; diff --git a/packages/engine/src/services/systemMemory.ts b/packages/engine/src/services/systemMemory.ts index f60c893ae4f..c37d62c74fa 100644 --- a/packages/engine/src/services/systemMemory.ts +++ b/packages/engine/src/services/systemMemory.ts @@ -80,7 +80,11 @@ export function _resetCgroupLimitCacheForTests(): void { _warnedCgroupReadFailure = false; } -function getCgroupLimitMb(): number | null { +/** + * Actual Linux cgroup memory ceiling in MiB, or null when the process is not + * cgroup-limited. Unlike getSystemTotalMb this never falls back to host RAM. + */ +export function getCgroupMemoryLimitMb(): number | null { if (_cachedCgroupLimitMb !== undefined) return _cachedCgroupLimitMb; if (process.platform !== "linux") { @@ -142,7 +146,7 @@ function warnCgroupReadFailure(path: string, error: unknown): void { /** Total physical RAM in MiB. */ export function getSystemTotalMb(): number { const hostTotalMb = Math.floor(totalmem() / BYTES_PER_MIB); - const cgroupLimitMb = getCgroupLimitMb(); + const cgroupLimitMb = getCgroupMemoryLimitMb(); return cgroupLimitMb === null ? hostTotalMb : Math.min(hostTotalMb, cgroupLimitMb); } diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index d494565db9b..9d28eb18913 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -24,6 +24,7 @@ import { resolveFrameFormat, codecMayHaveAlpha, decoderForCodec, + resolveVideoExtractionDuration, getFrameAtTime, analyzeClipMediaFit, classifyVideoExtractionError, @@ -45,6 +46,55 @@ import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.j // synthesized VFR fixture. const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +describe("resolveVideoExtractionDuration", () => { + const metadata = (durationSeconds: number): VideoMetadata => ({ + durationSeconds, + videoStreamDurationSeconds: durationSeconds, + width: 1920, + height: 1080, + fps: 30, + videoCodec: "h264", + hasAudio: false, + isVFR: false, + hasAlpha: false, + colorSpace: null, + }); + const video = (overrides: Partial = {}): VideoElement => ({ + id: "root-video", + src: "video.mp4", + start: 0, + end: Number.POSITIVE_INFINITY, + mediaStart: 0, + loop: false, + hasAudio: false, + ...overrides, + }); + + it("caps an open 60-second root source to a two-second composition", () => { + expect(resolveVideoExtractionDuration(video(), metadata(60), 2)).toBe(2); + }); + + it("keeps a shorter natural source duration inside a longer composition", () => { + expect(resolveVideoExtractionDuration(video(), metadata(2), 10)).toBe(2); + }); + + it("preserves explicit bounds and loop flags while applying the timeline ceiling", () => { + const explicitLoop = video({ end: 8, loop: true }); + expect(resolveVideoExtractionDuration(explicitLoop, metadata(60), 10)).toBe(8); + expect(explicitLoop.loop).toBe(true); + }); + + it("preserves negative start and mediaStart semantics", () => { + const preroll = video({ start: -3, mediaStart: 5 }); + expect(resolveVideoExtractionDuration(preroll, metadata(60), 2)).toBe(5); + expect(preroll).toMatchObject({ start: -3, mediaStart: 5 }); + }); + + it("retains legacy behavior when no timeline end is supplied", () => { + expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60); + }); +}); + describe("video extraction failure taxonomy and bounded retry", () => { it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => { expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({ diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index a3d7df8699f..05e69316fa3 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -83,6 +83,12 @@ export interface ExtractionOptions { quality?: number; format?: VideoFrameFormat; sdrToHdrTransfer?: HdrTransfer; + /** + * Absolute composition/timeline end in seconds. Applied only after source + * metadata resolves open-ended/natural-duration media, so authored start, + * mediaStart, loop, and sentinel semantics remain unchanged. + */ + timelineEnd?: number; /** * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior; * the producer may canary at most one retry after observing typed failures. @@ -444,7 +450,10 @@ export function parseVideoElements(html: string): VideoElement[] { // reference; the resolver handles both. const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0; // Derive end from data-end → data-start+data-duration → Infinity (natural duration). - // The caller (htmlCompiler) clamps Infinity to the composition's absoluteEnd. + // Static compilation cannot always clamp root media because GSAP may supply + // the root duration at runtime. The producer passes the resolved timeline + // end into frame extraction, which caps the source duration only after the + // natural duration is known without rewriting authored timing metadata. let end = 0; if (endAttr) { end = parseFloat(endAttr); @@ -725,6 +734,24 @@ function resolveSegmentDuration( return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; } +/** Resolve source duration first, then cap it at the remaining render timeline. */ +export function resolveVideoExtractionDuration( + video: Pick, + metadata: VideoMetadata, + timelineEnd?: number, +): number { + const resolvedDuration = resolveSegmentDuration( + video.end - video.start, + video.mediaStart, + metadata, + ); + if (timelineEnd === undefined) return resolvedDuration; + if (!Number.isFinite(timelineEnd)) { + throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); + } + return Math.min(resolvedDuration, Math.max(0, timelineEnd - video.start)); +} + /** * Codecs whose bitstream is allowed to carry an alpha channel. Default the * extraction path to PNG output for these regardless of `metadata.hasAlpha` @@ -1029,6 +1056,11 @@ export async function extractAllVideoFrames( >, compiledDir?: string, ): Promise { + if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) { + throw new Error( + `Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`, + ); + } const startTime = Date.now(); const extracted: ExtractedFrames[] = []; const errors: VideoExtractionFailure[] = []; @@ -1062,6 +1094,7 @@ export async function extractAllVideoFrames( const warnedSrcs = new Set(); for (const video of videos) { if (signal?.aborted) break; + if (options.timelineEnd !== undefined && video.start >= options.timelineEnd) continue; try { let videoPath = video.src; if (!isHttpUrl(videoPath)) { @@ -1113,8 +1146,8 @@ export async function extractAllVideoFrames( breakdown.resolveMs = Date.now() - phase1Start; // Snapshot the pre-preflight key inputs so the extraction cache keys on the - // user-visible source (original path, original mediaStart, original segment - // bounds) rather than the workDir-local normalized file produced by the + // user-visible source (original path and mediaStart) rather than the + // workDir-local normalized file produced by the // HDR preflight. Without this, every render would write a new // normalized file with a fresh mtime → fresh cache key → perpetual misses. const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => { @@ -1129,8 +1162,6 @@ export async function extractAllVideoFrames( mtimeMs: stat.mtimeMs, size: stat.size, mediaStart: video.mediaStart, - start: video.start, - end: video.end, }; }); @@ -1316,17 +1347,12 @@ export async function extractAllVideoFrames( ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined; - const keyDuration = resolveSegmentDuration( - keyInput.end - keyInput.start, - keyInput.mediaStart, - work.metadata, - ); const lookup = lookupCacheEntry(cacheRootDir, { videoPath: keyInput.videoPath, mtimeMs: keyInput.mtimeMs, size: keyInput.size, mediaStart: keyInput.mediaStart, - duration: keyDuration, + duration: work.videoDuration, fps: options.fps, format: work.format, transform, @@ -1516,11 +1542,7 @@ export async function extractAllVideoFrames( } try { const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath)); - const videoDuration = resolveSegmentDuration( - video.end - video.start, - video.mediaStart, - metadata, - ); + const videoDuration = resolveVideoExtractionDuration(video, metadata, options.timelineEnd); if (video.end - video.start !== videoDuration) { video.end = video.start + videoDuration; } diff --git a/packages/producer/src/server.hdrMode.test.ts b/packages/producer/src/server.hdrMode.test.ts new file mode 100644 index 00000000000..d7a214e6413 --- /dev/null +++ b/packages/producer/src/server.hdrMode.test.ts @@ -0,0 +1,73 @@ +import { Hono } from "hono"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const capturedRenderConfigs = vi.hoisted(() => new Array>()); + +vi.mock("./services/renderOrchestrator.js", () => { + class RenderCancelledError extends Error {} + + return { + RenderCancelledError, + createRenderJob: (config: Record) => { + capturedRenderConfigs.push(config); + return { + config, + progress: 0, + currentStage: "queued", + framesRendered: 0, + totalFrames: 0, + warnings: [], + }; + }, + executeRenderJob: async (job: Record) => { + job.outcome = "completed"; + job.currentStage = "complete"; + }, + }; +}); + +import { createRenderHandlers } from "./server.js"; + +function createInternalStreamingApp(): Hono { + const app = new Hono(); + const handlers = createRenderHandlers({ + getRequestId: () => "hdr-mode-test", + maxConcurrentRenders: 1, + }); + app.post("/v1/render-stream", handlers.renderStream); + return app; +} + +describe("POST /v1/render-stream — hdrMode", () => { + beforeEach(() => capturedRenderConfigs.splice(0)); + + it.each(["auto", "force-hdr", "force-sdr"] as const)( + "forwards %s through createRenderRequest into RenderConfig", + async (hdrMode) => { + const response = await createInternalStreamingApp().request("/v1/render-stream", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ html: "", hdrMode }), + }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain('"type":"complete"'); + expect(capturedRenderConfigs).toHaveLength(1); + expect(capturedRenderConfigs[0]?.hdrMode).toBe(hdrMode); + }, + ); + + it("rejects an invalid mode before creating a render job", async () => { + const response = await createInternalStreamingApp().request("/v1/render-stream", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ html: "", hdrMode: "hdr" }), + }); + + expect(response.status).toBe(200); + expect(await response.text()).toContain( + 'hdrMode must be one of: \\"auto\\", \\"force-hdr\\", \\"force-sdr\\"', + ); + expect(capturedRenderConfigs).toHaveLength(0); + }); +}); diff --git a/packages/producer/src/server.test.ts b/packages/producer/src/server.test.ts index 813a2fba188..bac854f7467 100644 --- a/packages/producer/src/server.test.ts +++ b/packages/producer/src/server.test.ts @@ -46,6 +46,17 @@ describe("parseRenderOptions — render strictness", () => { }); }); +describe("parseRenderOptions — hdrMode", () => { + it.each(["auto", "force-hdr", "force-sdr"] as const)("forwards %s", (hdrMode) => { + expect(parseRenderOptions({ hdrMode }).hdrMode).toBe(hdrMode); + }); + + it("drops invalid values from the lenient parser", () => { + expect(parseRenderOptions({ hdrMode: "hdr" }).hdrMode).toBeUndefined(); + expect(parseRenderOptions({ hdrMode: true }).hdrMode).toBeUndefined(); + }); +}); + describe("prepareRenderBody — validation", () => { it.each(["", " "])( "treats an empty projectDir as absent and uses inline HTML", @@ -67,6 +78,14 @@ describe("prepareRenderBody — validation", () => { expect((result as { error: string }).error).toContain("variables must be a JSON object"); }); + it("rejects an explicitly-supplied invalid hdrMode", async () => { + const result = await prepareRenderBody({ hdrMode: "hdr", html: "" }); + expect(result).toHaveProperty("error"); + expect((result as { error: string }).error).toContain( + 'hdrMode must be one of: "auto", "force-hdr", "force-sdr"', + ); + }); + it("rejects an explicitly-supplied invalid outputResolution", async () => { const result = await prepareRenderBody({ outputResolution: "8k", html: "" }); expect(result).toHaveProperty("error"); diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 75611b2624a..624f61a1fd8 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -84,6 +84,7 @@ interface RenderInput { quality: "draft" | "standard" | "high"; format?: "mp4" | "webm" | "mov"; videoFrameFormat?: RenderConfig["videoFrameFormat"]; + hdrMode?: RenderConfig["hdrMode"]; workers?: number; useGpu: boolean; debug: boolean; @@ -158,6 +159,10 @@ function parseServerFormat(value: unknown): RenderInput["format"] { return value === "mp4" || value === "webm" || value === "mov" ? value : undefined; } +function parseServerHdrMode(value: unknown): RenderInput["hdrMode"] { + return value === "auto" || value === "force-hdr" || value === "force-sdr" ? value : undefined; +} + export function parseRenderOptions(body: Record): Omit { // Accept either a JSON `number` (integer fps) or a JSON `string` (rational // like "30000/1001"). Falls back to 30 fps on parse failure to preserve the @@ -176,6 +181,7 @@ export function parseRenderOptions(body: Record): Omit): Omit): string | undefi if (body.variables !== undefined && !isPlainObject(body.variables)) { return 'variables must be a JSON object keyed by variable id (e.g. {"title":"Hello"})'; } + if (body.hdrMode !== undefined && parseServerHdrMode(body.hdrMode) === undefined) { + return 'hdrMode must be one of: "auto", "force-hdr", "force-sdr"'; + } return validateOutputResolutionOverride(body); } diff --git a/packages/producer/src/services/render/stages/captureHdrFrameShared.ts b/packages/producer/src/services/render/stages/captureHdrFrameShared.ts index 2e25a640558..8ef1cc3d28e 100644 --- a/packages/producer/src/services/render/stages/captureHdrFrameShared.ts +++ b/packages/producer/src/services/render/stages/captureHdrFrameShared.ts @@ -9,7 +9,6 @@ * centralized here. */ -import { rmSync } from "node:fs"; import { type BeforeCaptureHook, type CaptureSession, @@ -28,9 +27,9 @@ import { type TransitionRange, blitHdrImageLayer, blitHdrVideoLayer, - closeHdrVideoFrameSource, selectDomLayerShowIds, } from "../../hdrCompositor.js"; +import { cleanupHdrVideoFrameSource } from "./captureHdrResources.js"; import { type HdrPerfCollector, type HdrPerfTimingKey, @@ -402,17 +401,7 @@ export function cleanupEndedHdrVideos(args: { if (!stillNeeded) { const frameSource = hdrVideoFrameSources.get(videoId); if (frameSource) { - closeHdrVideoFrameSource(frameSource, log); - try { - rmSync(frameSource.dir, { recursive: true, force: true }); - } catch (err) { - log.warn("Failed to clean up HDR raw frame directory", { - videoId, - frameDir: frameSource.dir, - rawPath: frameSource.rawPath, - error: err instanceof Error ? err.message : String(err), - }); - } + cleanupHdrVideoFrameSource(frameSource, log); hdrVideoFrameSources.delete(videoId); } cleanedUpVideos.add(videoId); diff --git a/packages/producer/src/services/render/stages/captureHdrResources.test.ts b/packages/producer/src/services/render/stages/captureHdrResources.test.ts index 503b97f1ed9..27bcb315171 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.test.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.test.ts @@ -1,5 +1,88 @@ -import { describe, expect, it } from "vitest"; -import { estimateHdrExtractionBytes } from "./captureHdrResources.js"; +import { + closeSync, + constants, + existsSync, + mkdtempSync, + openSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import type { RunFfmpegResult, VideoElement } from "@hyperframes/engine"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createRenderJob } from "../../renderOrchestrator.js"; +import { + cleanupHdrVideoFrameSource, + estimateHdrExtractionBytes, + extractHdrVideoFrames, + getHdrExtractionReservedBytes, + reserveHdrExtractionBytes, + resolveHdrExtractionActiveBudgetBytes, + resolveHdrExtractionBudgetBytes, + resolveHdrExtractionWindow, +} from "./captureHdrResources.js"; + +afterEach(() => { + vi.unstubAllEnvs(); + expect(getHdrExtractionReservedBytes()).toBe(0); +}); + +function ffmpegResult(success: boolean): RunFfmpegResult { + return { + success, + exitCode: success ? 0 : 1, + stderr: success ? "" : "mock extraction failure", + durationMs: 1, + terminationReason: "exit", + }; +} + +function hdrVideo(id: string, overrides: Partial = {}): VideoElement { + return { + id, + src: `${id}.mov`, + start: 0, + end: Number.POSITIVE_INFINITY, + mediaStart: 0, + loop: false, + hasAudio: false, + ...overrides, + }; +} + +function hdrExtractionFixture(videos: VideoElement[], framesDir: string) { + return { + job: createRenderJob({ fps: { num: 2, den: 1 }, quality: "standard" }), + log: { + error: vi.fn(), + warn: vi.fn(), + info: vi.fn(), + debug: vi.fn(), + }, + framesDir, + composition: { + duration: 2, + videos, + audios: [], + images: [], + width: 1, + height: 1, + }, + prep: { + hdrVideoIds: videos.map((video) => video.id), + hdrVideoSrcPaths: new Map(videos.map((video) => [video.id, video.src])), + hdrVideoStartTimes: new Map(videos.map((video) => [video.id, video.start])), + hdrImageStartTimes: new Map(), + hdrExtractionDims: new Map(videos.map((video) => [video.id, { width: 1, height: 1 }])), + hdrImageFitInfo: new Map(), + }, + width: 1, + height: 1, + abortSignal: undefined, + hdrDiagnostics: { videoExtractionFailures: 0, imageDecodeFailures: 0 }, + }; +} describe("estimateHdrExtractionBytes", () => { it("sums 6 bytes per pixel per frame across videos", () => { @@ -26,3 +109,204 @@ describe("estimateHdrExtractionBytes", () => { ); }); }); + +describe("resolveHdrExtractionWindow", () => { + it("bounds a two-second composition backed by an unbounded HDR source to 60 raw frames", () => { + const { durationSeconds } = resolveHdrExtractionWindow( + { id: "long-hdr", start: 0, end: Number.POSITIVE_INFINITY, mediaStart: 0 }, + 2, + ); + expect(durationSeconds).toBe(2); + expect(estimateHdrExtractionBytes([{ durationSeconds, width: 3840, height: 2160 }], 30)).toBe( + 60 * 3840 * 2160 * 6, + ); + }); + + it("independently caps a stale finite media end at the composition duration", () => { + expect(resolveHdrExtractionWindow({ id: "hdr", start: 0, end: 60, mediaStart: 0 }, 2)).toEqual({ + compositionStart: 0, + mediaStart: 0, + durationSeconds: 2, + }); + }); + + it("preserves negative starts and source offsets while bounding the timeline end", () => { + expect(resolveHdrExtractionWindow({ id: "hdr", start: -3, end: 60, mediaStart: 5 }, 2)).toEqual( + { compositionStart: -3, mediaStart: 5, durationSeconds: 5 }, + ); + }); + + it("rejects HDR media with no interval inside the composition", () => { + expect(() => + resolveHdrExtractionWindow({ id: "hdr", start: 3, end: Infinity, mediaStart: 0 }, 2), + ).toThrow('HDR video "hdr" has no finite interval inside the 2s composition'); + }); +}); + +describe("resolveHdrExtractionBudgetBytes", () => { + it("uses half an actual cgroup limit when no env budget is configured", () => { + expect(resolveHdrExtractionBudgetBytes(undefined, 24 * 1024)).toBe(12 * 1024 ** 3); + expect(resolveHdrExtractionBudgetBytes(undefined, null)).toBeUndefined(); + }); + + it("uses the stricter of the environment and cgroup budgets", () => { + expect(resolveHdrExtractionBudgetBytes(String(8 * 1024 ** 3), 24 * 1024)).toBe(8 * 1024 ** 3); + expect(resolveHdrExtractionBudgetBytes(String(20 * 1024 ** 3), 24 * 1024)).toBe(12 * 1024 ** 3); + expect(resolveHdrExtractionBudgetBytes("1234.9", null)).toBe(1234); + }); + + it("rejects invalid budgets", () => { + expect(() => resolveHdrExtractionBudgetBytes("0", null)).toThrow("must be a positive finite"); + expect(() => resolveHdrExtractionBudgetBytes("Infinity", null)).toThrow( + "must be a positive finite", + ); + }); +}); + +describe("reserveHdrExtractionBytes", () => { + it("prevents concurrent aggregate overcommit and releases idempotently", () => { + const releaseFirst = reserveHdrExtractionBytes(60, 100); + try { + expect(getHdrExtractionReservedBytes()).toBe(60); + expect(() => reserveHdrExtractionBytes(50, 100)).toThrow("Concurrent HDR pre-extractions"); + } finally { + releaseFirst(); + releaseFirst(); + } + + const releaseAfter = reserveHdrExtractionBytes(100, 100); + expect(getHdrExtractionReservedBytes()).toBe(100); + releaseAfter(); + }); + + it("prevents two concurrent jobs from overcommitting a disk-only budget", () => { + const diskOnlyBudget = resolveHdrExtractionActiveBudgetBytes(undefined, 100); + expect(diskOnlyBudget).toBe(90); + + const releaseFirstJob = reserveHdrExtractionBytes(60, diskOnlyBudget); + try { + expect(() => reserveHdrExtractionBytes(40, diskOnlyBudget)).toThrow( + "Concurrent HDR pre-extractions", + ); + } finally { + releaseFirstJob(); + } + }); +}); + +describe("extractHdrVideoFrames", () => { + it("pins FFmpeg seek/duration, raw frame count, and reservation lifetime", async () => { + const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-extract-")); + const video = hdrVideo("preroll", { start: -1, end: 60, mediaStart: 7 }); + const fixture = hdrExtractionFixture([video], framesDir); + const calls: string[][] = []; + + try { + const extracted = await extractHdrVideoFrames({ + ...fixture, + runFfmpegImpl: async (args) => { + calls.push(args); + const rawPath = args.at(-1); + if (!rawPath) throw new Error("mock FFmpeg output path missing"); + // start=-1 through composition end=2 => 3s * 2fps = 6 rgb48le 1x1 frames. + writeFileSync(rawPath, Buffer.alloc(6 * 6)); + return ffmpegResult(true); + }, + }); + try { + expect(calls).toHaveLength(1); + const args = calls[0] ?? []; + expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual(["-ss", "7"]); + expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "3"]); + expect(extracted.sources.get("preroll")?.frameCount).toBe(6); + expect(extracted.estimatedBytes).toBe(36); + expect(getHdrExtractionReservedBytes()).toBe(36); + } finally { + for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source); + extracted.releaseReservation(); + } + } finally { + rmSync(framesDir, { recursive: true, force: true }); + } + }); + + it("closes/removes completed and partial sources and releases reservation on failure", async () => { + const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-partial-")); + const fixture = hdrExtractionFixture([hdrVideo("first"), hdrVideo("second")], framesDir); + const createdRawPaths: string[] = []; + let call = 0; + + try { + await expect( + extractHdrVideoFrames({ + ...fixture, + runFfmpegImpl: async (args) => { + call += 1; + const rawPath = args.at(-1); + if (!rawPath) throw new Error("mock FFmpeg output path missing"); + createdRawPaths.push(rawPath); + if (call === 2) return ffmpegResult(false); + writeFileSync(rawPath, Buffer.alloc(4 * 6)); + return ffmpegResult(true); + }, + }), + ).rejects.toThrow('HDR frame extraction failed for video "second"'); + + expect(fixture.hdrDiagnostics.videoExtractionFailures).toBe(1); + expect(createdRawPaths).toHaveLength(2); + for (const rawPath of createdRawPaths) expect(existsSync(dirname(rawPath))).toBe(false); + expect(getHdrExtractionReservedBytes()).toBe(0); + } finally { + rmSync(framesDir, { recursive: true, force: true }); + } + }); +}); + +describe("cleanupHdrVideoFrameSource", () => { + it("closes the raw descriptor and immediately removes its directory", () => { + const dir = mkdtempSync(join(tmpdir(), "hf-hdr-cleanup-")); + const rawPath = join(dir, "frames.rgb48le"); + writeFileSync(rawPath, Buffer.alloc(12)); + const fd = openSync(rawPath, constants.O_RDONLY); + + cleanupHdrVideoFrameSource({ + dir, + rawPath, + fd, + width: 1, + height: 2, + frameSize: 12, + frameCount: 1, + scratch: Buffer.alloc(12), + }); + + expect(existsSync(dir)).toBe(false); + expect(() => closeSync(fd)).toThrow(); + }); + + it("closes the descriptor but retains raw files with KEEP_TEMP=1", () => { + vi.stubEnv("KEEP_TEMP", "1"); + const dir = mkdtempSync(join(tmpdir(), "hf-hdr-keep-temp-")); + const rawPath = join(dir, "frames.rgb48le"); + writeFileSync(rawPath, Buffer.alloc(12)); + const fd = openSync(rawPath, constants.O_RDONLY); + + try { + cleanupHdrVideoFrameSource({ + dir, + rawPath, + fd, + width: 1, + height: 2, + frameSize: 12, + frameCount: 1, + scratch: Buffer.alloc(12), + }); + + expect(existsSync(rawPath)).toBe(true); + expect(() => closeSync(fd)).toThrow(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.ts index fd65930a755..576088daf93 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.ts @@ -24,12 +24,14 @@ import { mkdtempSync, openSync, readFileSync, + rmSync, statfsSync, } from "node:fs"; import { join } from "node:path"; import { type CaptureSession, decodePngToRgb48le, + getCgroupMemoryLimitMb, normalizeObjectFit, queryElementStacking, resampleRgb48leObjectFit, @@ -37,7 +39,11 @@ import { } from "@hyperframes/engine"; import { fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core"; import type { ProducerLogger } from "../../../logger.js"; -import type { HdrImageBuffer, HdrVideoFrameSource } from "../../hdrCompositor.js"; +import { + closeHdrVideoFrameSource, + type HdrImageBuffer, + type HdrVideoFrameSource, +} from "../../hdrCompositor.js"; import type { HdrDiagnostics, RenderJob } from "../../renderOrchestrator.js"; import type { CompositionMetadata } from "../shared.js"; @@ -107,6 +113,7 @@ export function planHdrResources(args: { * probe for HDR images whose `data-start` instant reports zero dims (GSAP * `from` tweens animate the element in slightly later). */ +// fallow-ignore-next-line complexity code-duplication export async function probeHdrExtractionDims(args: { domSession: CaptureSession; nativeHdrIds: Set; @@ -186,7 +193,132 @@ export function estimateHdrExtractionBytes( } const HDR_EXTRACTION_HEADROOM_FRACTION = 0.9; +const HDR_EXTRACTION_CGROUP_BUDGET_FRACTION = 0.5; const HDR_EXTRACTION_WARN_BYTES = 10e9; +const HDR_EXTRACTION_MAX_BYTES_ENV = "HDR_EXTRACTION_MAX_BYTES"; +const BYTES_PER_MIB = 1024 * 1024; +let aggregateHdrExtractionReservedBytes = 0; + +export function resolveHdrExtractionBudgetBytes( + raw: string | undefined, + cgroupLimitMb: number | null = getCgroupMemoryLimitMb(), +): number | undefined { + let configuredBudget: number | undefined; + if (raw !== undefined && raw.trim() !== "") { + const value = Number(raw); + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${HDR_EXTRACTION_MAX_BYTES_ENV} must be a positive finite byte count`); + } + configuredBudget = Math.floor(value); + } + const cgroupBudget = + cgroupLimitMb !== null && Number.isFinite(cgroupLimitMb) && cgroupLimitMb > 0 + ? Math.floor(cgroupLimitMb * BYTES_PER_MIB * HDR_EXTRACTION_CGROUP_BUDGET_FRACTION) + : undefined; + if (configuredBudget === undefined) return cgroupBudget; + if (cgroupBudget === undefined) return configuredBudget; + return Math.min(configuredBudget, cgroupBudget); +} + +export function resolveHdrExtractionActiveBudgetBytes( + configuredBudgetBytes: number | undefined, + freeBytes: number, +): number { + const diskBudgetBytes = Math.floor(freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION); + return configuredBudgetBytes === undefined + ? diskBudgetBytes + : Math.min(configuredBudgetBytes, diskBudgetBytes); +} + +export function reserveHdrExtractionBytes( + estimatedBytes: number, + budgetBytes: number | undefined, +): () => void { + if (!Number.isFinite(estimatedBytes) || estimatedBytes < 0) { + throw new Error(`HDR extraction reservation must be a finite non-negative byte count`); + } + const aggregateBytes = aggregateHdrExtractionReservedBytes + estimatedBytes; + if (budgetBytes !== undefined && aggregateBytes > budgetBytes) { + throw new Error( + `Concurrent HDR pre-extractions need ~${(aggregateBytes / 1e9).toFixed(1)} GB of raw ` + + `16-bit frame scratch, exceeding the active ${(budgetBytes / 1e9).toFixed(1)} GB budget.`, + ); + } + aggregateHdrExtractionReservedBytes = aggregateBytes; + let released = false; + return () => { + if (released) return; + released = true; + aggregateHdrExtractionReservedBytes = Math.max( + 0, + aggregateHdrExtractionReservedBytes - estimatedBytes, + ); + }; +} + +export function getHdrExtractionReservedBytes(): number { + return aggregateHdrExtractionReservedBytes; +} + +export interface HdrExtractionWindow { + compositionStart: number; + mediaStart: number; + durationSeconds: number; +} + +export function resolveHdrExtractionWindow( + video: { id: string; start: number; end: number; mediaStart: number }, + compositionDuration: number, +): HdrExtractionWindow { + if (!Number.isFinite(compositionDuration) || compositionDuration <= 0) { + throw new Error( + `Cannot extract HDR video "${video.id}" with invalid composition duration ${String(compositionDuration)}`, + ); + } + const requestedEnd = + Number.isFinite(video.end) && video.end > video.start ? video.end : compositionDuration; + const effectiveEnd = Math.min(requestedEnd, compositionDuration); + const compositionStart = video.start; + const durationSeconds = effectiveEnd - video.start; + const mediaStart = video.mediaStart; + if ( + !Number.isFinite(durationSeconds) || + durationSeconds <= 0 || + !Number.isFinite(mediaStart) || + mediaStart < 0 + ) { + throw new Error( + `HDR video "${video.id}" has no finite interval inside the ${compositionDuration}s composition`, + ); + } + return { compositionStart, mediaStart, durationSeconds }; +} + +function cleanupHdrFrameDirectory( + frameDir: string, + rawPath: string | undefined, + log?: ProducerLogger, +): void { + if (process.env.KEEP_TEMP === "1") return; + try { + rmSync(frameDir, { recursive: true, force: true }); + } catch (err) { + log?.warn("Failed to clean up HDR raw frame directory", { + frameDir, + rawPath, + error: err instanceof Error ? err.message : String(err), + }); + } +} + +/** Close the raw-frame descriptor and release its directory unless KEEP_TEMP=1. */ +export function cleanupHdrVideoFrameSource( + source: HdrVideoFrameSource, + log?: ProducerLogger, +): void { + closeHdrVideoFrameSource(source, log); + cleanupHdrFrameDirectory(source.dir, source.rawPath, log); +} /** * Disk-headroom gate for raw rgb48le pre-extraction: throws if the planned @@ -202,18 +334,30 @@ function assertHdrExtractionDiskHeadroom( plannedVideos: Array<{ durationSeconds: number; width: number; height: number }>, fps: number, log: ProducerLogger, -): void { +): { estimatedBytes: number; budgetBytes: number | undefined } { const estimatedBytes = estimateHdrExtractionBytes(plannedVideos, fps); + const configuredBudget = resolveHdrExtractionBudgetBytes( + process.env[HDR_EXTRACTION_MAX_BYTES_ENV], + ); + const estimatedGb = (estimatedBytes / 1e9).toFixed(1); + if (configuredBudget !== undefined && estimatedBytes > configuredBudget) { + throw new Error( + `HDR pre-extraction needs ~${estimatedGb} GB of raw 16-bit frames, exceeding the ` + + `active scratch budget of ${(configuredBudget / 1e9).toFixed(1)} GB. ` + + `If the composition doesn't need HDR output, re-run with --sdr; otherwise reduce its ` + + `HDR duration/resolution or use a render container with a larger memory limit.`, + ); + } let freeBytes: number; try { const stat = statfsSync(framesDir); freeBytes = stat.bavail * stat.bsize; } catch { // statfs unsupported on this platform/filesystem — skip the gate. - return; + return { estimatedBytes, budgetBytes: configuredBudget }; } - const estimatedGb = (estimatedBytes / 1e9).toFixed(1); - if (estimatedBytes > freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION) { + const diskBudgetBytes = Math.floor(freeBytes * HDR_EXTRACTION_HEADROOM_FRACTION); + if (estimatedBytes > diskBudgetBytes) { throw new Error( `HDR pre-extraction needs ~${estimatedGb} GB of raw 16-bit frames but only ` + `${(freeBytes / 1e9).toFixed(1)} GB is free at ${framesDir}. ` + @@ -228,13 +372,25 @@ function assertHdrExtractionDiskHeadroom( { estimatedBytes, freeBytes }, ); } + return { + estimatedBytes, + budgetBytes: resolveHdrExtractionActiveBudgetBytes(configuredBudget, freeBytes), + }; +} + +export interface HdrVideoExtractionResult { + sources: Map; + estimatedBytes: number; + releaseReservation: () => void; } /** * Extract each HDR video into a raw rgb48le frame file via a single FFmpeg - * pass per video, and open a file descriptor for each. Returns a map keyed - * by video id. Caller owns lifecycle teardown (closing fds + rm-rf). + * pass per video, and open a file descriptor for each. The caller owns both + * source teardown and the aggregate scratch reservation, which intentionally + * remains held until the capture-stage finally block. */ +// fallow-ignore-next-line complexity export async function extractHdrVideoFrames(args: { job: RenderJob; log: ProducerLogger; @@ -245,90 +401,115 @@ export async function extractHdrVideoFrames(args: { height: number; abortSignal: AbortSignal | undefined; hdrDiagnostics: HdrDiagnostics; -}): Promise> { + runFfmpegImpl?: typeof runFfmpeg; +}): Promise { const { job, log, framesDir, composition, prep, width, height, abortSignal, hdrDiagnostics } = args; + const runFfmpegImpl = args.runFfmpegImpl ?? runFfmpeg; const out = new Map(); mkdirSync(framesDir, { recursive: true }); const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = []; + const extractionWindows = new Map(); for (const [videoId] of prep.hdrVideoSrcPaths) { const video = composition.videos.find((v) => v.id === videoId); if (!video) continue; const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; + const window = resolveHdrExtractionWindow(video, composition.duration); + extractionWindows.set(videoId, window); plannedVideos.push({ - durationSeconds: video.end - video.start, + durationSeconds: window.durationSeconds, width: dims.width, height: dims.height, }); } - assertHdrExtractionDiskHeadroom(framesDir, plannedVideos, fpsToNumber(job.config.fps), log); - for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) { - const video = composition.videos.find((v) => v.id === videoId); - if (!video) continue; - mkdirSync(framesDir, { recursive: true }); - const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`)); - const duration = video.end - video.start; - const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; - const rawPath = join(frameDir, "frames.rgb48le"); - const ffmpegArgs = [ - "-ss", - String(video.mediaStart), - "-i", - srcPath, - "-t", - String(duration), - "-r", - fpsToFfmpegArg(job.config.fps), - "-vf", - `scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}`, - "-pix_fmt", - "rgb48le", - "-f", - "rawvideo", - "-y", - rawPath, - ]; - const result = await runFfmpeg(ffmpegArgs, { signal: abortSignal }); - if (!result.success) { - hdrDiagnostics.videoExtractionFailures += 1; - log.error("HDR frame pre-extraction failed; aborting render", { - videoId, + const { estimatedBytes, budgetBytes } = assertHdrExtractionDiskHeadroom( + framesDir, + plannedVideos, + fpsToNumber(job.config.fps), + log, + ); + const releaseReservation = reserveHdrExtractionBytes(estimatedBytes, budgetBytes); + const createdFrameDirs = new Set(); + try { + for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) { + const video = composition.videos.find((v) => v.id === videoId); + const window = extractionWindows.get(videoId); + if (!video || !window) continue; + mkdirSync(framesDir, { recursive: true }); + const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`)); + createdFrameDirs.add(frameDir); + const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; + const rawPath = join(frameDir, "frames.rgb48le"); + const ffmpegArgs = [ + "-ss", + String(window.mediaStart), + "-i", srcPath, - stderr: result.stderr.slice(-400), - }); - throw new Error( - `HDR frame extraction failed for video "${videoId}". ` + - `Aborting render to avoid shipping black HDR layers.`, - ); - } - const frameSize = dims.width * dims.height * 6; - const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG); - let handedOff = false; - try { - const frameCount = Math.floor(fstatSync(fd).size / frameSize); - if (frameCount < 1) { + "-t", + String(window.durationSeconds), + "-r", + fpsToFfmpegArg(job.config.fps), + "-vf", + `scale=${dims.width}:${dims.height}:force_original_aspect_ratio=increase,crop=${dims.width}:${dims.height}`, + "-pix_fmt", + "rgb48le", + "-f", + "rawvideo", + "-y", + rawPath, + ]; + const result = await runFfmpegImpl(ffmpegArgs, { signal: abortSignal }); + if (!result.success) { hdrDiagnostics.videoExtractionFailures += 1; + log.error("HDR frame pre-extraction failed; aborting render", { + videoId, + srcPath, + stderr: result.stderr.slice(-400), + }); throw new Error( - `HDR frame extraction produced no frames for video "${videoId}". ` + + `HDR frame extraction failed for video "${videoId}". ` + `Aborting render to avoid shipping black HDR layers.`, ); } - out.set(videoId, { - dir: frameDir, - rawPath, - fd, - width: dims.width, - height: dims.height, - frameSize, - frameCount, - scratch: Buffer.allocUnsafe(frameSize), - }); - handedOff = true; - } finally { - if (!handedOff) closeSync(fd); + const frameSize = dims.width * dims.height * 6; + const fd = openSync(rawPath, constants.O_RDONLY | NO_FOLLOW_FLAG); + let handedOff = false; + try { + const frameCount = Math.floor(fstatSync(fd).size / frameSize); + if (frameCount < 1) { + hdrDiagnostics.videoExtractionFailures += 1; + throw new Error( + `HDR frame extraction produced no frames for video "${videoId}". ` + + `Aborting render to avoid shipping black HDR layers.`, + ); + } + out.set(videoId, { + dir: frameDir, + rawPath, + fd, + width: dims.width, + height: dims.height, + frameSize, + frameCount, + scratch: Buffer.allocUnsafe(frameSize), + }); + handedOff = true; + } finally { + if (!handedOff) closeSync(fd); + } + } + return { sources: out, estimatedBytes, releaseReservation }; + } catch (error) { + for (const source of out.values()) { + cleanupHdrVideoFrameSource(source, log); + createdFrameDirs.delete(source.dir); } + for (const frameDir of createdFrameDirs) { + cleanupHdrFrameDirectory(frameDir, undefined, log); + } + releaseReservation(); + throw error; } - return out; } /** diff --git a/packages/producer/src/services/render/stages/captureHdrStage.ts b/packages/producer/src/services/render/stages/captureHdrStage.ts index 5a0e44bd2ca..d7f168ab2aa 100644 --- a/packages/producer/src/services/render/stages/captureHdrStage.ts +++ b/packages/producer/src/services/render/stages/captureHdrStage.ts @@ -60,7 +60,6 @@ import { type HdrTransitionMeta, type HdrVideoFrameSource, type TransitionRange, - closeHdrVideoFrameSource, resolveCompositeTransfer, } from "../../hdrCompositor.js"; import { type HdrPerfCollector, createHdrPerfCollector } from "../hdrPerf.js"; @@ -68,6 +67,7 @@ import type { HdrDiagnostics, ProgressCallback, RenderJob } from "../../renderOr import type { CompositionMetadata } from "../shared.js"; import { decodeHdrImageBuffers, + cleanupHdrVideoFrameSource, extractHdrVideoFrames, planHdrResources, probeHdrExtractionDims, @@ -129,6 +129,7 @@ export interface CaptureHdrStageResult { warnings: CaptureWarning[]; } +// fallow-ignore-next-line complexity export async function runCaptureHdrStage( input: CaptureHdrStageInput, ): Promise { @@ -215,6 +216,7 @@ export async function runCaptureHdrStage( let hdrEncoder: StreamingEncoder | null = null; let hdrEncoderClosed = false; let domSessionClosed = false; + let releaseHdrExtractionReservation: (() => void) | null = null; const hdrVideoFrameSources = new Map(); try { await initializeSession(domSession); @@ -296,7 +298,8 @@ export async function runCaptureHdrStage( abortSignal, hdrDiagnostics, }); - for (const [id, source] of extracted) hdrVideoFrameSources.set(id, source); + releaseHdrExtractionReservation = extracted.releaseReservation; + for (const [id, source] of extracted.sources) hdrVideoFrameSources.set(id, source); const hdrImageBuffers = decodeHdrImageBuffers({ log, hdrImageSrcPaths, @@ -454,9 +457,11 @@ export async function runCaptureHdrStage( }); } for (const frameSource of hdrVideoFrameSources.values()) { - closeHdrVideoFrameSource(frameSource, log); + cleanupHdrVideoFrameSource(frameSource, log); } hdrVideoFrameSources.clear(); + releaseHdrExtractionReservation?.(); + releaseHdrExtractionReservation = null; } return { diff --git a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts index 326c9d1b51c..2b552873ef3 100644 --- a/packages/producer/src/services/render/stages/captureStreamingStage.test.ts +++ b/packages/producer/src/services/render/stages/captureStreamingStage.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { describe, expect, it, mock } from "bun:test"; import { getCaptureStageBrowserConsole } from "../captureStageError.js"; import { createCapturePlan } from "../capturePlan.js"; @@ -131,8 +132,13 @@ mock.module("../../hdrCompositor.js", () => ({ })); mock.module("./captureHdrResources.js", () => ({ + cleanupHdrVideoFrameSource: () => {}, decodeHdrImageBuffers: () => new Map(), - extractHdrVideoFrames: async () => new Map(), + extractHdrVideoFrames: async () => ({ + sources: new Map(), + estimatedBytes: 0, + releaseReservation: () => {}, + }), planHdrResources: () => ({ hdrVideoStartTimes: new Map(), nativeHdrVideos: [], diff --git a/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts new file mode 100644 index 00000000000..d6a27a957ed --- /dev/null +++ b/packages/producer/src/services/render/stages/extractVideosStage.timelineBound.test.ts @@ -0,0 +1,120 @@ +import { resolveConfig, type ExtractionResult, type VideoElement } from "@hyperframes/engine"; +import { describe, expect, it, vi } from "vitest"; + +const extractionCalls = vi.hoisted( + () => new Array<{ timelineEnd: number | undefined; durationSeconds: number }>(), +); +const fixtureState = vi.hoisted(() => ({ sourceDurationSeconds: 60 })); + +vi.mock("@hyperframes/engine", async (importOriginal) => { + const real = await importOriginal(); + return { + ...real, + extractAllVideoFrames: async ( + videos: VideoElement[], + _baseDir: string, + options: { timelineEnd?: number }, + ): Promise => { + const sourceDurationSeconds = fixtureState.sourceDurationSeconds; + const video = videos[0]; + if (!video) throw new Error("timeline-bound fixture requires one video"); + const requestedDuration = video.end - video.start; + const naturalDuration = sourceDurationSeconds - video.mediaStart; + const resolvedDuration = + Number.isFinite(requestedDuration) && requestedDuration > 0 + ? requestedDuration + : naturalDuration; + const durationSeconds = + options.timelineEnd === undefined + ? resolvedDuration + : Math.min(resolvedDuration, Math.max(0, options.timelineEnd - video.start)); + video.end = video.start + durationSeconds; + extractionCalls.push({ timelineEnd: options.timelineEnd, durationSeconds }); + return { + success: true, + extracted: [], + errors: [], + totalFramesExtracted: 0, + durationMs: 0, + phaseBreakdown: { + resolveMs: 0, + cachePublishFailures: 0, + cacheGcEvictions: 0, + cacheGcBytesFreed: 0, + cacheAgedPartialsCleared: 0, + hdrProbeMs: 0, + hdrPreflightMs: 0, + hdrPreflightCount: 0, + vfrProbeMs: 0, + vfrPreflightMs: 0, + vfrPreflightCount: 0, + extractMs: 0, + cacheHits: 0, + cacheMisses: 0, + transientRetries: 0, + }, + }; + }, + }; +}); + +import { createRenderJob } from "../../renderOrchestrator.js"; +import { runExtractVideosStage } from "./extractVideosStage.js"; + +async function runStage(compositionDuration: number, materializeSymlinks: boolean): Promise { + const composition = { + duration: compositionDuration, + videos: [ + { + id: "root-video", + src: "long.mp4", + start: 0, + end: Number.POSITIVE_INFINITY, + mediaStart: 0, + loop: false, + hasAudio: false, + }, + ], + audios: [], + images: [], + width: 1920, + height: 1080, + }; + await runExtractVideosStage({ + projectDir: "/tmp/hf-timeline-bound-project", + compiledDir: "/tmp/hf-timeline-bound-compiled", + job: createRenderJob({ + fps: { num: 30, den: 1 }, + quality: "standard", + hdrMode: "force-sdr", + }), + cfg: resolveConfig(), + composition, + abortSignal: undefined, + assertNotAborted: () => {}, + materializeSymlinks, + }); +} + +describe.each([ + ["in-process", false], + ["distributed plan", true], +] as const)("%s video extraction timeline bound", (_mode, materializeSymlinks) => { + it("caps an open 60-second source to a two-second composition", async () => { + extractionCalls.splice(0); + fixtureState.sourceDurationSeconds = 60; + + await runStage(2, materializeSymlinks); + + expect(extractionCalls).toEqual([{ timelineEnd: 2, durationSeconds: 2 }]); + }); + + it("keeps a two-second natural source inside a ten-second composition", async () => { + extractionCalls.splice(0); + fixtureState.sourceDurationSeconds = 2; + + await runStage(10, materializeSymlinks); + + expect(extractionCalls).toEqual([{ timelineEnd: 10, durationSeconds: 2 }]); + }); +}); diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts index 5b0248be613..12db159a983 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.ts @@ -376,6 +376,7 @@ export async function runExtractVideosStage( fps: fpsToNumber(job.config.fps), outputDir: join(compiledDir, "__hyperframes_video_frames"), format: job.config.videoFrameFormat ?? "auto", + timelineEnd: composition.duration, maxTransientRetries: extractionPolicy.maxTransientRetries, collectProbeFailures: extractionPolicy.failureMode === "enforce", }, From ba55214dd98c8c5d2324ae51f42d843ee8c18066 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 2 Aug 2026 21:12:01 +0000 Subject: [PATCH 02/13] fix: trim negative video extraction preroll --- packages/engine/src/index.ts | 3 + .../src/services/videoFrameExtractor.test.ts | 13 +++- .../src/services/videoFrameExtractor.ts | 74 +++++++++++++++---- .../render/stages/captureHdrResources.test.ts | 25 ++++--- .../render/stages/captureHdrResources.ts | 28 +++---- 5 files changed, 99 insertions(+), 44 deletions(-) diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 6793688dea7..886ec734d78 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -180,6 +180,8 @@ export { parseImageElements, extractVideoFramesRange, extractAllVideoFrames, + resolveTimelineExtractionWindow, + resolveVideoExtractionWindow, resolveVideoExtractionDuration, resolveProjectRelativeSrc, getFrameAtTime, @@ -196,6 +198,7 @@ export { type ExtractionOptions, type ExtractionResult, type ExtractionPhaseBreakdown, + type TimelineExtractionWindow, type VideoExtractionFailure, type VideoExtractionFailureKind, type VideoFrameFormat, diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 9d28eb18913..c0edff92ec5 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -24,6 +24,7 @@ import { resolveFrameFormat, codecMayHaveAlpha, decoderForCodec, + resolveVideoExtractionWindow, resolveVideoExtractionDuration, getFrameAtTime, analyzeClipMediaFit, @@ -84,10 +85,14 @@ describe("resolveVideoExtractionDuration", () => { expect(explicitLoop.loop).toBe(true); }); - it("preserves negative start and mediaStart semantics", () => { - const preroll = video({ start: -3, mediaStart: 5 }); - expect(resolveVideoExtractionDuration(preroll, metadata(60), 2)).toBe(5); - expect(preroll).toMatchObject({ start: -3, mediaStart: 5 }); + it("trims materially negative preroll and advances the source offset", () => { + const preroll = video({ start: -60, end: 120, mediaStart: 0 }); + expect(resolveVideoExtractionWindow(preroll, metadata(120), 2)).toEqual({ + compositionStart: 0, + mediaStart: 60, + durationSeconds: 2, + }); + expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2); }); it("retains legacy behavior when no timeline end is supplied", () => { diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 05e69316fa3..a8592956347 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -85,8 +85,8 @@ export interface ExtractionOptions { sdrToHdrTransfer?: HdrTransfer; /** * Absolute composition/timeline end in seconds. Applied only after source - * metadata resolves open-ended/natural-duration media, so authored start, - * mediaStart, loop, and sentinel semantics remain unchanged. + * metadata resolves open-ended/natural-duration media. Invisible negative + * preroll is trimmed while advancing mediaStart to preserve source alignment. */ timelineEnd?: number; /** @@ -734,22 +734,61 @@ function resolveSegmentDuration( return sourceRemaining > 0 ? sourceRemaining : metadata.durationSeconds; } -/** Resolve source duration first, then cap it at the remaining render timeline. */ -export function resolveVideoExtractionDuration( +export interface TimelineExtractionWindow { + compositionStart: number; + mediaStart: number; + durationSeconds: number; +} + +/** Intersect a resolved source interval with the render timeline. */ +export function resolveTimelineExtractionWindow( + video: Pick, + resolvedDuration: number, + timelineEnd?: number, +): TimelineExtractionWindow { + if (timelineEnd === undefined) { + return { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: resolvedDuration, + }; + } + if (!Number.isFinite(timelineEnd)) { + throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); + } + const compositionStart = Math.max(0, video.start); + const trimmedPreroll = compositionStart - video.start; + const durationSeconds = Math.max( + 0, + Math.min(resolvedDuration - trimmedPreroll, timelineEnd - compositionStart), + ); + return { + compositionStart, + mediaStart: video.mediaStart + trimmedPreroll, + durationSeconds, + }; +} + +/** Resolve source duration first, then intersect it with the render timeline. */ +export function resolveVideoExtractionWindow( video: Pick, metadata: VideoMetadata, timelineEnd?: number, -): number { +): TimelineExtractionWindow { const resolvedDuration = resolveSegmentDuration( video.end - video.start, video.mediaStart, metadata, ); - if (timelineEnd === undefined) return resolvedDuration; - if (!Number.isFinite(timelineEnd)) { - throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`); - } - return Math.min(resolvedDuration, Math.max(0, timelineEnd - video.start)); + return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd); +} + +export function resolveVideoExtractionDuration( + video: Pick, + metadata: VideoMetadata, + timelineEnd?: number, +): number { + return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds; } /** @@ -1146,10 +1185,11 @@ export async function extractAllVideoFrames( breakdown.resolveMs = Date.now() - phase1Start; // Snapshot the pre-preflight key inputs so the extraction cache keys on the - // user-visible source (original path and mediaStart) rather than the + // user-visible source path rather than the // workDir-local normalized file produced by the // HDR preflight. Without this, every render would write a new // normalized file with a fresh mtime → fresh cache key → perpetual misses. + // Phase 3 updates mediaStart after trimming any invisible negative preroll. const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => { const stat = readKeyStat(videoPath); // Missing files return null — skip the cache path for that entry. The @@ -1542,10 +1582,16 @@ export async function extractAllVideoFrames( } try { const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath)); - const videoDuration = resolveVideoExtractionDuration(video, metadata, options.timelineEnd); - if (video.end - video.start !== videoDuration) { - video.end = video.start + videoDuration; + const window = resolveVideoExtractionWindow(video, metadata, options.timelineEnd); + const videoDuration = window.durationSeconds; + if (videoDuration <= 0) { + throw new Error(`Video "${video.id}" has no interval inside the render timeline`); } + video.start = window.compositionStart; + video.end = window.compositionStart + videoDuration; + video.mediaStart = window.mediaStart; + const keyInput = cacheKeyInputs[index]; + if (keyInput) keyInput.mediaStart = window.mediaStart; const format = resolveFrameFormat(metadata, options.format); const sdrToHdrTransfer = sdrToHdrTransfers[index]; diff --git a/packages/producer/src/services/render/stages/captureHdrResources.test.ts b/packages/producer/src/services/render/stages/captureHdrResources.test.ts index 27bcb315171..d91b25dfb41 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.test.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.test.ts @@ -130,10 +130,10 @@ describe("resolveHdrExtractionWindow", () => { }); }); - it("preserves negative starts and source offsets while bounding the timeline end", () => { - expect(resolveHdrExtractionWindow({ id: "hdr", start: -3, end: 60, mediaStart: 5 }, 2)).toEqual( - { compositionStart: -3, mediaStart: 5, durationSeconds: 5 }, - ); + it("trims materially negative preroll and advances the source offset", () => { + expect( + resolveHdrExtractionWindow({ id: "hdr", start: -60, end: 120, mediaStart: 0 }, 2), + ).toEqual({ compositionStart: 0, mediaStart: 60, durationSeconds: 2 }); }); it("rejects HDR media with no interval inside the composition", () => { @@ -197,7 +197,7 @@ describe("reserveHdrExtractionBytes", () => { describe("extractHdrVideoFrames", () => { it("pins FFmpeg seek/duration, raw frame count, and reservation lifetime", async () => { const framesDir = mkdtempSync(join(tmpdir(), "hf-hdr-extract-")); - const video = hdrVideo("preroll", { start: -1, end: 60, mediaStart: 7 }); + const video = hdrVideo("preroll", { start: -60, end: 120, mediaStart: 0 }); const fixture = hdrExtractionFixture([video], framesDir); const calls: string[][] = []; @@ -208,19 +208,20 @@ describe("extractHdrVideoFrames", () => { calls.push(args); const rawPath = args.at(-1); if (!rawPath) throw new Error("mock FFmpeg output path missing"); - // start=-1 through composition end=2 => 3s * 2fps = 6 rgb48le 1x1 frames. - writeFileSync(rawPath, Buffer.alloc(6 * 6)); + // The visible [0, 2] interval is two seconds at 2fps = 4 rgb48le 1x1 frames. + writeFileSync(rawPath, Buffer.alloc(4 * 6)); return ffmpegResult(true); }, }); try { expect(calls).toHaveLength(1); const args = calls[0] ?? []; - expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual(["-ss", "7"]); - expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "3"]); - expect(extracted.sources.get("preroll")?.frameCount).toBe(6); - expect(extracted.estimatedBytes).toBe(36); - expect(getHdrExtractionReservedBytes()).toBe(36); + expect(args.slice(args.indexOf("-ss"), args.indexOf("-ss") + 2)).toEqual(["-ss", "60"]); + expect(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2)).toEqual(["-t", "2"]); + expect(fixture.prep.hdrVideoStartTimes.get("preroll")).toBe(0); + expect(extracted.sources.get("preroll")?.frameCount).toBe(4); + expect(extracted.estimatedBytes).toBe(24); + expect(getHdrExtractionReservedBytes()).toBe(24); } finally { for (const source of extracted.sources.values()) cleanupHdrVideoFrameSource(source); extracted.releaseReservation(); diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.ts index 576088daf93..b5689b4643a 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.ts @@ -35,7 +35,9 @@ import { normalizeObjectFit, queryElementStacking, resampleRgb48leObjectFit, + resolveTimelineExtractionWindow, runFfmpeg, + type TimelineExtractionWindow, } from "@hyperframes/engine"; import { fpsToFfmpegArg, fpsToNumber } from "@hyperframes/core"; import type { ProducerLogger } from "../../../logger.js"; @@ -260,11 +262,7 @@ export function getHdrExtractionReservedBytes(): number { return aggregateHdrExtractionReservedBytes; } -export interface HdrExtractionWindow { - compositionStart: number; - mediaStart: number; - durationSeconds: number; -} +export type HdrExtractionWindow = TimelineExtractionWindow; export function resolveHdrExtractionWindow( video: { id: string; start: number; end: number; mediaStart: number }, @@ -277,21 +275,22 @@ export function resolveHdrExtractionWindow( } const requestedEnd = Number.isFinite(video.end) && video.end > video.start ? video.end : compositionDuration; - const effectiveEnd = Math.min(requestedEnd, compositionDuration); - const compositionStart = video.start; - const durationSeconds = effectiveEnd - video.start; - const mediaStart = video.mediaStart; + const window = resolveTimelineExtractionWindow( + video, + requestedEnd - video.start, + compositionDuration, + ); if ( - !Number.isFinite(durationSeconds) || - durationSeconds <= 0 || - !Number.isFinite(mediaStart) || - mediaStart < 0 + !Number.isFinite(window.durationSeconds) || + window.durationSeconds <= 0 || + !Number.isFinite(window.mediaStart) || + window.mediaStart < 0 ) { throw new Error( `HDR video "${video.id}" has no finite interval inside the ${compositionDuration}s composition`, ); } - return { compositionStart, mediaStart, durationSeconds }; + return window; } function cleanupHdrFrameDirectory( @@ -416,6 +415,7 @@ export async function extractHdrVideoFrames(args: { const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; const window = resolveHdrExtractionWindow(video, composition.duration); extractionWindows.set(videoId, window); + prep.hdrVideoStartTimes.set(videoId, window.compositionStart); plannedVideos.push({ durationSeconds: window.durationSeconds, width: dims.width, From 297a3942a63128b84a7fef51f9b01d47b44988a1 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 2 Aug 2026 21:19:42 +0000 Subject: [PATCH 03/13] fix: skip invisible video extraction windows --- .../src/services/videoFrameExtractor.test.ts | 28 ++++++++++++++++ .../src/services/videoFrameExtractor.ts | 33 +++++++++++++------ .../render/stages/captureHdrResources.test.ts | 10 +++--- .../render/stages/captureHdrResources.ts | 14 ++++---- 4 files changed, 64 insertions(+), 21 deletions(-) diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index c0edff92ec5..84de6249834 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -95,6 +95,12 @@ describe("resolveVideoExtractionDuration", () => { expect(resolveVideoExtractionDuration(preroll, metadata(120), 2)).toBe(2); }); + it("returns an empty window for a clip entirely before composition time zero", () => { + expect(resolveVideoExtractionWindow(video({ start: -60, end: -10 }), metadata(120), 2)).toEqual( + { compositionStart: 0, mediaStart: 60, durationSeconds: 0 }, + ); + }); + it("retains legacy behavior when no timeline end is supplied", () => { expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60); }); @@ -1072,6 +1078,28 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { if (existsSync(FIXTURE_DIR)) rmSync(FIXTURE_DIR, { recursive: true, force: true }); }); + it("skips a clip entirely before time zero without reporting an extraction error", async () => { + const outputDir = join(FIXTURE_DIR, "out-before-timeline"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "before-timeline", + src: VFR_FIXTURE, + start: -2, + end: -1, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result).toMatchObject({ success: true, extracted: [], errors: [] }); + }); + it("detects the synthesized fixture as VFR", async () => { const md = await extractVideoMetadata(VFR_FIXTURE); expect(md.isVFR).toBe(true); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index a8592956347..37dd4c1ae77 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -1360,7 +1360,10 @@ export async function extractAllVideoFrames( }; } - type PreparedExtractionResult = { work: PreparedExtraction } | { error: VideoExtractionFailure }; + type PreparedExtractionResult = + | { work: PreparedExtraction } + | { error: VideoExtractionFailure } + | { skipped: true }; type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure }; @@ -1585,7 +1588,7 @@ export async function extractAllVideoFrames( const window = resolveVideoExtractionWindow(video, metadata, options.timelineEnd); const videoDuration = window.durationSeconds; if (videoDuration <= 0) { - throw new Error(`Video "${video.id}" has no interval inside the render timeline`); + return { skipped: true }; } video.start = window.compositionStart; video.end = window.compositionStart + videoDuration; @@ -1649,11 +1652,20 @@ export async function extractAllVideoFrames( for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome); } - const results: ExtractionOutcome[] = preparedExtractions.map((prepared) => { - if ("error" in prepared) return prepared; + const results: ExtractionOutcome[] = []; + for (const prepared of preparedExtractions) { + if ("skipped" in prepared) continue; + if ("error" in prepared) { + results.push(prepared); + continue; + } const outcome = uniqueOutcomes.get(prepared.work.dedupeKey); - if (!outcome) - return { error: extractionError(prepared.work.video.id, "missing extraction result") }; + if (!outcome) { + results.push({ + error: extractionError(prepared.work.video.id, "missing extraction result"), + }); + continue; + } if ("error" in outcome) { // A shared (deduped/superset) failure fans out to every element with the // same key; annotate followers with the leader's videoId so N copies of @@ -1662,17 +1674,18 @@ export async function extractAllVideoFrames( const message = isFollower ? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}` : outcome.error.error; - return { + results.push({ error: { videoId: prepared.work.video.id, kind: outcome.error.kind, retryable: outcome.error.retryable, error: message, }, - }; + }); + continue; } - return { result: { ...outcome.result, videoId: prepared.work.video.id } }; - }); + results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } }); + } breakdown.extractMs = Date.now() - phase3Start; diff --git a/packages/producer/src/services/render/stages/captureHdrResources.test.ts b/packages/producer/src/services/render/stages/captureHdrResources.test.ts index d91b25dfb41..6a4757f08e9 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.test.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.test.ts @@ -112,10 +112,12 @@ describe("estimateHdrExtractionBytes", () => { describe("resolveHdrExtractionWindow", () => { it("bounds a two-second composition backed by an unbounded HDR source to 60 raw frames", () => { - const { durationSeconds } = resolveHdrExtractionWindow( + const window = resolveHdrExtractionWindow( { id: "long-hdr", start: 0, end: Number.POSITIVE_INFINITY, mediaStart: 0 }, 2, ); + if (!window) throw new Error("expected a visible HDR extraction window"); + const { durationSeconds } = window; expect(durationSeconds).toBe(2); expect(estimateHdrExtractionBytes([{ durationSeconds, width: 3840, height: 2160 }], 30)).toBe( 60 * 3840 * 2160 * 6, @@ -136,10 +138,10 @@ describe("resolveHdrExtractionWindow", () => { ).toEqual({ compositionStart: 0, mediaStart: 60, durationSeconds: 2 }); }); - it("rejects HDR media with no interval inside the composition", () => { - expect(() => + it("skips HDR media with no interval inside the composition", () => { + expect( resolveHdrExtractionWindow({ id: "hdr", start: 3, end: Infinity, mediaStart: 0 }, 2), - ).toThrow('HDR video "hdr" has no finite interval inside the 2s composition'); + ).toBeNull(); }); }); diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.ts index b5689b4643a..5a39ac28cae 100644 --- a/packages/producer/src/services/render/stages/captureHdrResources.ts +++ b/packages/producer/src/services/render/stages/captureHdrResources.ts @@ -267,7 +267,7 @@ export type HdrExtractionWindow = TimelineExtractionWindow; export function resolveHdrExtractionWindow( video: { id: string; start: number; end: number; mediaStart: number }, compositionDuration: number, -): HdrExtractionWindow { +): HdrExtractionWindow | null { if (!Number.isFinite(compositionDuration) || compositionDuration <= 0) { throw new Error( `Cannot extract HDR video "${video.id}" with invalid composition duration ${String(compositionDuration)}`, @@ -280,16 +280,15 @@ export function resolveHdrExtractionWindow( requestedEnd - video.start, compositionDuration, ); - if ( - !Number.isFinite(window.durationSeconds) || - window.durationSeconds <= 0 || - !Number.isFinite(window.mediaStart) || - window.mediaStart < 0 - ) { + if (!Number.isFinite(window.durationSeconds)) { throw new Error( `HDR video "${video.id}" has no finite interval inside the ${compositionDuration}s composition`, ); } + if (window.durationSeconds <= 0) return null; + if (!Number.isFinite(window.mediaStart) || window.mediaStart < 0) { + throw new Error(`HDR video "${video.id}" has invalid mediaStart ${String(window.mediaStart)}`); + } return window; } @@ -414,6 +413,7 @@ export async function extractHdrVideoFrames(args: { if (!video) continue; const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height }; const window = resolveHdrExtractionWindow(video, composition.duration); + if (!window) continue; extractionWindows.set(videoId, window); prep.hdrVideoStartTimes.set(videoId, window.compositionStart); plannedVideos.push({ From adedcff911d11f191717863cf17e2f519b973b70 Mon Sep 17 00:00:00 2001 From: James Date: Sun, 2 Aug 2026 23:13:35 +0000 Subject: [PATCH 04/13] fix: preserve negative-start loop and held tails --- .../src/services/videoFrameExtractor.test.ts | 98 +++++++++++++++++++ .../src/services/videoFrameExtractor.ts | 59 +++++++++-- .../producer/src/services/hdrCompositor.ts | 34 +++++-- .../render/stages/captureHdrResources.test.ts | 96 ++++++++++++++++-- .../render/stages/captureHdrResources.ts | 29 +++--- 5 files changed, 281 insertions(+), 35 deletions(-) diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 84de6249834..a5738316ca6 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -101,6 +101,46 @@ describe("resolveVideoExtractionDuration", () => { ); }); + it("preserves a short source cycle when negative preroll crosses a loop boundary", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: true }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: -5, + mediaStart: 0, + durationSeconds: 3, + preserveTimelinePhase: true, + }); + }); + + it("preserves source frames needed to hold a non-loop final frame", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: false }), + metadata(3), + 2, + ), + ).toEqual({ + compositionStart: -5, + mediaStart: 0, + durationSeconds: 3, + preserveTimelinePhase: true, + }); + }); + + it("rebases a loop phase when the visible window stays within one cycle", () => { + expect( + resolveVideoExtractionWindow( + video({ start: -5, end: 10, mediaStart: 0, loop: true }), + metadata(3), + 0.5, + ), + ).toEqual({ compositionStart: 0, mediaStart: 2, durationSeconds: 0.5 }); + }); + it("retains legacy behavior when no timeline end is supplied", () => { expect(resolveVideoExtractionDuration(video(), metadata(60))).toBe(60); }); @@ -1100,6 +1140,64 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { expect(result).toMatchObject({ success: true, extracted: [], errors: [] }); }); + it("preserves loop phase when negative preroll crosses the source boundary", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-loop"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-loop", + src: VFR_FIXTURE, + start: -19, + end: 5, + mediaStart: 0, + loop: true, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected loop source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + expect(video).toMatchObject({ start: -19, mediaStart: 0, loop: true }); + expect(lookup.getFrame("negative-loop", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + + it("preserves the held final frame after negative preroll exhausts a source", async () => { + const outputDir = join(FIXTURE_DIR, "out-negative-held-tail"); + mkdirSync(outputDir, { recursive: true }); + const video: VideoElement = { + id: "negative-held-tail", + src: VFR_FIXTURE, + start: -15, + end: 5, + mediaStart: 0, + loop: false, + hasAudio: false, + }; + + const result = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 1, + outputDir, + timelineEnd: 2, + }); + + expect(result.errors).toEqual([]); + const extracted = result.extracted[0]; + if (!extracted) throw new Error("expected held-tail source frames"); + const lookup = createFrameLookupTable([video], result.extracted); + expect(video).toMatchObject({ start: -15, mediaStart: 0, loop: false }); + expect(lookup.getFrame("negative-held-tail", 0)).toBe( + extracted.framePaths.get(extracted.totalFrames - 1), + ); + }, 30_000); + it("detects the synthesized fixture as VFR", async () => { const md = await extractVideoMetadata(VFR_FIXTURE); expect(md.isVFR).toBe(true); diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 37dd4c1ae77..e7bab493990 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -738,13 +738,24 @@ export interface TimelineExtractionWindow { compositionStart: number; mediaStart: number; durationSeconds: number; + /** + * Preserve the authored timeline origin and mediaStart for lookup. This is + * required when a negative preroll crosses a source boundary: looped media + * still needs its modulo phase, while a non-looping authored slot still + * needs the extracted final frame for held-tail playback. + */ + preserveTimelinePhase?: boolean; } +type TimelineWindowVideo = Pick & + Partial>; + /** Intersect a resolved source interval with the render timeline. */ export function resolveTimelineExtractionWindow( - video: Pick, + video: TimelineWindowVideo, resolvedDuration: number, timelineEnd?: number, + sourceDuration?: number, ): TimelineExtractionWindow { if (timelineEnd === undefined) { return { @@ -762,16 +773,41 @@ export function resolveTimelineExtractionWindow( 0, Math.min(resolvedDuration - trimmedPreroll, timelineEnd - compositionStart), ); + let mediaStart = video.mediaStart + trimmedPreroll; + if (durationSeconds > 0 && sourceDuration !== undefined) { + const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart); + const prerollCrossesSourceEnd = trimmedPreroll >= sourceRemaining; + if (sourceRemaining > 0 && video.loop && trimmedPreroll > 0) { + const phaseOffset = trimmedPreroll % sourceRemaining; + const phaseRemaining = sourceRemaining - phaseOffset; + if (durationSeconds > phaseRemaining) { + return { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: sourceRemaining, + preserveTimelinePhase: true, + }; + } + mediaStart = video.mediaStart + phaseOffset; + } else if (sourceRemaining > 0 && prerollCrossesSourceEnd) { + return { + compositionStart: video.start, + mediaStart: video.mediaStart, + durationSeconds: sourceRemaining, + preserveTimelinePhase: true, + }; + } + } return { compositionStart, - mediaStart: video.mediaStart + trimmedPreroll, + mediaStart, durationSeconds, }; } /** Resolve source duration first, then intersect it with the render timeline. */ export function resolveVideoExtractionWindow( - video: Pick, + video: TimelineWindowVideo, metadata: VideoMetadata, timelineEnd?: number, ): TimelineExtractionWindow { @@ -780,11 +816,16 @@ export function resolveVideoExtractionWindow( video.mediaStart, metadata, ); - return resolveTimelineExtractionWindow(video, resolvedDuration, timelineEnd); + return resolveTimelineExtractionWindow( + video, + resolvedDuration, + timelineEnd, + metadata.durationSeconds, + ); } export function resolveVideoExtractionDuration( - video: Pick, + video: TimelineWindowVideo, metadata: VideoMetadata, timelineEnd?: number, ): number { @@ -1590,9 +1631,11 @@ export async function extractAllVideoFrames( if (videoDuration <= 0) { return { skipped: true }; } - video.start = window.compositionStart; - video.end = window.compositionStart + videoDuration; - video.mediaStart = window.mediaStart; + if (!window.preserveTimelinePhase) { + video.start = window.compositionStart; + video.end = window.compositionStart + videoDuration; + video.mediaStart = window.mediaStart; + } const keyInput = cacheKeyInputs[index]; if (keyInput) keyInput.mediaStart = window.mediaStart; diff --git a/packages/producer/src/services/hdrCompositor.ts b/packages/producer/src/services/hdrCompositor.ts index 970cba39d8a..8ace0d41ded 100644 --- a/packages/producer/src/services/hdrCompositor.ts +++ b/packages/producer/src/services/hdrCompositor.ts @@ -139,6 +139,8 @@ export interface HdrVideoFrameSource { frameSize: number; frameCount: number; scratch: Buffer; + /** The raw file contains one playable source cycle and must wrap at EOF. */ + loop?: boolean; } export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void { @@ -152,6 +154,18 @@ export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: Prod } } +export function resolveHdrVideoFrameIndex( + time: number, + startTime: number, + fps: number, + frameCount: number, + loop = false, +): number | null { + const frameIndex = Math.round((time - startTime) * fps); + if (frameIndex < 0 || frameCount < 1) return null; + return loop ? frameIndex % frameCount : Math.min(frameIndex, frameCount - 1); +} + // fallow-ignore-next-line complexity export function blitHdrVideoLayer( canvas: Buffer, @@ -173,14 +187,18 @@ export function blitHdrVideoLayer( return; } - // Frame index within the video. Clamp to the extracted raw frame count so - // a composition that outlives the source clip freezes on the last frame, - // matching Chrome's