diff --git a/packages/engine/src/services/captureFailure.test.ts b/packages/engine/src/services/captureFailure.test.ts index 4cb720813f..264cbef493 100644 --- a/packages/engine/src/services/captureFailure.test.ts +++ b/packages/engine/src/services/captureFailure.test.ts @@ -6,6 +6,8 @@ describe("classifyCaptureFailure", () => { ["Target closed", "transient_browser"], ["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"], ["Runtime.evaluate timed out", "protocol_timeout"], + ["Network.enable timed out. Increase the protocolTimeout setting.", "protocol_timeout"], + ["[Parallel] Capture failed: Worker 0: Network.enable timed out", "protocol_timeout"], [ "Page.captureScreenshot timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.", "protocol_timeout", diff --git a/packages/engine/src/services/captureFailure.ts b/packages/engine/src/services/captureFailure.ts index a2428a4a58..99951d5188 100644 --- a/packages/engine/src/services/captureFailure.ts +++ b/packages/engine/src/services/captureFailure.ts @@ -55,6 +55,7 @@ const TRANSIENT_BROWSER_ERROR_PATTERNS = [ ]; const PROTOCOL_TIMEOUT_PATTERNS = [ + /Network\.enable timed out/i, /Runtime\.callFunctionOn timed out/i, /Runtime\.evaluate timed out/i, /Page\.captureScreenshot timed out/i, diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 05d5256d7a..13fb4e8182 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { join, win32 } from "node:path"; import { tmpdir } from "node:os"; import type { CaptureOptions, EngineConfig, ExtractedFrames } from "@hyperframes/engine"; -import { executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine"; +import { DEFAULT_CONFIG, executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine"; import type { CompiledComposition } from "./htmlCompiler.js"; // Replace only the two engine functions the adaptive-retry loop uses to touch @@ -312,43 +312,43 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single } }); - it("does NOT retry a transient error when the render was aborted", async () => { - const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-")); - const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-")); + it("retries Network.enable startup timeout once with fewer workers and zero progress", async () => { + const workDir = mkdtempSync(join(tmpdir(), "hf-transient-work-")); + const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-frames-")); const log = makeLog(); - const controller = new AbortController(); - // Cancellation tears the browser down, surfacing as a transient-looking - // "Target closed" — but an aborted render must fail immediately, not retry. + let call = 0; vi.mocked(executeParallelCapture).mockImplementation(async () => { - controller.abort(); - throw new Error("Target closed"); + call++; + if (call === 1) { + throw new Error("[Parallel] Capture failed: Worker 0: Network.enable timed out"); + } + writeAllFrames(framesDir, 4); + return []; }); vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined); try { - await expect( - executeDiskCaptureWithAdaptiveRetry({ - serverUrl: "http://localhost:0", - workDir, - framesDir, - totalFrames: 4, - initialWorkerCount: 2, - allowRetry: true, - frameExt: "jpg", - captureOptions: {} as CaptureOptions, - createBeforeCaptureHook: () => null, - abortSignal: controller.signal, - cfg: {} as EngineConfig, - log, - dedupPerfs: [], - }), - ).rejects.toThrow(/Target closed/); + const attempts = await executeDiskCaptureWithAdaptiveRetry({ + serverUrl: "http://localhost:0", + workDir, + framesDir, + totalFrames: 4, + initialWorkerCount: 4, + allowRetry: true, + frameExt: "jpg", + captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } }, + createBeforeCaptureHook: () => null, + cfg: DEFAULT_CONFIG, + log, + dedupPerfs: [], + }); - // Exactly one attempt — no transient retry burned on a cancelled render. - expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1); - expect(log.warn).not.toHaveBeenCalledWith( - expect.stringContaining("Transient browser failure"), - expect.anything(), + expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(2); + expect(attempts.map((a) => a.workers)).toEqual([4, 2]); + expect(attempts.map((a) => a.reason)).toEqual(["initial", "retry"]); + expect(log.warn).toHaveBeenCalledWith( + expect.stringContaining("Browser initialization timed out"), + expect.objectContaining({ fromWorkers: 4, toWorkers: 2 }), ); } finally { rmSync(workDir, { recursive: true, force: true }); @@ -356,40 +356,90 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single } }); - it("gives up after MAX_TRANSIENT_CAPTURE_RETRIES when the tab keeps dying", async () => { - const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-")); - const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-")); - const log = makeLog(); - vi.mocked(executeParallelCapture).mockRejectedValue(new Error("Session closed")); - vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined); - - try { - await expect( - executeDiskCaptureWithAdaptiveRetry({ - serverUrl: "http://localhost:0", - workDir, - framesDir, - totalFrames: 4, - initialWorkerCount: 1, - allowRetry: true, - frameExt: "jpg", - captureOptions: {} as CaptureOptions, - createBeforeCaptureHook: () => null, - cfg: {} as EngineConfig, - log, - dedupPerfs: [], - }), - ).rejects.toThrow(/Session closed/); - - // 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries. - expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes( - 1 + MAX_TRANSIENT_CAPTURE_RETRIES, - ); - } finally { - rmSync(workDir, { recursive: true, force: true }); - rmSync(framesDir, { recursive: true, force: true }); - } - }); + it.each(["Target closed", "Network.enable timed out"])( + "does NOT retry %s after cancellation", + async (message) => { + const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-")); + const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-")); + const log = makeLog(); + const controller = new AbortController(); + // Cancellation tears the browser down, surfacing as a transient-looking + // "Target closed" — but an aborted render must fail immediately, not retry. + vi.mocked(executeParallelCapture).mockImplementation(async () => { + controller.abort(); + throw new Error(message); + }); + vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined); + + try { + await expect( + executeDiskCaptureWithAdaptiveRetry({ + serverUrl: "http://localhost:0", + workDir, + framesDir, + totalFrames: 4, + initialWorkerCount: 2, + allowRetry: true, + frameExt: "jpg", + captureOptions: {} as CaptureOptions, + createBeforeCaptureHook: () => null, + abortSignal: controller.signal, + cfg: {} as EngineConfig, + log, + dedupPerfs: [], + }), + ).rejects.toThrow(message); + + // Exactly one attempt — no transient retry burned on a cancelled render. + expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1); + expect(log.warn).not.toHaveBeenCalledWith( + expect.stringContaining("Transient browser failure"), + expect.anything(), + ); + } finally { + rmSync(workDir, { recursive: true, force: true }); + rmSync(framesDir, { recursive: true, force: true }); + } + }, + ); + + it.each(["Session closed", "Network.enable timed out"])( + "bounds repeated %s failures", + async (message) => { + const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-")); + const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-")); + const log = makeLog(); + vi.mocked(executeParallelCapture).mockRejectedValue(new Error(message)); + vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined); + + try { + await expect( + executeDiskCaptureWithAdaptiveRetry({ + serverUrl: "http://localhost:0", + workDir, + framesDir, + totalFrames: 4, + initialWorkerCount: 1, + allowRetry: true, + frameExt: "jpg", + captureOptions: {} as CaptureOptions, + createBeforeCaptureHook: () => null, + cfg: {} as EngineConfig, + log, + dedupPerfs: [], + }), + ).rejects.toThrow(message); + + // 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries. + expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes( + 1 + MAX_TRANSIENT_CAPTURE_RETRIES, + ); + } finally { + rmSync(workDir, { recursive: true, force: true }); + rmSync(framesDir, { recursive: true, force: true }); + } + }, + ); }); describe("describeMemoryExhaustion", () => { diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 6745261d48..4a0f0a883b 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -1044,6 +1044,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: { let missingRanges: FrameRange[] | null = null; let attempt = 0; let transientRetriesUsed = 0; + let initializationRetriesUsed = 0; // Set when the *previous* iteration retried after a transient browser death, // so the attempt it spawns is tagged `"transient-retry"` (vs the worker-halving // `"retry"`) for telemetry. Reset after each attempt is recorded. @@ -1210,6 +1211,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: { continue; } + // CDP initialization can time out before any frame exists. Give that + // specific startup failure one fresh attempt with less concurrency; + // arbitrary zero-progress authoring/capture errors still fail below. + if ( + options.allowRetry && + !madeProgress && + initializationRetriesUsed === 0 && + failure.kind === "protocol_timeout" && + /\bNetwork\.enable timed out/i.test(failure.message) + ) { + initializationRetriesUsed++; + const nextWorkers = getNextRetryWorkerCount(currentWorkers); + options.log.warn( + "[Render] Browser initialization timed out; retrying once with fresh sessions.", + { + fromWorkers: currentWorkers, + toWorkers: nextWorkers, + error: failure.message, + }, + ); + currentWorkers = nextWorkers; + missingRanges = remaining; + attempt++; + continue; + } + if (!madeProgress) { options.log.warn( "[Render] Capture attempt made no forward progress; composition is likely structurally broken — not retrying.",