Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/engine/src/services/captureFailure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ describe("classifyCaptureFailure", () => {
["Target closed", "transient_browser"],
["Runtime.callFunctionOn timed out after 30000ms", "protocol_timeout"],
["Runtime.evaluate timed out", "protocol_timeout"],
["Network.enable timed out. Increase the protocolTimeout setting.", "protocol_timeout"],
["[Parallel] Capture failed: Worker 0: Network.enable timed out", "protocol_timeout"],
[
"Page.captureScreenshot timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed.",
"protocol_timeout",
Expand Down
1 change: 1 addition & 0 deletions packages/engine/src/services/captureFailure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ const TRANSIENT_BROWSER_ERROR_PATTERNS = [
];

const PROTOCOL_TIMEOUT_PATTERNS = [
/Network\.enable timed out/i,
/Runtime\.callFunctionOn timed out/i,
/Runtime\.evaluate timed out/i,
/Page\.captureScreenshot timed out/i,
Expand Down
180 changes: 115 additions & 65 deletions packages/producer/src/services/renderOrchestrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync
import { join, win32 } from "node:path";
import { tmpdir } from "node:os";
import type { CaptureOptions, EngineConfig, ExtractedFrames } from "@hyperframes/engine";
import { executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine";
import { DEFAULT_CONFIG, executeParallelCapture, mergeWorkerFrames } from "@hyperframes/engine";
import type { CompiledComposition } from "./htmlCompiler.js";

// Replace only the two engine functions the adaptive-retry loop uses to touch
Expand Down Expand Up @@ -312,84 +312,134 @@ describe("executeDiskCaptureWithAdaptiveRetry — transient Target-closed single
}
});

it("does NOT retry a transient error when the render was aborted", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-"));
it("retries Network.enable startup timeout once with fewer workers and zero progress", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-frames-"));
const log = makeLog();
const controller = new AbortController();
// Cancellation tears the browser down, surfacing as a transient-looking
// "Target closed" — but an aborted render must fail immediately, not retry.
let call = 0;
vi.mocked(executeParallelCapture).mockImplementation(async () => {
controller.abort();
throw new Error("Target closed");
call++;
if (call === 1) {
throw new Error("[Parallel] Capture failed: Worker 0: Network.enable timed out");
}
writeAllFrames(framesDir, 4);
return [];
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);

try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 2,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
abortSignal: controller.signal,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Target closed/);
const attempts = await executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 4,
allowRetry: true,
frameExt: "jpg",
captureOptions: { width: 64, height: 64, fps: { num: 30, den: 1 } },
createBeforeCaptureHook: () => null,
cfg: DEFAULT_CONFIG,
log,
dedupPerfs: [],
});

// Exactly one attempt — no transient retry burned on a cancelled render.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1);
expect(log.warn).not.toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.anything(),
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(2);
expect(attempts.map((a) => a.workers)).toEqual([4, 2]);
expect(attempts.map((a) => a.reason)).toEqual(["initial", "retry"]);
expect(log.warn).toHaveBeenCalledWith(
expect.stringContaining("Browser initialization timed out"),
expect.objectContaining({ fromWorkers: 4, toWorkers: 2 }),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});

it("gives up after MAX_TRANSIENT_CAPTURE_RETRIES when the tab keeps dying", async () => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-"));
const log = makeLog();
vi.mocked(executeParallelCapture).mockRejectedValue(new Error("Session closed"));
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);

try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(/Session closed/);

// 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(
1 + MAX_TRANSIENT_CAPTURE_RETRIES,
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
});
it.each(["Target closed", "Network.enable timed out"])(
"does NOT retry %s after cancellation",
async (message) => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient-abort-frames-"));
const log = makeLog();
const controller = new AbortController();
// Cancellation tears the browser down, surfacing as a transient-looking
// "Target closed" — but an aborted render must fail immediately, not retry.
vi.mocked(executeParallelCapture).mockImplementation(async () => {
controller.abort();
throw new Error(message);
});
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);

try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 2,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
abortSignal: controller.signal,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(message);

// Exactly one attempt — no transient retry burned on a cancelled render.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(1);
expect(log.warn).not.toHaveBeenCalledWith(
expect.stringContaining("Transient browser failure"),
expect.anything(),
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
},
);

it.each(["Session closed", "Network.enable timed out"])(
"bounds repeated %s failures",
async (message) => {
const workDir = mkdtempSync(join(tmpdir(), "hf-transient2-work-"));
const framesDir = mkdtempSync(join(tmpdir(), "hf-transient2-frames-"));
const log = makeLog();
vi.mocked(executeParallelCapture).mockRejectedValue(new Error(message));
vi.mocked(mergeWorkerFrames).mockResolvedValue(undefined);

try {
await expect(
executeDiskCaptureWithAdaptiveRetry({
serverUrl: "http://localhost:0",
workDir,
framesDir,
totalFrames: 4,
initialWorkerCount: 1,
allowRetry: true,
frameExt: "jpg",
captureOptions: {} as CaptureOptions,
createBeforeCaptureHook: () => null,
cfg: {} as EngineConfig,
log,
dedupPerfs: [],
}),
).rejects.toThrow(message);

// 1 initial attempt + exactly MAX_TRANSIENT_CAPTURE_RETRIES retries.
expect(vi.mocked(executeParallelCapture)).toHaveBeenCalledTimes(
1 + MAX_TRANSIENT_CAPTURE_RETRIES,
);
} finally {
rmSync(workDir, { recursive: true, force: true });
rmSync(framesDir, { recursive: true, force: true });
}
},
);
});

describe("describeMemoryExhaustion", () => {
Expand Down
27 changes: 27 additions & 0 deletions packages/producer/src/services/renderOrchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1044,6 +1044,7 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
let missingRanges: FrameRange[] | null = null;
let attempt = 0;
let transientRetriesUsed = 0;
let initializationRetriesUsed = 0;
// Set when the *previous* iteration retried after a transient browser death,
// so the attempt it spawns is tagged `"transient-retry"` (vs the worker-halving
// `"retry"`) for telemetry. Reset after each attempt is recorded.
Expand Down Expand Up @@ -1210,6 +1211,32 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
continue;
}

// CDP initialization can time out before any frame exists. Give that
// specific startup failure one fresh attempt with less concurrency;
// arbitrary zero-progress authoring/capture errors still fail below.
if (
options.allowRetry &&
!madeProgress &&
initializationRetriesUsed === 0 &&
failure.kind === "protocol_timeout" &&
/\bNetwork\.enable timed out/i.test(failure.message)
) {
initializationRetriesUsed++;
const nextWorkers = getNextRetryWorkerCount(currentWorkers);
options.log.warn(
"[Render] Browser initialization timed out; retrying once with fresh sessions.",
{
fromWorkers: currentWorkers,
toWorkers: nextWorkers,
error: failure.message,
},
);
currentWorkers = nextWorkers;
missingRanges = remaining;
attempt++;
continue;
}

if (!madeProgress) {
options.log.warn(
"[Render] Capture attempt made no forward progress; composition is likely structurally broken — not retrying.",
Expand Down
Loading