Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions packages/producer/src/services/renderOrchestrator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type { CompiledComposition } from "./htmlCompiler.js";

import {
buildMissingFrameRetryBatches,
captureAttemptMadeProgress,
collectVideoMetadataHints,
collectVideoReadinessSkipIds,
extractStandaloneEntryFromIndex,
Expand Down Expand Up @@ -95,6 +96,21 @@ describe("extractStandaloneEntryFromIndex", () => {
});
});

describe("captureAttemptMadeProgress", () => {
it("retries when the attempt captured at least one frame toward its target", () => {
// targeted 100 frames, 40 still missing -> 60 captured -> worth retrying the rest
expect(captureAttemptMadeProgress(100, 40)).toBe(true);
expect(captureAttemptMadeProgress(100, 99)).toBe(true);
});

it("bails when the attempt captured nothing (structurally broken composition)", () => {
// targeted 100 frames, 100 still missing -> zero progress -> don't burn another timeout cycle
expect(captureAttemptMadeProgress(100, 100)).toBe(false);
// defensive: never-greater-than guard, treat >= target as no progress
expect(captureAttemptMadeProgress(100, 120)).toBe(false);
});
});

describe("shouldUseStreamingEncode", () => {
const streamingEnabledConfig = {
enableStreamingEncode: true,
Expand Down
37 changes: 32 additions & 5 deletions packages/producer/src/services/renderOrchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -548,6 +548,24 @@ export function getNextRetryWorkerCount(currentWorkers: number): number {
return Math.max(1, Math.floor(currentWorkers / 2));
}

/**
* A retry only pays off if the attempt that just finished captured at least one
* frame toward its target. When it captured nothing (frames still missing >=
* frames it set out to capture), the composition is structurally broken — a
* never-ready page, zero duration, or unparseable HTML — not a flaky worker.
* Re-running it at lower parallelism just burns another full readiness/protocol
* timeout per worker, which is what blew up render P95 in Jun 2026 (broken
* renders walked 16->8->4->2->1 workers, ~46min each before finally failing).
* ponytail: total zero-progress == broken; a partially-captured attempt still
* retries, so genuine flaky-worker gaps are unaffected.
*/
export function captureAttemptMadeProgress(
attemptTargetFrameCount: number,
remainingFrameCount: number,
): boolean {
return remainingFrameCount < attemptTargetFrameCount;
}

export function isRecoverableParallelCaptureError(error: unknown): boolean {
const message = normalizeErrorMessage(error);
return (
Expand Down Expand Up @@ -673,10 +691,13 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
if (remaining.length === 0) {
return attempts;
}
if (!options.allowRetry || currentWorkers <= 1) {
throw new Error(
`[Render] Capture completed but ${countFrameRanges(remaining)} frame(s) are missing`,
);
const remainingCount = countFrameRanges(remaining);
if (
!options.allowRetry ||
currentWorkers <= 1 ||
!captureAttemptMadeProgress(frameCount, remainingCount)
) {
throw new Error(`[Render] Capture completed but ${remainingCount} frame(s) are missing`);
}

const nextWorkers = getNextRetryWorkerCount(currentWorkers);
Expand All @@ -697,7 +718,13 @@ export async function executeDiskCaptureWithAdaptiveRetry(options: {
if (remaining.length === 0) {
return attempts;
}
if (!options.allowRetry || currentWorkers <= 1 || !isRecoverableParallelCaptureError(error)) {
const remainingCount = countFrameRanges(remaining);
if (
!options.allowRetry ||
currentWorkers <= 1 ||
!isRecoverableParallelCaptureError(error) ||
!captureAttemptMadeProgress(frameCount, remainingCount)
) {
throw error;
}

Expand Down
Loading