diff --git a/packages/cli/src/browser/manager.ts b/packages/cli/src/browser/manager.ts index 78d5116d73..ed1eb0eb39 100644 --- a/packages/cli/src/browser/manager.ts +++ b/packages/cli/src/browser/manager.ts @@ -20,7 +20,22 @@ async function loadPuppeteerBrowsers(): Promise { } } -const CHROME_VERSION = "152.0.7928.2"; +// Bumped from 152.0.7928.2 on 2026-08-10 to pick up crbug 522872457's fix +// (CL 8032671, "Force-merge PendingLayer for canvas child descendant", merged +// 2026-07-07 — after the 152.0.7935.0 canary cut, so the old pin predated it). +// +// Measured on the shipping headless-shell binary, drawElementImage vs a CDP +// screenshot of the identical state: +// sibling content dropped from 3D captures — FIXED +// background lost from 3D captures — FIXED +// backface-visibility:hidden ignored — STILL BROKEN (1.4 dB -> 14.8 dB; +// better, still far under the +// 32 dB floor) +// So the 3D compile gate must stay. See PRINFRA-486. +// +// Deliberately Beta, not Canary. 153.0.8000.0 measured identical to this build +// on every probe variant, so crossing a major buys nothing measurable. +const CHROME_VERSION = "152.0.7977.30"; const CACHE_ROOT_DIR = join(homedir(), ".cache", "hyperframes"); const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "chrome"); // Puppeteer's managed cache — where `@puppeteer/browsers install @@ -332,7 +347,7 @@ async function findFromCache(): Promise { // this is the non-`preferManagedChrome` path, which exists so a user who // installed chrome-headless-shell separately (via `@puppeteer/browsers // install`) keeps using that binary instead of being silently switched to - // the HF-pinned one. Note `CHROME_VERSION` (above) is a Dev-channel pin + // the HF-pinned one. Note `CHROME_VERSION` (above) is a pre-Stable pin // that may be NEWER than a user's puppeteer-cache Stable build — this is // about respecting an explicit prior choice, not "newest wins". const fromPuppeteer = findFromPuppeteerCache(); diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index 06d03da67d..a939ea9f98 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -1514,6 +1514,7 @@ function trackRenderMetrics( deBlankRecaptures: perf?.drawElement?.blankRecaptures, deBoundaryFrames: perf?.drawElement?.boundaryFrames, deNcprFallbacks: perf?.drawElement?.ncprFallbacks, + deFrameTimeouts: perf?.drawElement?.frameTimeouts, compositionDurationMs, compositionWidth: perf?.resolution.width, compositionHeight: perf?.resolution.height, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index de3e371cd5..b529dbe022 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -235,6 +235,7 @@ export function trackRenderComplete( deBlankRecaptures?: number; deBoundaryFrames?: number; deNcprFallbacks?: number; + deFrameTimeouts?: number; // "cli" when triggered by `hyperframes render` (default), "studio" when // triggered by a studio preview-server render (POST /api/projects/:id/render). source?: "cli" | "studio"; @@ -336,6 +337,7 @@ export function trackRenderComplete( de_blank_recaptures: props.deBlankRecaptures, de_boundary_frames: props.deBoundaryFrames, de_ncpr_fallbacks: props.deNcprFallbacks, + de_frame_timeouts: props.deFrameTimeouts, ...powerStateFields(), source: props.source ?? "cli", composition_duration_ms: props.compositionDurationMs, diff --git a/packages/engine/src/services/frameCapture-frameDeadline.test.ts b/packages/engine/src/services/frameCapture-frameDeadline.test.ts new file mode 100644 index 0000000000..33716a829b --- /dev/null +++ b/packages/engine/src/services/frameCapture-frameDeadline.test.ts @@ -0,0 +1,81 @@ +/** + * Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488). + * + * The deadline races the capture round-trip from OUTSIDE `captureFrameCore`, + * because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly + * why the stall counter has to live in the `onTimeout` hook: a wedged renderer + * never returns, so no catch block inside the work promise ever runs. The first + * shipped version incremented `session.deFrameTimeouts` in that unreachable + * catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0 + * on every stalled render. + */ + +import { readFileSync } from "node:fs"; +import { describe, expect, it, vi } from "vitest"; +import { withFrameDeadline } from "./frameCapture.js"; + +describe("withFrameDeadline", () => { + it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + // Never settles — the wedged-renderer shape. + const raced = withFrameDeadline(new Promise(() => {}), "frame 7", 15_000, onTimeout); + const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/); + await vi.advanceTimersByTimeAsync(15_000); + await assertion; + expect(onTimeout).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it("passes the value through and leaves onTimeout alone when work wins", async () => { + vi.useFakeTimers(); + try { + const onTimeout = vi.fn(); + const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout); + await expect(raced).resolves.toBe("buffer"); + // Past the deadline: the cleared timer must not fire late. + await vi.advanceTimersByTimeAsync(30_000); + expect(onTimeout).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); +}); + +// The three drawElement capture entry points are only reachable with a real +// Chrome page, so the wiring is pinned at the source level instead. The first +// shipped deadline bounded ONLY the streaming path, leaving the disk and +// worker-encode paths to hit the 60 s render-level watchdog on the same wedged +// renderer — a coverage gap invisible to any unit test of the deadline itself. +describe("drawElement capture entry points", () => { + const source = readFileSync(new URL("./frameCapture.ts", import.meta.url), "utf8"); + + const bodyOf = (name: string): string => { + const start = source.indexOf(`export async function ${name}(`); + expect(start).toBeGreaterThan(-1); + // Up to the next top-level declaration — enough to cover the function body. + const next = source.indexOf("\nexport ", start + 1); + return source.slice(start, next === -1 ? undefined : next); + }; + + it.each(["captureFrameToBuffer", "captureFrame", "captureFrameToBufferPipelined"])( + "%s bounds its drawElement round-trip", + (entryPoint) => { + expect(bodyOf(entryPoint)).toContain("withDeFrameDeadline("); + }, + ); + + // Diagnostics screenshot and evaluate against the page that just stopped + // scheduling, so running them on a stall spends the whole budget the deadline + // saved. The pipelined catch must bail before reaching them. + it("skips per-frame diagnostics when the pipelined path times out", () => { + const body = bodyOf("captureFrameToBufferPipelined"); + const bail = body.indexOf("if (isDeFrameTimeoutError(captureError)) throw captureError;"); + const diagnostics = body.indexOf("captureFrameErrorDiagnostics("); + expect(bail).toBeGreaterThan(-1); + expect(diagnostics).toBeGreaterThan(bail); + }); +}); diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts index d7d1c59c0e..22fd3546e4 100644 --- a/packages/engine/src/services/frameCapture.ts +++ b/packages/engine/src/services/frameCapture.ts @@ -212,6 +212,13 @@ export interface CaptureSession { deVerifyInitMs?: number; /** Count of per-frame "No cached paint record" screenshot fallbacks (telemetry). */ deNcprFallbacks?: number; + /** + * Count of drawElement frame captures that blew `HF_DE_FRAME_TIMEOUT_MS` + * because the renderer stopped scheduling after drawElementImage returned + * (PRINFRA-488). Each one aborts the drawElement attempt so the whole render + * retries via screenshot. + */ + deFrameTimeouts?: number; /** * drawElement init passed every gate but stopped before verification + * canvas injection: the session has no video-frame injector yet (probe @@ -3133,6 +3140,104 @@ function isNoCachedPaintRecordError(err: unknown): boolean { return msg.includes("No cached paint record"); } +/** + * Per-frame deadline for the drawElement capture round-trip. + * + * drawElementImage can return normally and then leave the renderer not draining + * its task queue: the `setTimeout(…, 0)` that drawAndEncode schedules to run + * `toDataURL` never fires, so the capture `page.evaluate` never settles. + * Reproduced deterministically on Chromium 152.0.7977.30, one comp, always the + * same frame (PRINFRA-488). Nothing below the render-level watchdog bounded + * this, so a single bad frame failed the ENTIRE render after a 60 s stall. + * + * This bounds the round-trip so the frame can take the same per-frame screenshot + * fallback the `No cached paint record` case already takes — one slow frame + * instead of a dead render. Tune with `HF_DE_FRAME_TIMEOUT_MS`; 0 disables. + */ +const DE_FRAME_TIMEOUT_MS = Number(process.env.HF_DE_FRAME_TIMEOUT_MS ?? "15000"); + +class DeFrameTimeoutError extends Error { + constructor(label: string, ms: number) { + super(`drawElement ${label} exceeded ${ms}ms (renderer stopped scheduling; see PRINFRA-488)`); + this.name = "DeFrameTimeoutError"; + } +} + +/** + * Race `work` against a deadline. The losing promise is NOT cancellable — + * puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is + * swallowed to avoid an unhandled rejection when it eventually settles (or + * never does). The orphaned round-trip keeps running in its Chrome worker; + * that worker is reclaimed by the outer retry rebuilding the page + * (`closeOrphanedProbeForRetry`), not by anything here. + * + * `onTimeout` fires exactly when the deadline wins, and is the ONLY place the + * stall is observable: because the deadline races `work` from outside, nothing + * inside `work` — including its own catch blocks — ever sees this error. + * + * Exported for the deadline unit test; `captureFrameToBuffer` is the only + * production caller. + */ +export async function withFrameDeadline( + work: Promise, + label: string, + ms: number, + onTimeout?: () => void, +): Promise { + if (!(ms > 0)) return work; + let timer: ReturnType | undefined; + const guard = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + onTimeout?.(); + reject(new DeFrameTimeoutError(label, ms)); + }, ms); + }); + try { + return await Promise.race([work, guard]); + } finally { + if (timer) clearTimeout(timer); + void work.catch(() => { + /* orphaned round-trip — see doc above */ + }); + } +} + +function isDeFrameTimeoutError(err: unknown): boolean { + return err instanceof DeFrameTimeoutError; +} + +/** + * Bound a drawElement round-trip, counting and logging the stall if the + * deadline wins. Every drawElement capture entry point goes through this — + * streaming (`captureFrameToBuffer`), disk (`captureFrame`) and worker-encode + * (`captureFrameToBufferPipelined`). Bounding only one of them left the other + * two hitting the 60 s render-level watchdog on the same wedged renderer, + * which is the whole failure this deadline exists to replace. + */ +function withDeFrameDeadline( + session: CaptureSession, + frameIndex: number, + work: Promise, +): Promise { + if (session.captureMode !== "drawelement") return work; + return withFrameDeadline(work, `frame ${frameIndex}`, DE_FRAME_TIMEOUT_MS, () => { + // Deliberately NO per-frame screenshot fallback. When the renderer stops + // scheduling it is wedged for EVERY subsequent round-trip on that page — + // measured: the screenshot fallback blew the same deadline. Fail fast and + // let the producer re-render the whole comp on a fresh page via the + // screenshot path, the only recovery that works. Counted here rather than + // in a capture catch: the deadline rejects from OUTSIDE the work promise, + // so no catch inside it ever runs. + session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1; + console.log( + `[engine] fast capture: frame ${frameIndex} — capture exceeded ` + + `${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` + + `(PRINFRA-488). Failing the drawElement attempt so the whole render ` + + `retries via screenshot.`, + ); + }); +} + async function captureFrameCore( session: CaptureSession, frameIndex: number, @@ -3307,10 +3412,10 @@ export async function captureFrame( frameIndex: number, time: number, ): Promise { - const { buffer, quantizedTime, captureTimeMs } = await captureFrameCore( + const { buffer, quantizedTime, captureTimeMs } = await withDeFrameDeadline( session, frameIndex, - time, + captureFrameCore(session, frameIndex, time), ); const framePath = writeCapturedFrame(session, frameIndex, buffer); return { frameIndex, time: quantizedTime, path: framePath, captureTimeMs }; @@ -3344,7 +3449,11 @@ export async function captureFrameToBuffer( frameIndex: number, time: number, ): Promise { - const { buffer, captureTimeMs } = await captureFrameCore(session, frameIndex, time); + const { buffer, captureTimeMs } = await withDeFrameDeadline( + session, + frameIndex, + captureFrameCore(session, frameIndex, time), + ); return { buffer, captureTimeMs }; } @@ -3443,12 +3552,10 @@ export async function captureFrameToBufferPipelined( // syncToPaintEvent = true); see initDrawElementOrTransparentBackground. The // BeginFrame branch present in the synchronous captureFrameCore is therefore // unreachable here and intentionally omitted. - const { encodeResult } = await produceDrawElementFrame( - page, - options.width, - options.height, - options.quality ?? 80, - true, + const { encodeResult } = await withDeFrameDeadline( + session, + frameIndex, + produceDrawElementFrame(page, options.width, options.height, options.quality ?? 80, true), ); const captureTimeMs = Date.now() - startTime; @@ -3482,6 +3589,11 @@ export async function captureFrameToBufferPipelined( const buffer = await pageScreenshotCapture(page, options); return { encodeResult: Promise.resolve(buffer), captureTimeMs: Date.now() - startTime }; } + // A blown deadline means the renderer is not draining its task queue, so + // the diagnostics below — which screenshot and evaluate against that same + // page — would block until the render-level watchdog fires, spending the + // whole budget the deadline just saved. + if (isDeFrameTimeoutError(captureError)) throw captureError; // Mirror captureFrameCore: capture per-frame diagnostics (frame-error // PNG/HTML/JSON + console tail) before rethrowing so pipelined-path // failures are debuggable like the serial path. @@ -3951,5 +4063,6 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma deVerifyInitMs: session.deVerifyInitMs ?? 0, deBoundaryFrames: session.clipBoundaryFrames?.size ?? 0, deNcprFallbacks: session.deNcprFallbacks ?? 0, + deFrameTimeouts: session.deFrameTimeouts ?? 0, }; } diff --git a/packages/engine/src/services/threeDProjection.ts b/packages/engine/src/services/threeDProjection.ts index c01dacf26b..55cf4b8fa5 100644 --- a/packages/engine/src/services/threeDProjection.ts +++ b/packages/engine/src/services/threeDProjection.ts @@ -786,7 +786,41 @@ export async function initThreeDProjectionInPage(): Promise { }); }); + // A wedged renderer (PRINFRA-488) on the disk path recovers the same way. It + // needs its own failure kind because `sdr_disk` accepts no other, and the + // stall verified nothing — reusing the verification label would put a + // fabricated diagnosis in the retry telemetry. + it("forces the screenshot baseline on a disk plan after a renderer stall", () => { + const disk = createCapturePlan({ + workerCount: 2, + forceScreenshot: false, + useStreamingEncode: false, + useLayeredComposite: false, + usePageSideCompositing: false, + hasHdrContent: false, + needsAlpha: false, + }); + expect(replanAfterFailure(disk, { kind: "renderer_stall" })).toMatchObject({ + kind: "sdr_disk", + forceScreenshot: true, + forceParallelStream: false, + workerCount: 2, + }); + }); + it("rejects a streaming transition from a non-streaming plan", () => { const disk = createCapturePlan({ workerCount: 2, diff --git a/packages/producer/src/services/render/capturePlan.ts b/packages/producer/src/services/render/capturePlan.ts index 6e04857124..0b13acd761 100644 --- a/packages/producer/src/services/render/capturePlan.ts +++ b/packages/producer/src/services/render/capturePlan.ts @@ -66,6 +66,13 @@ export interface CreateCapturePlanInput { export type CapturePlanFailure = | Readonly<{ kind: "streaming_unavailable" }> | Readonly<{ kind: "draw_element_verification" }> + /** + * drawElement wedged the renderer and blew the per-frame deadline + * (PRINFRA-488). Distinct from `draw_element_verification` — nothing was + * verified and no score exists — but the recovery is identical: stay on this + * plan's path, force the screenshot baseline. + */ + | Readonly<{ kind: "renderer_stall" }> | Readonly<{ kind: "capture_failure"; memoryExhaustion: boolean }>; function assertWorkerCount(workerCount: number): void { @@ -126,8 +133,12 @@ function revertedRouting(routing: CaptureRouting): CaptureRouting { export function replanAfterFailure(plan: CapturePlan, failure: CapturePlanFailure): CapturePlan { // Disk-path drawElement self-verification (parallel disk workers under the // explicit fast-capture opt-in) can also trip — the retry stays on the disk - // path but forces the screenshot baseline. - if (plan.kind === "sdr_disk" && failure.kind === "draw_element_verification") { + // path but forces the screenshot baseline. A wedged renderer takes the same + // route: different diagnosis, same recovery. + if ( + plan.kind === "sdr_disk" && + (failure.kind === "draw_element_verification" || failure.kind === "renderer_stall") + ) { return createCapturePlan({ ...plan, forceScreenshot: true, diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts index e4f4db3fc8..c100e3dac3 100644 --- a/packages/producer/src/services/render/perfSummary.ts +++ b/packages/producer/src/services/render/perfSummary.ts @@ -149,6 +149,7 @@ function aggregateDrawElement( blankRecaptures: drain?.blankRecaptures ?? 0, boundaryFrames: perfs.reduce((sum, p) => sum + (p.deBoundaryFrames ?? 0), 0), ncprFallbacks: perfs.reduce((sum, p) => sum + (p.deNcprFallbacks ?? 0), 0), + frameTimeouts: perfs.reduce((sum, p) => sum + (p.deFrameTimeouts ?? 0), 0), }; } diff --git a/packages/producer/src/services/renderOrchestrator.test.ts b/packages/producer/src/services/renderOrchestrator.test.ts index 93c04e51d0..29d9f39c43 100644 --- a/packages/producer/src/services/renderOrchestrator.test.ts +++ b/packages/producer/src/services/renderOrchestrator.test.ts @@ -34,6 +34,7 @@ import { resolveParallelRouterRetryPlan, resetCaptureAttemptProgress, shouldRetryViaPinnedFallback, + isDeRendererStallError, countElementTags, envInt, isDeParallelRouterEnabled, @@ -2457,6 +2458,60 @@ describe("resolveParallelRouterRetryPlan (self-verify retry rollback)", () => { }); describe("shouldRetryViaPinnedFallback (widen the self-verify retry to generic capture failures, including OOM)", () => { + // PRINFRA-488: a wedged renderer must be retryable on ANY routing. Before this, + // a comp that engaged drawElement on the ordinary single-worker path had no + // whole-render fallback, so one stalled frame failed the entire render. + it("retries a drawElement renderer stall even with no pinned routing", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: true, + }), + ).toBe(true); + }); + + it("still does NOT retry a generic capture failure with no pinned routing", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: false, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: false, + }), + ).toBe(false); + }); + + it("never retries a cancellation, even for a renderer stall", () => { + expect( + shouldRetryViaPinnedFallback({ + isVerifyError: false, + isCancellation: true, + deWorkerInversion: undefined, + deParallelRouter: undefined, + isDeRendererStall: true, + }), + ).toBe(false); + }); + + it("recognizes the engine's stall error across the package boundary", () => { + const byName = new Error("whatever"); + byName.name = "DeFrameTimeoutError"; + expect(isDeRendererStallError(byName)).toBe(true); + expect( + isDeRendererStallError( + new Error( + "drawElement frame 50 exceeded 15000ms (renderer stopped scheduling; see PRINFRA-488)", + ), + ), + ).toBe(true); + expect(isDeRendererStallError(new Error("some other capture failure"))).toBe(false); + expect(isDeRendererStallError("not an error")).toBe(false); + }); + it("always retries a drawElement self-verify failure, pinned or not", () => { expect( shouldRetryViaPinnedFallback({ diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 69100e1e1a..ff065f1df1 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -529,9 +529,9 @@ export interface RenderPerfSummary { * `fallbackReason` being set is the "any fallback fired" signal. */ selfVerifyFallback: boolean; - /** What tripped the fallback retry: psnr | blank | oom | capture_error. */ + /** What tripped the fallback retry: psnr | blank | oom | de_renderer_stall | capture_error. */ fallbackReason?: string; - /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for blank/oom/capture_error (no score exists). */ + /** The failing PSNR (dB) when `fallbackReason === "psnr"`; undefined for every other reason (no score exists). */ fallbackFailedDb?: number; /** Frame index the verification failure was detected at; set for both "psnr" and "blank" fallback reasons. */ fallbackFrameIndex?: number; @@ -545,6 +545,13 @@ export interface RenderPerfSummary { boundaryFrames: number; /** Per-frame "No cached paint record" screenshot fallbacks. */ ncprFallbacks: number; + /** + * Frames that blew `HF_DE_FRAME_TIMEOUT_MS` — a wedged renderer + * (PRINFRA-488). Distinct from the other fallback counters: this one always + * costs a whole-render re-run via screenshot, so its rate is worth graphing + * on its own rather than inside `capture_error`. + */ + frameTimeouts: number; }; } @@ -1808,12 +1815,34 @@ export function shouldRetryViaPinnedFallback(args: { isCancellation: boolean; deWorkerInversion: "inverted" | "reverted" | undefined; deParallelRouter: "routed" | "reverted" | undefined; + /** + * The drawElement capture wedged the renderer (PRINFRA-488). Retryable on ANY + * routing, not just a pinned one: the failure is a property of drawElement + * itself, and the retry re-renders on a fresh page via screenshot — the only + * recovery that works once the renderer stops scheduling. Without this a comp + * that engaged drawElement on the ordinary single-worker path (neither + * inverted nor routed) had NO whole-render fallback, so one wedged frame + * failed the entire render. + */ + isDeRendererStall?: boolean; }): boolean { if (args.isCancellation) return false; if (args.isVerifyError) return true; + if (args.isDeRendererStall === true) return true; return args.deWorkerInversion === "inverted" || args.deParallelRouter === "routed"; } +/** + * True for the drawElement per-frame deadline breach raised by the engine when + * the renderer stops scheduling after `drawElementImage` returns (PRINFRA-488). + * Matched on name+message rather than by class because the error crosses the + * engine/producer package boundary. + */ +export function isDeRendererStallError(err: unknown): boolean { + if (!(err instanceof Error)) return false; + return err.name === "DeFrameTimeoutError" || err.message.includes("renderer stopped scheduling"); +} + /** * When a self-verify (or pinned-fallback) retry is triggered mid-capture, the * caller may still hold a live probe session that the failed stage was passed @@ -3556,6 +3585,7 @@ async function executeRenderPipeline(input: { // spawns on retry. See shouldRetryViaPinnedFallback for exactly // which errors qualify. const isVerifyError = isDrawElementVerificationError(err); + const isDeStall = isDeRendererStallError(err); const isCancellation = err instanceof RenderCancelledError || executionSignal?.aborted === true; if ( @@ -3564,6 +3594,7 @@ async function executeRenderPipeline(input: { isCancellation, deWorkerInversion, deParallelRouter, + isDeRendererStall: isDeStall, }) ) throw err; @@ -3576,7 +3607,11 @@ async function executeRenderPipeline(input: { deFallbackFrameIndex = t.frameIndex; deFallbackThresholdDb = t.thresholdDb; } else { - deFallbackReason = isMemoryExhaustion ? "oom" : "capture_error"; + deFallbackReason = isMemoryExhaustion + ? "oom" + : isDeStall + ? "de_renderer_stall" + : "capture_error"; } log.warn( isVerifyError @@ -3737,32 +3772,50 @@ async function executeRenderPipeline(input: { try { captureRes = await invokeDiskCapture(capturePlan); } catch (err) { - // Disk-path drawElement self-verification tripped (a parallel disk - // worker's sampled frame diverged from its pre-injection ground - // truth — reachable only under the explicit fast-capture opt-in). - // Same recovery contract as the streaming drain: re-render on the - // screenshot baseline. Anything else keeps its existing semantics. + // Two disk-path failures re-render on the screenshot baseline, and + // they are NOT the same failure: + // - self-verification tripped (a parallel disk worker's sampled frame + // diverged from its pre-injection ground truth — reachable only + // under the explicit fast-capture opt-in); + // - the renderer wedged and blew the per-frame deadline + // (PRINFRA-488). The streaming drain already routed this; without + // it here, the same stall on the disk path threw straight out and + // failed the whole render, which is the behaviour the deadline was + // added to remove. + // Anything else keeps its existing semantics. + const isDiskDeStall = isDeRendererStallError(err); if ( - !isDrawElementVerificationError(err) || + (!isDrawElementVerificationError(err) && !isDiskDeStall) || err instanceof RenderCancelledError || executionSignal?.aborted === true ) { throw err; } - deSelfVerifyFallback = true; - const t = deVerifyFallbackTelemetry(err); - deFallbackReason = t.reason; - deFallbackFailedDb = t.failedDb; - deFallbackFrameIndex = t.frameIndex; - deFallbackThresholdDb = t.thresholdDb; + deSelfVerifyFallback = !isDiskDeStall; + if (isDiskDeStall) { + // No score exists for a stall, so only the reason is set — same + // shape the streaming catch uses for this reason. + deFallbackReason = "de_renderer_stall"; + } else { + const t = deVerifyFallbackTelemetry(err); + deFallbackReason = t.reason; + deFallbackFailedDb = t.failedDb; + deFallbackFrameIndex = t.frameIndex; + deFallbackThresholdDb = t.thresholdDb; + } log.warn( - "[Render] drawElement self-verification failed on the parallel disk path; " + - "re-rendering via screenshot", + isDiskDeStall + ? "[Render] drawElement wedged the renderer on the parallel disk path; " + + "re-rendering via screenshot" + : "[Render] drawElement self-verification failed on the parallel disk path; " + + "re-rendering via screenshot", { error: err instanceof Error ? err.message : String(err) }, ); observability.checkpoint( "capture_disk", - "drawElement self-verify failed; retrying with forceScreenshot", + isDiskDeStall + ? "drawElement renderer stall; retrying with forceScreenshot" + : "drawElement self-verify failed; retrying with forceScreenshot", ); // The failed attempt's frames are untrusted BUT satisfy the // completeness check — wipe them so the retry re-captures everything @@ -3783,7 +3836,10 @@ async function executeRenderPipeline(input: { probeSession = null; await closeOrphanedProbeForRetry(orphaned, closeCaptureSession, log, "disk verify"); } - capturePlan = replanAfterFailure(capturePlan, { kind: "draw_element_verification" }); + capturePlan = replanAfterFailure( + capturePlan, + isDiskDeStall ? { kind: "renderer_stall" } : { kind: "draw_element_verification" }, + ); syncCapturePlan(); updateCaptureObservability({ forceScreenshot: capturePlan.forceScreenshot,