Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions packages/cli/src/browser/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,22 @@ async function loadPuppeteerBrowsers(): Promise<PuppeteerBrowsers> {
}
}

const CHROME_VERSION = "152.0.7928.2";
// Bumped from 152.0.7928.2 on 2026-08-10 to pick up crbug 522872457's fix
// (CL 8032671, "Force-merge PendingLayer for canvas child descendant", merged
// 2026-07-07 — after the 152.0.7935.0 canary cut, so the old pin predated it).
//
// Measured on the shipping headless-shell binary, drawElementImage vs a CDP
// screenshot of the identical state:
// sibling content dropped from 3D captures — FIXED
// background lost from 3D captures — FIXED
// backface-visibility:hidden ignored — STILL BROKEN (1.4 dB -> 14.8 dB;
// better, still far under the
// 32 dB floor)
// So the 3D compile gate must stay. See PRINFRA-486.
//
// Deliberately Beta, not Canary. 153.0.8000.0 measured identical to this build
// on every probe variant, so crossing a major buys nothing measurable.
const CHROME_VERSION = "152.0.7977.30";
const CACHE_ROOT_DIR = join(homedir(), ".cache", "hyperframes");
const CACHE_DIR = join(homedir(), ".cache", "hyperframes", "chrome");
// Puppeteer's managed cache — where `@puppeteer/browsers install
Expand Down Expand Up @@ -332,7 +347,7 @@ async function findFromCache(): Promise<CacheLookupResult> {
// this is the non-`preferManagedChrome` path, which exists so a user who
// installed chrome-headless-shell separately (via `@puppeteer/browsers
// install`) keeps using that binary instead of being silently switched to
// the HF-pinned one. Note `CHROME_VERSION` (above) is a Dev-channel pin
// the HF-pinned one. Note `CHROME_VERSION` (above) is a pre-Stable pin
// that may be NEWER than a user's puppeteer-cache Stable build — this is
// about respecting an explicit prior choice, not "newest wins".
const fromPuppeteer = findFromPuppeteerCache();
Expand Down
1 change: 1 addition & 0 deletions packages/cli/src/commands/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1514,6 +1514,7 @@ function trackRenderMetrics(
deBlankRecaptures: perf?.drawElement?.blankRecaptures,
deBoundaryFrames: perf?.drawElement?.boundaryFrames,
deNcprFallbacks: perf?.drawElement?.ncprFallbacks,
deFrameTimeouts: perf?.drawElement?.frameTimeouts,
compositionDurationMs,
compositionWidth: perf?.resolution.width,
compositionHeight: perf?.resolution.height,
Expand Down
2 changes: 2 additions & 0 deletions packages/cli/src/telemetry/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ export function trackRenderComplete(
deBlankRecaptures?: number;
deBoundaryFrames?: number;
deNcprFallbacks?: number;
deFrameTimeouts?: number;
// "cli" when triggered by `hyperframes render` (default), "studio" when
// triggered by a studio preview-server render (POST /api/projects/:id/render).
source?: "cli" | "studio";
Expand Down Expand Up @@ -336,6 +337,7 @@ export function trackRenderComplete(
de_blank_recaptures: props.deBlankRecaptures,
de_boundary_frames: props.deBoundaryFrames,
de_ncpr_fallbacks: props.deNcprFallbacks,
de_frame_timeouts: props.deFrameTimeouts,
...powerStateFields(),
source: props.source ?? "cli",
composition_duration_ms: props.compositionDurationMs,
Expand Down
81 changes: 81 additions & 0 deletions packages/engine/src/services/frameCapture-frameDeadline.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/**
* Tests for the per-frame drawElement deadline (`withFrameDeadline`, PRINFRA-488).
*
* The deadline races the capture round-trip from OUTSIDE `captureFrameCore`,
* because puppeteer cannot abort an in-flight `page.evaluate`. That is exactly
* why the stall counter has to live in the `onTimeout` hook: a wedged renderer
* never returns, so no catch block inside the work promise ever runs. The first
* shipped version incremented `session.deFrameTimeouts` in that unreachable
* catch, so the counter — and the `CapturePerfSummary` field it feeds — read 0
* on every stalled render.
*/

import { readFileSync } from "node:fs";
import { describe, expect, it, vi } from "vitest";
import { withFrameDeadline } from "./frameCapture.js";

describe("withFrameDeadline", () => {
it("rejects with DeFrameTimeoutError and fires onTimeout when work outlives the deadline", async () => {
vi.useFakeTimers();
try {
const onTimeout = vi.fn();
// Never settles — the wedged-renderer shape.
const raced = withFrameDeadline(new Promise<string>(() => {}), "frame 7", 15_000, onTimeout);
const assertion = expect(raced).rejects.toThrow(/frame 7 exceeded 15000ms/);
await vi.advanceTimersByTimeAsync(15_000);
await assertion;
expect(onTimeout).toHaveBeenCalledTimes(1);
} finally {
vi.useRealTimers();
}
});

it("passes the value through and leaves onTimeout alone when work wins", async () => {
vi.useFakeTimers();
try {
const onTimeout = vi.fn();
const raced = withFrameDeadline(Promise.resolve("buffer"), "frame 7", 15_000, onTimeout);
await expect(raced).resolves.toBe("buffer");
// Past the deadline: the cleared timer must not fire late.
await vi.advanceTimersByTimeAsync(30_000);
expect(onTimeout).not.toHaveBeenCalled();
} finally {
vi.useRealTimers();
}
});
});

// The three drawElement capture entry points are only reachable with a real
// Chrome page, so the wiring is pinned at the source level instead. The first
// shipped deadline bounded ONLY the streaming path, leaving the disk and
// worker-encode paths to hit the 60 s render-level watchdog on the same wedged
// renderer — a coverage gap invisible to any unit test of the deadline itself.
describe("drawElement capture entry points", () => {
const source = readFileSync(new URL("./frameCapture.ts", import.meta.url), "utf8");

const bodyOf = (name: string): string => {
const start = source.indexOf(`export async function ${name}(`);
expect(start).toBeGreaterThan(-1);
// Up to the next top-level declaration — enough to cover the function body.
const next = source.indexOf("\nexport ", start + 1);
return source.slice(start, next === -1 ? undefined : next);
};

it.each(["captureFrameToBuffer", "captureFrame", "captureFrameToBufferPipelined"])(
"%s bounds its drawElement round-trip",
(entryPoint) => {
expect(bodyOf(entryPoint)).toContain("withDeFrameDeadline(");
},
);

// Diagnostics screenshot and evaluate against the page that just stopped
// scheduling, so running them on a stall spends the whole budget the deadline
// saved. The pipelined catch must bail before reaching them.
it("skips per-frame diagnostics when the pipelined path times out", () => {
const body = bodyOf("captureFrameToBufferPipelined");
const bail = body.indexOf("if (isDeFrameTimeoutError(captureError)) throw captureError;");
const diagnostics = body.indexOf("captureFrameErrorDiagnostics(");
expect(bail).toBeGreaterThan(-1);
expect(diagnostics).toBeGreaterThan(bail);
});
});
131 changes: 122 additions & 9 deletions packages/engine/src/services/frameCapture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,13 @@ export interface CaptureSession {
deVerifyInitMs?: number;
/** Count of per-frame "No cached paint record" screenshot fallbacks (telemetry). */
deNcprFallbacks?: number;
/**
* Count of drawElement frame captures that blew `HF_DE_FRAME_TIMEOUT_MS`
* because the renderer stopped scheduling after drawElementImage returned
* (PRINFRA-488). Each one aborts the drawElement attempt so the whole render
* retries via screenshot.
*/
deFrameTimeouts?: number;
/**
* drawElement init passed every gate but stopped before verification +
* canvas injection: the session has no video-frame injector yet (probe
Expand Down Expand Up @@ -3133,6 +3140,104 @@ function isNoCachedPaintRecordError(err: unknown): boolean {
return msg.includes("No cached paint record");
}

/**
* Per-frame deadline for the drawElement capture round-trip.
*
* drawElementImage can return normally and then leave the renderer not draining
* its task queue: the `setTimeout(…, 0)` that drawAndEncode schedules to run
* `toDataURL` never fires, so the capture `page.evaluate` never settles.
* Reproduced deterministically on Chromium 152.0.7977.30, one comp, always the
* same frame (PRINFRA-488). Nothing below the render-level watchdog bounded
* this, so a single bad frame failed the ENTIRE render after a 60 s stall.
*
* This bounds the round-trip so the frame can take the same per-frame screenshot
* fallback the `No cached paint record` case already takes — one slow frame
* instead of a dead render. Tune with `HF_DE_FRAME_TIMEOUT_MS`; 0 disables.
*/
const DE_FRAME_TIMEOUT_MS = Number(process.env.HF_DE_FRAME_TIMEOUT_MS ?? "15000");

class DeFrameTimeoutError extends Error {
constructor(label: string, ms: number) {
super(`drawElement ${label} exceeded ${ms}ms (renderer stopped scheduling; see PRINFRA-488)`);
this.name = "DeFrameTimeoutError";
}
}

/**
* Race `work` against a deadline. The losing promise is NOT cancellable —
* puppeteer cannot abort an in-flight `page.evaluate` — so its rejection is
* swallowed to avoid an unhandled rejection when it eventually settles (or
* never does). The orphaned round-trip keeps running in its Chrome worker;
* that worker is reclaimed by the outer retry rebuilding the page
* (`closeOrphanedProbeForRetry`), not by anything here.
*
* `onTimeout` fires exactly when the deadline wins, and is the ONLY place the
* stall is observable: because the deadline races `work` from outside, nothing
* inside `work` — including its own catch blocks — ever sees this error.
*
* Exported for the deadline unit test; `captureFrameToBuffer` is the only
* production caller.
*/
export async function withFrameDeadline<T>(
work: Promise<T>,
label: string,
ms: number,
onTimeout?: () => void,
): Promise<T> {
if (!(ms > 0)) return work;
let timer: ReturnType<typeof setTimeout> | undefined;
const guard = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
onTimeout?.();
reject(new DeFrameTimeoutError(label, ms));
}, ms);
});
try {
return await Promise.race([work, guard]);
} finally {
if (timer) clearTimeout(timer);
void work.catch(() => {
/* orphaned round-trip — see doc above */
});
}
}

function isDeFrameTimeoutError(err: unknown): boolean {
return err instanceof DeFrameTimeoutError;
}

/**
* Bound a drawElement round-trip, counting and logging the stall if the
* deadline wins. Every drawElement capture entry point goes through this —
* streaming (`captureFrameToBuffer`), disk (`captureFrame`) and worker-encode
* (`captureFrameToBufferPipelined`). Bounding only one of them left the other
* two hitting the 60 s render-level watchdog on the same wedged renderer,
* which is the whole failure this deadline exists to replace.
*/
function withDeFrameDeadline<T>(
session: CaptureSession,
frameIndex: number,
work: Promise<T>,
): Promise<T> {
if (session.captureMode !== "drawelement") return work;
return withFrameDeadline(work, `frame ${frameIndex}`, DE_FRAME_TIMEOUT_MS, () => {
// Deliberately NO per-frame screenshot fallback. When the renderer stops
// scheduling it is wedged for EVERY subsequent round-trip on that page —
// measured: the screenshot fallback blew the same deadline. Fail fast and
// let the producer re-render the whole comp on a fresh page via the
// screenshot path, the only recovery that works. Counted here rather than
// in a capture catch: the deadline rejects from OUTSIDE the work promise,
// so no catch inside it ever runs.
session.deFrameTimeouts = (session.deFrameTimeouts ?? 0) + 1;
console.log(
`[engine] fast capture: frame ${frameIndex} — capture exceeded ` +
`${DE_FRAME_TIMEOUT_MS}ms; renderer stalled after drawElementImage ` +
`(PRINFRA-488). Failing the drawElement attempt so the whole render ` +
`retries via screenshot.`,
);
});
}

async function captureFrameCore(
session: CaptureSession,
frameIndex: number,
Expand Down Expand Up @@ -3307,10 +3412,10 @@ export async function captureFrame(
frameIndex: number,
time: number,
): Promise<CaptureResult> {
const { buffer, quantizedTime, captureTimeMs } = await captureFrameCore(
const { buffer, quantizedTime, captureTimeMs } = await withDeFrameDeadline(
session,
frameIndex,
time,
captureFrameCore(session, frameIndex, time),
);
const framePath = writeCapturedFrame(session, frameIndex, buffer);
return { frameIndex, time: quantizedTime, path: framePath, captureTimeMs };
Expand Down Expand Up @@ -3344,7 +3449,11 @@ export async function captureFrameToBuffer(
frameIndex: number,
time: number,
): Promise<CaptureBufferResult> {
const { buffer, captureTimeMs } = await captureFrameCore(session, frameIndex, time);
const { buffer, captureTimeMs } = await withDeFrameDeadline(
session,
frameIndex,
captureFrameCore(session, frameIndex, time),
);

return { buffer, captureTimeMs };
}
Expand Down Expand Up @@ -3443,12 +3552,10 @@ export async function captureFrameToBufferPipelined(
// syncToPaintEvent = true); see initDrawElementOrTransparentBackground. The
// BeginFrame branch present in the synchronous captureFrameCore is therefore
// unreachable here and intentionally omitted.
const { encodeResult } = await produceDrawElementFrame(
page,
options.width,
options.height,
options.quality ?? 80,
true,
const { encodeResult } = await withDeFrameDeadline(
session,
frameIndex,
produceDrawElementFrame(page, options.width, options.height, options.quality ?? 80, true),
);

const captureTimeMs = Date.now() - startTime;
Expand Down Expand Up @@ -3482,6 +3589,11 @@ export async function captureFrameToBufferPipelined(
const buffer = await pageScreenshotCapture(page, options);
return { encodeResult: Promise.resolve(buffer), captureTimeMs: Date.now() - startTime };
}
// A blown deadline means the renderer is not draining its task queue, so
// the diagnostics below — which screenshot and evaluate against that same
// page — would block until the render-level watchdog fires, spending the
// whole budget the deadline just saved.
if (isDeFrameTimeoutError(captureError)) throw captureError;
// Mirror captureFrameCore: capture per-frame diagnostics (frame-error
// PNG/HTML/JSON + console tail) before rethrowing so pipelined-path
// failures are debuggable like the serial path.
Expand Down Expand Up @@ -3951,5 +4063,6 @@ export function getCapturePerfSummary(session: CaptureSession): CapturePerfSumma
deVerifyInitMs: session.deVerifyInitMs ?? 0,
deBoundaryFrames: session.clipBoundaryFrames?.size ?? 0,
deNcprFallbacks: session.deNcprFallbacks ?? 0,
deFrameTimeouts: session.deFrameTimeouts ?? 0,
};
}
38 changes: 36 additions & 2 deletions packages/engine/src/services/threeDProjection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -786,7 +786,41 @@ export async function initThreeDProjectionInPage(): Promise<ThreeDProjectionResu
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.useProgram(prog);

// canvas px → clip space, y flipped, z scaled into the clip volume
// canvas px → clip space, y flipped, z scaled into the clip volume AND
// NEGATED.
//
// The negation is load-bearing. CSS puts +z toward the viewer (see the
// convention note on the matrix helpers above); GL's depth buffer treats
// LARGER ndc z as FARTHER, and this context clears depth to 1 and tests
// LEQUAL. Passing CSS z through with a positive scale therefore inverted
// every depth comparison: the nearest quad got the largest depth, lost
// the test, and was occluded by the quad behind it.
//
// The y-flip on the row above changes handedness, which was already
// compensated for winding (frontFace(gl.CW) at group setup) but never
// for depth — so the bug only showed up once two quads in one context
// could be visible at the same time. A single-quad context has nothing to
// lose a depth test against, which is why it went unnoticed.
//
// Measured on a comp with two planes at ±45° in one preserve-3d context,
// drawElement render vs a screenshot render of the same comp:
// 18.8 dB (near plane painted behind the far one, labels and colours
// swapped) → 51.2 dB, visually identical to the screenshot arm. The
// single-quad cases are unchanged by the negation: backface flip card at
// rest 54.5 dB, perspective+rotationX 51.1 dB, matrix3d 49.5 dB.
//
// NOTE: this cannot be covered by a unit test — the whole enclosing
// function is contractually self-contained (no outer-scope references,
// it is shipped into the page via page.evaluate), so the matrix is not
// reachable from a test without breaking that contract. Verify it by
// rendering a two-plane preserve-3d comp on both capture paths and
// comparing; the self-verify net CANNOT catch a regression here, because
// it captures ground truth after this rewrite and would compare wrong
// geometry against wrong geometry (measured: a 65 dB "pass" over a
// truly-broken 18.8 dB render). The only backstop that can see this class
// of regression is a render-parity comp in a regression shard, which
// needs harness plumbing rather than a fixture drop — PRINFRA-570 carries
// the scope. Until it lands, THIS SIGN IS UNCOVERED BY CI.
const ndc: Mat4 = [
2 / canvasW,
0,
Expand All @@ -798,7 +832,7 @@ export async function initThreeDProjectionInPage(): Promise<ThreeDProjectionResu
1,
0,
0,
Z_SCALE,
-Z_SCALE,
0,
0,
0,
Expand Down
8 changes: 8 additions & 0 deletions packages/engine/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,14 @@ export interface CapturePerfSummary {
deBoundaryFrames: number;
/** Per-frame "No cached paint record" screenshot fallbacks during capture. */
deNcprFallbacks: number;
/**
* Per-frame drawElement captures that blew the `HF_DE_FRAME_TIMEOUT_MS`
* deadline and took the screenshot fallback (renderer stopped scheduling
* after drawElementImage returned — PRINFRA-488). Non-zero means the render
* completed only because the deadline caught a stall that previously failed
* the whole render.
*/
deFrameTimeouts: number;
}

// ── Global Augmentation ────────────────────────────────────────────────────────
Expand Down
Loading
Loading