From 8896d045ef682c3813396b11f5df4afea5241d96 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 15:17:24 +0800 Subject: [PATCH 1/6] test(app): add runtime CLS source gate --- .github/workflows/perf-probe-baseline.yml | 5 + .../app/e2e/perf/runtime-cls-gate.spec.ts | 349 ++++++++++ packages/app/e2e/perf/runtime-cls-probe.ts | 603 ++++++++++++++++++ .../app/e2e/perf/runtime-cls-probe.unit.ts | 165 +++++ packages/app/package.json | 2 + 5 files changed, 1124 insertions(+) create mode 100644 packages/app/e2e/perf/runtime-cls-gate.spec.ts create mode 100644 packages/app/e2e/perf/runtime-cls-probe.ts create mode 100644 packages/app/e2e/perf/runtime-cls-probe.unit.ts diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index d2f60abe9..3a51ca47e 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -148,6 +148,11 @@ jobs: PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head.json run: bun --cwd head/packages/app test:e2e:local:perf + - name: Run runtime CLS gate (head) + env: + CI: "true" + run: bun --cwd head/packages/app test:e2e:local:runtime-cls + - name: Run low-end perf probe baseline (base) if: steps.low_end_scope.outputs.run_low_end == 'true' env: diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts new file mode 100644 index 000000000..52f42b83a --- /dev/null +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -0,0 +1,349 @@ +import type { Page } from "@playwright/test" +import { test, expect } from "../fixtures" +import { cleanupSession, seedSessionQuestion, withSession } from "../actions" +import { inputMatch } from "../prompt/mock" +import { + promptSelector, + questionDockSelector, + scrollViewportSelector, + sessionMessageItemSelector, + sessionTurnListSelector, +} from "../selectors" +import { sessionPath } from "../utils" +import { readTimelineDomBudget } from "./timeline-dom-budget" +import { + collectRuntimeClsFailures, + formatRuntimeClsFailure, + installRuntimeClsProbe, + isRuntimeClsPrimaryEntry, + startRuntimeClsProbe, + stopRuntimeClsProbe, + type RuntimeClsResult, +} from "./runtime-cls-probe" + +const runtimeClsSeedTurns = 60 +const runtimeClsMinimumRows = 52 +const runtimeClsMaximumMountedMessages = 48 +const composerGrowthText = Array.from({ length: 8 }, (_, index) => `composer growth line ${index + 1}`).join("\n") + +const question = [ + { + header: "Runtime CLS check", + question: "Pick one option to close the dock", + options: [ + { label: "Continue", description: "Continue now" }, + { label: "Stop", description: "Stop here" }, + ], + }, +] + +type RuntimeClsProject = { + directory: string + sdk: { + session: { + promptAsync(input: { + sessionID: string + noReply: true + parts: Array<{ type: "text"; text: string }> + }): Promise + } + } +} + +function buildRuntimeClsSeedText(turn: number) { + const body = Array.from( + { length: 6 + (turn % 4) }, + (_, line) => `runtime cls seed turn ${turn} line ${line}: ${"mixed content ".repeat(8)}`, + ).join("\n") + const mixed = [ + ["## Markdown status", "", `- turn ${turn}`, "- runtime cls gate"].join("\n"), + ["```ts", `export const runtimeClsTurn${turn} = ${turn}`, "```"].join("\n"), + ["```diff", `- stale runtime cls row ${turn}`, `+ stable runtime cls row ${turn}`, "```"].join("\n"), + ["Reasoning summary", `- checked dock pressure for turn ${turn}`].join("\n"), + ][turn % 4] + return [`runtime cls fixture turn ${turn}`, body, mixed].join("\n\n") +} + +async function settleFrames(page: Page, count = 4) { + await page.evaluate(async (frames) => { + for (let index = 0; index < frames; index += 1) { + await Promise.race([ + new Promise((resolve) => requestAnimationFrame(() => resolve())), + new Promise((resolve) => setTimeout(resolve, 100)), + ]) + } + }, count) +} + +async function seedRuntimeClsSession(project: RuntimeClsProject, sessionID: string) { + for (let turn = 0; turn < runtimeClsSeedTurns; turn += 1) { + await project.sdk.session.promptAsync({ + sessionID, + noReply: true, + parts: [{ type: "text", text: buildRuntimeClsSeedText(turn) }], + }) + } +} + +async function scrollTimelineToRatio(page: Page, ratio: number) { + const found = await page.evaluate( + ({ ratio, scrollViewportSelector, turnListSelector }) => { + const list = document.querySelector(turnListSelector) + const viewport = list?.closest(scrollViewportSelector) + if (!(viewport instanceof HTMLElement)) return false + const maxScrollTop = Math.max(0, viewport.scrollHeight - viewport.clientHeight) + viewport.scrollTop = maxScrollTop * ratio + viewport.dispatchEvent(new Event("scroll", { bubbles: true })) + return true + }, + { ratio, scrollViewportSelector, turnListSelector: sessionTurnListSelector }, + ) + expect(found).toBe(true) +} + +async function readTimelineMetrics(page: Page) { + const metrics = await page.evaluate( + ({ scrollViewportSelector, turnListSelector }) => { + const list = document.querySelector(turnListSelector) + const viewport = list?.closest(scrollViewportSelector) + if (!(viewport instanceof HTMLElement)) return undefined + const maxScrollTop = Math.max(0, viewport.scrollHeight - viewport.clientHeight) + return { + scrollTop: viewport.scrollTop, + scrollHeight: viewport.scrollHeight, + clientHeight: viewport.clientHeight, + maxScrollTop, + distanceFromBottom: Math.max(0, maxScrollTop - viewport.scrollTop), + } + }, + { scrollViewportSelector, turnListSelector: sessionTurnListSelector }, + ) + expect(metrics).toBeTruthy() + return metrics! +} + +async function moveMouseOverTimeline(page: Page) { + const box = await page.locator(scrollViewportSelector).first().boundingBox() + expect(box).toBeTruthy() + if (!box) return + await page.mouse.move(box.x + box.width / 2, box.y + Math.min(140, box.height * 0.25)) +} + +async function positionTimelineForMeasuredWindow(page: Page) { + // A tiny real wheel gesture marks the timeline as user-scrolled. Without + // that, the active question turn can briefly re-lock to bottom and the + // measured close window becomes a bottom-follow transaction instead of the + // controlled middle-of-history dock-height transaction this gate owns. + await expect + .poll( + async () => { + await scrollTimelineToRatio(page, 0.45) + await moveMouseOverTimeline(page) + await page.mouse.wheel(0, -120) + await settleFrames(page, 2) + return (await readTimelineMetrics(page)).distanceFromBottom + }, + { timeout: 7_000 }, + ) + .toBeGreaterThan(300) +} + +async function revealRuntimeClsRows(page: Page) { + await test.step("wait for first runtime CLS message", async () => { + await expect(page.locator(sessionMessageItemSelector).first()).toBeVisible({ timeout: 30_000 }) + }) + + for (let attempt = 0; attempt < 24; attempt += 1) { + const budget = await readTimelineDomBudget(page) + if (budget.totalRows >= runtimeClsMinimumRows) return budget + + await test.step(`load earlier runtime CLS rows attempt ${attempt + 1}`, async () => { + await scrollTimelineToRatio(page, 0) + await settleFrames(page, 2) + + const loadEarlier = page.getByRole("button", { name: /Load earlier messages|加载更早的消息/i }).first() + if (await loadEarlier.isVisible().catch(() => false)) { + await loadEarlier.click({ timeout: 1_000 }).catch(() => undefined) + await settleFrames(page, 4) + } + }) + } + + await expect + .poll(async () => (await readTimelineDomBudget(page)).totalRows, { timeout: 1_000 }) + .toBeGreaterThanOrEqual(runtimeClsMinimumRows) + return await readTimelineDomBudget(page) +} + +async function centerVisibleMessageID(page: Page) { + const target = await page.evaluate( + ({ messageSelector, scrollViewportSelector }) => { + const viewport = document.querySelector(scrollViewportSelector) + const viewportRect = viewport?.getBoundingClientRect() + if (!viewportRect) return undefined + const center = viewportRect.top + viewportRect.height / 2 + const candidates = Array.from(document.querySelectorAll(messageSelector)) + .map((node) => { + const rect = node.getBoundingClientRect() + return { + id: node.getAttribute("data-message-id") ?? undefined, + top: rect.top, + bottom: rect.bottom, + distance: Math.abs(rect.top + rect.height / 2 - center), + } + }) + .filter((item) => item.id && item.bottom > viewportRect.top && item.top < viewportRect.bottom) + .sort((left, right) => left.distance - right.distance) + return candidates[0]?.id + }, + { messageSelector: sessionMessageItemSelector, scrollViewportSelector }, + ) + expect(target).toBeTruthy() + return target! +} + +async function prepareRuntimeClsWindow(page: Page, project: RuntimeClsProject, sessionID: string) { + await test.step("navigate to runtime CLS session", async () => { + await page.goto(sessionPath(project.directory, sessionID)) + }) + const budget = await test.step("reveal enough timeline rows for virtualized runtime CLS coverage", async () => { + return await revealRuntimeClsRows(page) + }) + expect(budget.totalRows).toBeGreaterThanOrEqual(runtimeClsMinimumRows) + expect(budget.hasVirtualizer).toBe(true) + expect(budget.mountedMessages).toBeLessThanOrEqual(runtimeClsMaximumMountedMessages) + + await test.step("position viewport away from top and bottom", async () => { + await positionTimelineForMeasuredWindow(page) + }) + return await test.step("select center visible message target", async () => { + return await centerVisibleMessageID(page) + }) +} + +async function readPromptText(page: Page) { + return page + .locator(promptSelector) + .first() + .evaluate((element) => { + const text = element instanceof HTMLElement ? element.innerText : element.textContent + return (text ?? "").replace(/\u200B/g, "").trim() + }) +} + +async function readPromptHeight(page: Page) { + const box = await page.locator(promptSelector).first().boundingBox() + expect(box).toBeTruthy() + return box?.height ?? 0 +} + +async function assertNoPrimaryRuntimeClsFailures(result: RuntimeClsResult) { + const failures = collectRuntimeClsFailures(result.entries) + const primaryEntries = result.entries.filter(isRuntimeClsPrimaryEntry) + expect( + failures, + formatRuntimeClsFailure({ + action: result.action, + entries: primaryEntries.length > 0 ? primaryEntries : result.entries, + snapshot: result.snapshot, + }), + ).toEqual([]) +} + +test.describe("runtime CLS source gate", () => { + test.setTimeout(180_000) + + test("composer growth does not move visible timeline primary sources", async ({ page, project }) => { + await installRuntimeClsProbe(page) + await project.open() + await withSession(project.sdk, `runtime cls composer growth ${Date.now()}`, async (session) => { + await seedRuntimeClsSession(project, session.id) + const targetMessageID = await prepareRuntimeClsWindow(page, project, session.id) + const prompt = page.locator(promptSelector).first() + await expect(prompt).toBeVisible() + await prompt.click() + await prompt.fill("") + await expect.poll(async () => readPromptText(page)).toBe("") + const beforeHeight = await readPromptHeight(page) + await settleFrames(page, 4) + + await startRuntimeClsProbe(page, "composer-growth", { targetMessageID }) + await prompt.fill(composerGrowthText) + await expect.poll(async () => readPromptHeight(page)).toBeGreaterThan(beforeHeight + 16) + await settleFrames(page, 6) + const result = await stopRuntimeClsProbe(page) + + await assertNoPrimaryRuntimeClsFailures(result) + }) + }) + + test("composer shrink does not move visible timeline primary sources", async ({ page, project }) => { + await installRuntimeClsProbe(page) + await project.open() + await withSession(project.sdk, `runtime cls composer shrink ${Date.now()}`, async (session) => { + await seedRuntimeClsSession(project, session.id) + const targetMessageID = await prepareRuntimeClsWindow(page, project, session.id) + const prompt = page.locator(promptSelector).first() + await expect(prompt).toBeVisible() + await prompt.click() + await prompt.fill(composerGrowthText) + const grownHeight = await expect + .poll(async () => readPromptHeight(page)) + .toBeGreaterThan(64) + .then(() => readPromptHeight(page)) + await settleFrames(page, 6) + + await startRuntimeClsProbe(page, "composer-shrink", { targetMessageID }) + await prompt.fill("") + await expect.poll(async () => readPromptHeight(page)).toBeLessThan(grownHeight - 16) + await settleFrames(page, 6) + const result = await stopRuntimeClsProbe(page) + + await assertNoPrimaryRuntimeClsFailures(result) + }) + }) + + test("question dock close does not move visible timeline primary sources", async ({ page, project, llm }) => { + // #818 owns question dock open/growth. This first #814 gate keeps the + // question path to close/shrink because the current deterministic seeding + // flow would otherwise mix dock opening with tool-message hydration. + await installRuntimeClsProbe(page) + await project.open() + await withSession(project.sdk, `runtime cls question close ${Date.now()}`, async (session) => { + await seedRuntimeClsSession(project, session.id) + const child = await project.sdk.session + .create({ title: `runtime cls child question ${Date.now()}`, parentID: session.id }) + .then((response) => response.data) + if (!child?.id) throw new Error("Child session create did not return an id") + project.trackSession(child.id) + const dock = page.locator(questionDockSelector) + try { + await test.step("seed child question dock outside the measured window", async () => { + await llm.toolMatch(inputMatch({ questions: question }), "question", { questions: question }) + await seedSessionQuestion(project.sdk, { sessionID: child.id, questions: question }) + }) + const targetMessageID = + await test.step("reveal a long visible parent timeline window with the dock open", async () => { + const target = await prepareRuntimeClsWindow(page, project, session.id) + await expect(dock).toBeVisible({ timeout: 30_000 }) + await settleFrames(page, 6) + return target + }) + + const result = await test.step("close the child question dock under the runtime CLS probe", async () => { + await startRuntimeClsProbe(page, "question-dock-close", { targetMessageID }) + await dock.getByRole("radio", { name: /Continue/i }).click() + await dock.getByRole("button", { name: /submit/i }).click() + await expect(dock).toHaveCount(0) + await expect(page.locator(promptSelector).first()).toBeVisible() + await settleFrames(page, 6) + return await stopRuntimeClsProbe(page) + }) + + await assertNoPrimaryRuntimeClsFailures(result) + } finally { + await cleanupSession({ sdk: project.sdk, sessionID: child.id }) + } + }) + }) +}) diff --git a/packages/app/e2e/perf/runtime-cls-probe.ts b/packages/app/e2e/perf/runtime-cls-probe.ts new file mode 100644 index 000000000..a2af2e8cf --- /dev/null +++ b/packages/app/e2e/perf/runtime-cls-probe.ts @@ -0,0 +1,603 @@ +import type { Page } from "@playwright/test" + +export type RuntimeClsRect = { + x: number + y: number + width: number + height: number + top: number + right: number + bottom: number + left: number +} + +export type RuntimeClsScrollMetrics = { + scrollTop: number + scrollHeight: number + clientHeight: number + maxScrollTop: number +} + +export type RuntimeClsSourceKind = + | "primary-message-wrapper" + | "primary-turn" + | "primary-turn-descendant" + | "residual-assistant-message" + | "dock-or-scroll-recovery" + | "other" + +export type RuntimeClsSourceNode = { + label: string + rect?: RuntimeClsRect + path: string[] +} + +export type RuntimeClsPrimaryAncestor = { + label: string + beforeRect?: RuntimeClsRect + afterRect?: RuntimeClsRect + visibleBefore: boolean + visibleAfter: boolean +} + +export type RuntimeClsSourceClassification = { + kind: RuntimeClsSourceKind + source: RuntimeClsSourceNode + primaryAncestor?: RuntimeClsPrimaryAncestor +} + +export type RuntimeClsEntry = { + at: number + value: number + hadRecentInput: boolean + sources: RuntimeClsSourceClassification[] +} + +export type RuntimeClsSnapshot = { + targetMessageID?: string + targetBeforeRect?: RuntimeClsRect + targetAfterRect?: RuntimeClsRect + renderMode?: string + totalRows?: number + mountedRows?: number + scrollBefore?: RuntimeClsScrollMetrics + scrollAfter?: RuntimeClsScrollMetrics +} + +export type RuntimeClsResult = { + action: string + startedAt: number + endedAt: number + entries: RuntimeClsEntry[] + snapshot: RuntimeClsSnapshot +} + +type RuntimeClsStartOptions = { + targetMessageID?: string +} + +type RuntimeClsWindow = Window & { + __pawwork_runtime_cls_probe?: { + start: (action: string, options?: RuntimeClsStartOptions) => void + stop: () => RuntimeClsResult + } +} + +type PrimaryBeforeRectStore = Pick, "get"> + +const primarySelector = '[data-message-id], [data-component="session-turn"]' +const assistantResidualSelector = + '[data-component="assistant-message"], [data-slot="session-turn-assistant-content"], [data-component="markdown"], [data-component="message-part"]' +const dockOrScrollSelector = + '[data-component="dock-prompt"], [data-component="session-prompt-dock"], [data-slot="question-options"], [data-slot="question-option"], [data-component="scroll-jump"], [data-action="scroll-to-bottom"]' + +// Absolute single-entry threshold for a large primary timeline shift. This is +// intentionally not the Web Vitals cumulative CLS threshold: the runtime gate +// fails only when one measured action produces a >0.02 LayoutShift entry whose +// source belongs to visible timeline content. +export const RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD = 0.02 + +const primaryFailureKinds = new Set([ + "primary-message-wrapper", + "primary-turn", + "primary-turn-descendant", +]) + +export function isRuntimeClsPrimarySource(source: RuntimeClsSourceClassification) { + return primaryFailureKinds.has(source.kind) +} + +export function isRuntimeClsPrimaryEntry(entry: RuntimeClsEntry) { + return entry.sources.some(isRuntimeClsPrimarySource) +} + +export function rectFromDomRect(input: DOMRect | RuntimeClsRect): RuntimeClsRect { + return { + x: input.x, + y: input.y, + width: input.width, + height: input.height, + top: input.top, + right: input.right, + bottom: input.bottom, + left: input.left, + } +} + +function isVisibleRect(rect: RuntimeClsRect | undefined, viewportHeight: number, viewportWidth: number) { + if (!rect) return false + return rect.bottom > 0 && rect.top < viewportHeight && rect.right > 0 && rect.left < viewportWidth +} + +function stableElementLabel(element: Element) { + const messageID = element.getAttribute("data-message-id") + if (messageID) return `[data-message-id="${messageID}"]` + + const dataMessage = element.getAttribute("data-message") + if (dataMessage) return `[data-message="${dataMessage}"]` + + const component = element.getAttribute("data-component") + if (component) return `[data-component="${component}"]` + + const slot = element.getAttribute("data-slot") + if (slot) return `[data-slot="${slot}"]` + + if (element.id) return `#${element.id}` + + const tag = element.tagName.toLowerCase() + const classes = Array.from(element.classList).slice(0, 3) + return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag +} + +function elementPath(element: Element) { + const path: string[] = [] + let current: Element | null = element + while (current && path.length < 8) { + path.push(stableElementLabel(current)) + current = current.parentElement + } + return path +} + +function findPrimaryAncestor(source: Element) { + return source.closest(primarySelector) +} + +function findPrimaryBeforeRect(primary: Element, store: PrimaryBeforeRectStore) { + const direct = store.get(primary) + if (direct) return direct + + const message = primary.closest("[data-message-id]") + if (message) { + const messageRect = store.get(message) + if (messageRect) return messageRect + } + + const turn = primary.closest('[data-component="session-turn"]') + if (turn) return store.get(turn) +} + +function sourceNodeSnapshot(source: Element): RuntimeClsSourceNode { + return { + label: stableElementLabel(source), + rect: rectFromDomRect(source.getBoundingClientRect()), + path: elementPath(source), + } +} + +function primaryAncestorSnapshot(input: { + source: Element + primary: Element | null + viewportHeight: number + viewportWidth: number + primaryBeforeRects: PrimaryBeforeRectStore +}): RuntimeClsPrimaryAncestor | undefined { + if (!input.primary) return undefined + const beforeRect = findPrimaryBeforeRect(input.primary, input.primaryBeforeRects) + const afterRect = rectFromDomRect(input.primary.getBoundingClientRect()) + return { + label: stableElementLabel(input.primary), + beforeRect, + afterRect, + visibleBefore: isVisibleRect(beforeRect, input.viewportHeight, input.viewportWidth), + visibleAfter: isVisibleRect(afterRect, input.viewportHeight, input.viewportWidth), + } +} + +export function classifyRuntimeClsSource( + source: Element | null, + input: { + viewportHeight: number + viewportWidth?: number + primaryBeforeRects: PrimaryBeforeRectStore + }, +): RuntimeClsSourceClassification { + if (!source) { + return { kind: "other", source: { label: "", path: [] } } + } + + const viewportWidth = input.viewportWidth ?? 1024 + const primary = findPrimaryAncestor(source) + const ancestor = primaryAncestorSnapshot({ + source, + primary, + viewportHeight: input.viewportHeight, + viewportWidth, + primaryBeforeRects: input.primaryBeforeRects, + }) + const sourceSnapshot = sourceNodeSnapshot(source) + + if (source.matches("[data-message-id]")) { + return { kind: "primary-message-wrapper", source: sourceSnapshot, primaryAncestor: ancestor } + } + + if (source.matches('[data-component="session-turn"]')) { + return { kind: "primary-turn", source: sourceSnapshot, primaryAncestor: ancestor } + } + + if (source.closest(dockOrScrollSelector)) { + return { kind: "dock-or-scroll-recovery", source: sourceSnapshot, primaryAncestor: ancestor } + } + + if (primary && ancestor?.visibleBefore && ancestor.visibleAfter) { + return { kind: "primary-turn-descendant", source: sourceSnapshot, primaryAncestor: ancestor } + } + + if (source.closest(assistantResidualSelector)) { + return { kind: "residual-assistant-message", source: sourceSnapshot, primaryAncestor: ancestor } + } + + return { kind: "other", source: sourceSnapshot, primaryAncestor: ancestor } +} + +export function collectRuntimeClsFailures(entries: RuntimeClsEntry[], threshold = RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD) { + return entries.filter((entry) => entry.value > threshold && isRuntimeClsPrimaryEntry(entry)) +} + +export function formatRuntimeClsFailure(input: { + action: string + entries: RuntimeClsEntry[] + snapshot: RuntimeClsSnapshot +}) { + const primaryEntries = input.entries.map((entry) => ({ + at: entry.at, + value: entry.value, + hadRecentInput: entry.hadRecentInput, + sources: entry.sources.map((source) => ({ + kind: source.kind, + label: source.source.label, + path: source.source.path, + sourceRect: source.source.rect, + primaryAncestor: source.primaryAncestor, + })), + })) + const maxValue = Math.max(0, ...input.entries.map((entry) => entry.value)) + const sourceSummary = input.entries + .flatMap((entry) => + entry.sources.map((source) => + [ + `entry=${entry.value}`, + `hadRecentInput=${entry.hadRecentInput}`, + `kind=${source.kind}`, + `source=${source.source.label}`, + `primary=${source.primaryAncestor?.label ?? ""}`, + ].join(" "), + ), + ) + .join("\n") + return [ + `Runtime CLS primary source gate failed during ${input.action}.`, + `Threshold: single entry > ${RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD}; max primary entry: ${maxValue}.`, + sourceSummary, + JSON.stringify( + { + action: input.action, + entries: primaryEntries, + snapshot: input.snapshot, + }, + null, + 2, + ), + ].join("\n") +} + +export async function installRuntimeClsProbe(page: Page) { + await page.addInitScript(() => { + type RuntimeClsRect = { + x: number + y: number + width: number + height: number + top: number + right: number + bottom: number + left: number + } + + type RuntimeClsSourceKind = + | "primary-message-wrapper" + | "primary-turn" + | "primary-turn-descendant" + | "residual-assistant-message" + | "dock-or-scroll-recovery" + | "other" + + type RuntimeClsSourceClassification = { + kind: RuntimeClsSourceKind + source: { label: string; rect?: RuntimeClsRect; path: string[] } + primaryAncestor?: { + label: string + beforeRect?: RuntimeClsRect + afterRect?: RuntimeClsRect + visibleBefore: boolean + visibleAfter: boolean + } + } + + type RuntimeClsEntry = { + at: number + value: number + hadRecentInput: boolean + sources: RuntimeClsSourceClassification[] + } + + type RuntimeClsScrollMetrics = { + scrollTop: number + scrollHeight: number + clientHeight: number + maxScrollTop: number + } + + type RuntimeClsSnapshot = { + targetMessageID?: string + targetBeforeRect?: RuntimeClsRect + targetAfterRect?: RuntimeClsRect + renderMode?: string + totalRows?: number + mountedRows?: number + scrollBefore?: RuntimeClsScrollMetrics + scrollAfter?: RuntimeClsScrollMetrics + } + + type RuntimeClsWindow = Window & { + __pawwork_runtime_cls_probe?: { + start: (action: string, options?: { targetMessageID?: string }) => void + stop: () => { + action: string + startedAt: number + endedAt: number + entries: RuntimeClsEntry[] + snapshot: RuntimeClsSnapshot + } + } + } + + const win = window as RuntimeClsWindow + if (win.__pawwork_runtime_cls_probe) return + + const primarySelector = '[data-message-id], [data-component="session-turn"]' + const assistantResidualSelector = + '[data-component="assistant-message"], [data-slot="session-turn-assistant-content"], [data-component="markdown"], [data-component="message-part"]' + const dockOrScrollSelector = + '[data-component="dock-prompt"], [data-component="session-prompt-dock"], [data-slot="question-options"], [data-slot="question-option"], [data-component="scroll-jump"], [data-action="scroll-to-bottom"]' + + const maxEntries = 256 + let action = "unknown" + let startedAt = 0 + let entries: RuntimeClsEntry[] = [] + let primaryBeforeRects = new WeakMap() + let snapshotBefore: RuntimeClsSnapshot = {} + + const rectFromDomRect = (input: DOMRect): RuntimeClsRect => ({ + x: input.x, + y: input.y, + width: input.width, + height: input.height, + top: input.top, + right: input.right, + bottom: input.bottom, + left: input.left, + }) + + const isVisibleRect = (rect: RuntimeClsRect | undefined) => { + if (!rect) return false + return rect.bottom > 0 && rect.top < window.innerHeight && rect.right > 0 && rect.left < window.innerWidth + } + + const stableElementLabel = (element: Element) => { + const messageID = element.getAttribute("data-message-id") + if (messageID) return `[data-message-id="${messageID}"]` + const dataMessage = element.getAttribute("data-message") + if (dataMessage) return `[data-message="${dataMessage}"]` + const component = element.getAttribute("data-component") + if (component) return `[data-component="${component}"]` + const slot = element.getAttribute("data-slot") + if (slot) return `[data-slot="${slot}"]` + if (element.id) return `#${element.id}` + const tag = element.tagName.toLowerCase() + const classes = Array.from(element.classList).slice(0, 3) + return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag + } + + const elementPath = (element: Element) => { + const path: string[] = [] + let current: Element | null = element + while (current && path.length < 8) { + path.push(stableElementLabel(current)) + current = current.parentElement + } + return path + } + + const sourceSnapshot = (source: Element) => ({ + label: stableElementLabel(source), + rect: rectFromDomRect(source.getBoundingClientRect()), + path: elementPath(source), + }) + + const findPrimaryBeforeRect = (primary: Element) => { + const direct = primaryBeforeRects.get(primary) + if (direct) return direct + const message = primary.closest("[data-message-id]") + if (message) { + const messageRect = primaryBeforeRects.get(message) + if (messageRect) return messageRect + } + const turn = primary.closest('[data-component="session-turn"]') + if (turn) return primaryBeforeRects.get(turn) + } + + const classifyElement = (element: Element | null): RuntimeClsSourceClassification => { + if (!element) return { kind: "other", source: { label: "", path: [] } } + const primary = element.closest(primarySelector) + const primaryAncestor = primary + ? { + label: stableElementLabel(primary), + beforeRect: findPrimaryBeforeRect(primary), + afterRect: rectFromDomRect(primary.getBoundingClientRect()), + visibleBefore: false, + visibleAfter: false, + } + : undefined + if (primaryAncestor) { + primaryAncestor.visibleBefore = isVisibleRect(primaryAncestor.beforeRect) + primaryAncestor.visibleAfter = isVisibleRect(primaryAncestor.afterRect) + } + const source = sourceSnapshot(element) + + if (element.matches("[data-message-id]")) return { kind: "primary-message-wrapper", source, primaryAncestor } + if (element.matches('[data-component="session-turn"]')) return { kind: "primary-turn", source, primaryAncestor } + if (element.closest(dockOrScrollSelector)) return { kind: "dock-or-scroll-recovery", source, primaryAncestor } + if (primary && primaryAncestor?.visibleBefore && primaryAncestor.visibleAfter) { + return { kind: "primary-turn-descendant", source, primaryAncestor } + } + if (element.closest(assistantResidualSelector)) + return { kind: "residual-assistant-message", source, primaryAncestor } + return { kind: "other", source, primaryAncestor } + } + + const readScrollMetrics = (): RuntimeClsScrollMetrics | undefined => { + const list = document.querySelector('[data-slot="session-turn-list"]') + const viewport = list?.closest('[data-component="scroll-viewport"]') + if (!(viewport instanceof HTMLElement)) return undefined + const maxScrollTop = Math.max(0, viewport.scrollHeight - viewport.clientHeight) + return { + scrollTop: viewport.scrollTop, + scrollHeight: viewport.scrollHeight, + clientHeight: viewport.clientHeight, + maxScrollTop, + } + } + + const messageByID = (id: string | undefined) => { + if (!id) return undefined + return Array.from(document.querySelectorAll("[data-message-id]")).find( + (node) => node.getAttribute("data-message-id") === id, + ) + } + + const readSnapshot = (targetMessageID?: string): RuntimeClsSnapshot => { + const list = document.querySelector('[data-slot="session-turn-list"]') as HTMLElement | null + const virtualRows = document.querySelectorAll('[data-component="session-virtual-row"]').length + const messages = document.querySelectorAll("[data-message-id]").length + const target = messageByID(targetMessageID) + return { + targetMessageID, + targetAfterRect: target instanceof Element ? rectFromDomRect(target.getBoundingClientRect()) : undefined, + renderMode: list?.dataset.renderMode, + totalRows: list?.dataset.totalRows ? Number(list.dataset.totalRows) : undefined, + mountedRows: virtualRows > 0 ? virtualRows : messages, + scrollAfter: readScrollMetrics(), + } + } + + const capturePrimaryBeforeRects = () => { + primaryBeforeRects = new WeakMap() + for (const element of document.querySelectorAll(primarySelector)) { + primaryBeforeRects.set(element, rectFromDomRect(element.getBoundingClientRect())) + } + } + + if (typeof PerformanceObserver !== "undefined") { + try { + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries() as Array< + PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + } + >) { + if (startedAt <= 0 || entry.startTime < startedAt) continue + if (typeof entry.value !== "number") continue + const sources = (entry.sources ?? []).map((source) => + classifyElement(source.node instanceof Element ? source.node : null), + ) + entries.push({ + at: entry.startTime, + value: entry.value, + hadRecentInput: entry.hadRecentInput === true, + sources, + }) + } + if (entries.length > maxEntries) entries = entries.slice(entries.length - maxEntries) + }) + observer.observe({ type: "layout-shift", buffered: true }) + } catch {} + } + + win.__pawwork_runtime_cls_probe = { + start(nextAction, options) { + action = nextAction + entries = [] + startedAt = performance.now() + capturePrimaryBeforeRects() + const before = readSnapshot(options?.targetMessageID) + snapshotBefore = { + targetMessageID: options?.targetMessageID, + targetBeforeRect: before.targetAfterRect, + renderMode: before.renderMode, + totalRows: before.totalRows, + mountedRows: before.mountedRows, + scrollBefore: before.scrollAfter, + } + }, + stop() { + const after = readSnapshot(snapshotBefore.targetMessageID) + return { + action, + startedAt, + endedAt: performance.now(), + entries: entries.slice(), + snapshot: { + ...snapshotBefore, + targetAfterRect: after.targetAfterRect, + renderMode: after.renderMode ?? snapshotBefore.renderMode, + totalRows: after.totalRows ?? snapshotBefore.totalRows, + mountedRows: after.mountedRows ?? snapshotBefore.mountedRows, + scrollAfter: after.scrollAfter, + }, + } + }, + } + }) +} + +export async function startRuntimeClsProbe(page: Page, action: string, options?: RuntimeClsStartOptions) { + await page.evaluate( + ({ action, options }) => { + const probe = (window as RuntimeClsWindow).__pawwork_runtime_cls_probe + if (!probe) throw new Error("Runtime CLS probe is not installed") + probe.start(action, options) + }, + { action, options }, + ) +} + +export async function stopRuntimeClsProbe(page: Page): Promise { + return await page.evaluate(() => { + const probe = (window as RuntimeClsWindow).__pawwork_runtime_cls_probe + if (!probe) throw new Error("Runtime CLS probe is not installed") + return probe.stop() + }) +} diff --git a/packages/app/e2e/perf/runtime-cls-probe.unit.ts b/packages/app/e2e/perf/runtime-cls-probe.unit.ts new file mode 100644 index 000000000..82248bf16 --- /dev/null +++ b/packages/app/e2e/perf/runtime-cls-probe.unit.ts @@ -0,0 +1,165 @@ +import { describe, expect, test } from "bun:test" +import { + RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD, + classifyRuntimeClsSource, + collectRuntimeClsFailures, + formatRuntimeClsFailure, + type RuntimeClsRect, +} from "./runtime-cls-probe" + +const rect = (input: Partial = {}): RuntimeClsRect => ({ + x: input.x ?? 20, + y: input.y ?? 120, + width: input.width ?? 640, + height: input.height ?? 120, + top: input.top ?? input.y ?? 120, + right: input.right ?? (input.x ?? 20) + (input.width ?? 640), + bottom: input.bottom ?? (input.y ?? 120) + (input.height ?? 120), + left: input.left ?? input.x ?? 20, +}) + +function setRect(element: Element, value: RuntimeClsRect) { + Object.defineProperty(element, "getBoundingClientRect", { + configurable: true, + value: () => ({ ...value, toJSON: () => value }), + }) +} + +function buildTurnFixture(input?: { before?: RuntimeClsRect; after?: RuntimeClsRect }) { + document.body.innerHTML = [ + '
', + '
', + '
', + '
visible assistant markdown
', + "
", + "
", + "
", + ].join("") + + const message = document.querySelector('[data-message-id="msg-1"]')! + const turn = document.querySelector('[data-component="session-turn"]')! + const assistant = document.querySelector('[data-slot="session-turn-assistant-content"]')! + const markdown = document.querySelector('[data-component="markdown"]')! + const before = input?.before ?? rect() + const after = input?.after ?? rect({ y: before.y + 12, top: before.top + 12, bottom: before.bottom + 12 }) + + setRect(message, after) + setRect(turn, after) + setRect(assistant, rect({ y: after.y + 24, top: after.top + 24, bottom: after.top + 64 })) + setRect(markdown, rect({ y: after.y + 32, top: after.top + 32, bottom: after.top + 72 })) + + return { message, turn, assistant, markdown, before, after } +} + +describe("runtime CLS source classifier", () => { + test("classifies a message wrapper source as a primary failure source", () => { + const { message, before } = buildTurnFixture() + + const result = classifyRuntimeClsSource(message, { + viewportHeight: 720, + primaryBeforeRects: new Map([[message, before]]), + }) + + expect(result.kind).toBe("primary-message-wrapper") + expect(result.primaryAncestor?.label).toBe('[data-message-id="msg-1"]') + }) + + test("classifies a direct session turn source as a primary failure source", () => { + const { turn, before } = buildTurnFixture() + + const result = classifyRuntimeClsSource(turn, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + expect(result.kind).toBe("primary-turn") + expect(result.primaryAncestor?.label).toBe('[data-component="session-turn"]') + }) + + test("promotes assistant descendants inside visible primary ancestors to primary-turn-descendant", () => { + const { markdown, turn, before } = buildTurnFixture() + + const result = classifyRuntimeClsSource(markdown, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + expect(result.kind).toBe("primary-turn-descendant") + expect(result.source.label).toBe('[data-component="markdown"]') + expect(result.primaryAncestor?.label).toBe('[data-component="session-turn"]') + expect(result.primaryAncestor?.beforeRect).toEqual(before) + expect(result.primaryAncestor?.afterRect).toEqual(rect({ y: 132, top: 132, bottom: 252 })) + }) + + test("keeps assistant descendants as residual when the primary ancestor was not visible before and after", () => { + const before = rect({ y: -280, top: -280, bottom: -160 }) + const after = rect({ y: -260, top: -260, bottom: -140 }) + const { markdown, turn } = buildTurnFixture({ before, after }) + + const result = classifyRuntimeClsSource(markdown, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + expect(result.kind).toBe("residual-assistant-message") + }) + + test("classifies dock sources as diagnostics instead of primary failures", () => { + document.body.innerHTML = '
' + const dockChild = document.querySelector('[data-slot="question-options"]')! + + const result = classifyRuntimeClsSource(dockChild, { viewportHeight: 720, primaryBeforeRects: new Map() }) + + expect(result.kind).toBe("dock-or-scroll-recovery") + }) +}) + +describe("runtime CLS failure threshold", () => { + test("fails only single-entry large primary timeline shifts over the absolute threshold", () => { + const { markdown, turn, before } = buildTurnFixture() + const primarySource = classifyRuntimeClsSource(markdown, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + expect(RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD).toBe(0.02) + expect( + collectRuntimeClsFailures([ + { at: 1, value: RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD, hadRecentInput: true, sources: [primarySource] }, + { at: 2, value: RUNTIME_CLS_PRIMARY_SHIFT_THRESHOLD + 0.001, hadRecentInput: true, sources: [primarySource] }, + ]), + ).toEqual([{ at: 2, value: 0.021, hadRecentInput: true, sources: [primarySource] }]) + }) +}) + +describe("runtime CLS failure diagnostics", () => { + test("prints action, value, source, primary ancestor, scroll metrics, render mode, and row counts", () => { + const { markdown, turn, before } = buildTurnFixture() + const source = classifyRuntimeClsSource(markdown, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + const message = formatRuntimeClsFailure({ + action: "composer-growth", + entries: [{ at: 12, value: 0.031, hadRecentInput: true, sources: [source] }], + snapshot: { + targetMessageID: "msg-1", + renderMode: "virtualized", + totalRows: 104, + mountedRows: 24, + scrollBefore: { scrollTop: 1200, scrollHeight: 8000, clientHeight: 720, maxScrollTop: 7280 }, + scrollAfter: { scrollTop: 1236, scrollHeight: 8036, clientHeight: 720, maxScrollTop: 7316 }, + }, + }) + + expect(message).toContain("composer-growth") + expect(message).toContain("0.031") + expect(message).toContain("primary-turn-descendant") + expect(message).toContain('[data-component="markdown"]') + expect(message).toContain('[data-component="session-turn"]') + expect(message).toContain("scrollTop") + expect(message).toContain("virtualized") + expect(message).toContain("104") + }) +}) diff --git a/packages/app/package.json b/packages/app/package.json index 1d6e6dd29..ec9d2dd13 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -23,7 +23,9 @@ "test:e2e:smoke": "playwright test --grep @smoke", "test:e2e:local": "bun script/e2e-local.ts", "test:e2e:local:perf": "bun script/e2e-local.ts -- e2e/perf/perf-probe.spec.ts", + "test:e2e:local:runtime-cls": "bun script/e2e-local.ts -- e2e/perf/runtime-cls-gate.spec.ts", "test:e2e:perf": "playwright test e2e/perf/perf-probe.spec.ts", + "test:e2e:runtime-cls": "playwright test e2e/perf/runtime-cls-gate.spec.ts", "test:e2e:local:smoke": "bun script/e2e-local.ts -- --grep @smoke", "test:e2e:ui": "playwright test --ui", "test:e2e:report": "playwright show-report e2e/playwright-report", From 35aec58acd15ead1adaf1cc45e5ab369b40d3cca Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 15:57:35 +0800 Subject: [PATCH 2/6] test(app): harden runtime CLS probe lifecycle --- .../app/e2e/perf/runtime-cls-gate.spec.ts | 146 ++++++++++++++++++ packages/app/e2e/perf/runtime-cls-probe.ts | 30 +++- 2 files changed, 172 insertions(+), 4 deletions(-) diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts index 52f42b83a..d31d38875 100644 --- a/packages/app/e2e/perf/runtime-cls-gate.spec.ts +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -250,6 +250,152 @@ async function assertNoPrimaryRuntimeClsFailures(result: RuntimeClsResult) { ).toEqual([]) } +async function installMockRuntimeClsObserver(page: Page, mode: "ready" | "observe-error") { + await page.addInitScript((mode) => { + type MockEntry = PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + } + + const callbacks: Array<(entries: MockEntry[]) => void> = [] + class MockPerformanceObserver { + private readonly callback: PerformanceObserverCallback + + constructor(callback: PerformanceObserverCallback) { + this.callback = callback + callbacks.push((entries) => { + this.callback({ getEntries: () => entries } as PerformanceObserverEntryList, this as PerformanceObserver) + }) + } + + observe() { + if (mode === "observe-error") throw new Error("mock layout-shift unsupported") + } + + disconnect() {} + takeRecords() { + return [] + } + + static supportedEntryTypes = ["layout-shift"] + } + + ;(window as typeof window & { PerformanceObserver: typeof PerformanceObserver }).PerformanceObserver = + MockPerformanceObserver as typeof PerformanceObserver + ;(window as typeof window & { __emitRuntimeClsEntry?: (entry: MockEntry) => void }).__emitRuntimeClsEntry = ( + entry, + ) => { + for (const callback of callbacks) callback([entry]) + } + }, mode) +} + +test.describe("runtime CLS probe lifecycle", () => { + test("fails instead of silently passing when layout-shift observer cannot start", async ({ page }) => { + await installMockRuntimeClsObserver(page, "observe-error") + await installRuntimeClsProbe(page) + await page.goto("about:blank") + + let errorMessage = "" + try { + await startRuntimeClsProbe(page, "observer-failure-check") + } catch (error) { + errorMessage = error instanceof Error ? error.message : String(error) + } + expect(errorMessage).toContain("layout-shift") + }) + + test("ignores layout-shift entries after stop until the next measured window", async ({ page }) => { + await installMockRuntimeClsObserver(page, "ready") + await installRuntimeClsProbe(page) + await page.goto("about:blank") + await page.setContent('
visible message
') + + await startRuntimeClsProbe(page, "first-window", { targetMessageID: "msg-1" }) + await stopRuntimeClsProbe(page) + const repeatedStop = await page.evaluate(() => { + const win = window as typeof window & { + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { value: number; sources: Array<{ node: Node | null }> }, + ) => void + __pawwork_runtime_cls_probe?: { stop: () => RuntimeClsResult } + } + const source = document.querySelector('[data-message-id="msg-1"]') + win.__emitRuntimeClsEntry?.({ + name: "layout-shift", + entryType: "layout-shift", + startTime: performance.now() + 1, + duration: 0, + toJSON: () => ({}), + value: 0.04, + sources: [{ node: source }], + }) + return win.__pawwork_runtime_cls_probe?.stop() + }) + + expect(repeatedStop?.entries).toEqual([]) + }) + + test("classifies sources through the installed browser probe", async ({ page }) => { + await installMockRuntimeClsObserver(page, "ready") + await installRuntimeClsProbe(page) + await page.goto("about:blank") + await page.setContent( + [ + '
', + '
', + '
assistant
', + "
", + "
", + '
option
', + ].join(""), + ) + + await startRuntimeClsProbe(page, "browser-classifier-primary", { targetMessageID: "msg-1" }) + const primaryResult = await page.evaluate(() => { + const win = window as typeof window & { + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { value: number; sources: Array<{ node: Node | null }> }, + ) => void + __pawwork_runtime_cls_probe?: { stop: () => RuntimeClsResult } + } + win.__emitRuntimeClsEntry?.({ + name: "layout-shift", + entryType: "layout-shift", + startTime: performance.now() + 1, + duration: 0, + toJSON: () => ({}), + value: 0.04, + sources: [{ node: document.querySelector('[data-component="markdown"]') }], + }) + return win.__pawwork_runtime_cls_probe?.stop() + }) + expect(primaryResult?.entries[0]?.sources[0]?.kind).toBe("primary-turn-descendant") + + await startRuntimeClsProbe(page, "browser-classifier-dock", { targetMessageID: "msg-1" }) + const dockResult = await page.evaluate(() => { + const win = window as typeof window & { + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { value: number; sources: Array<{ node: Node | null }> }, + ) => void + __pawwork_runtime_cls_probe?: { stop: () => RuntimeClsResult } + } + win.__emitRuntimeClsEntry?.({ + name: "layout-shift", + entryType: "layout-shift", + startTime: performance.now() + 1, + duration: 0, + toJSON: () => ({}), + value: 0.01, + sources: [{ node: document.querySelector('[data-slot="question-options"]') }], + }) + return win.__pawwork_runtime_cls_probe?.stop() + }) + expect(dockResult?.entries[0]?.sources[0]?.kind).toBe("dock-or-scroll-recovery") + }) +}) + test.describe("runtime CLS source gate", () => { test.setTimeout(180_000) diff --git a/packages/app/e2e/perf/runtime-cls-probe.ts b/packages/app/e2e/perf/runtime-cls-probe.ts index a2af2e8cf..73bf4783f 100644 --- a/packages/app/e2e/perf/runtime-cls-probe.ts +++ b/packages/app/e2e/perf/runtime-cls-probe.ts @@ -384,9 +384,12 @@ export async function installRuntimeClsProbe(page: Page) { const maxEntries = 256 let action = "unknown" let startedAt = 0 + let active = false let entries: RuntimeClsEntry[] = [] let primaryBeforeRects = new WeakMap() let snapshotBefore: RuntimeClsSnapshot = {} + let observerReady = false + let observerError: string | undefined const rectFromDomRect = (input: DOMRect): RuntimeClsRect => ({ x: input.x, @@ -518,7 +521,14 @@ export async function installRuntimeClsProbe(page: Page) { } } - if (typeof PerformanceObserver !== "undefined") { + if (typeof PerformanceObserver === "undefined") { + observerError = "PerformanceObserver is unavailable; runtime CLS gate cannot observe layout-shift entries." + } else if ( + Array.isArray(PerformanceObserver.supportedEntryTypes) && + !PerformanceObserver.supportedEntryTypes.includes("layout-shift") + ) { + observerError = "PerformanceObserver does not support layout-shift entries; runtime CLS gate cannot run." + } else { try { const observer = new PerformanceObserver((list) => { for (const entry of list.getEntries() as Array< @@ -528,7 +538,7 @@ export async function installRuntimeClsProbe(page: Page) { sources?: Array<{ node?: Node | null }> } >) { - if (startedAt <= 0 || entry.startTime < startedAt) continue + if (!active || startedAt <= 0 || entry.startTime < startedAt) continue if (typeof entry.value !== "number") continue const sources = (entry.sources ?? []).map((source) => classifyElement(source.node instanceof Element ? source.node : null), @@ -543,14 +553,21 @@ export async function installRuntimeClsProbe(page: Page) { if (entries.length > maxEntries) entries = entries.slice(entries.length - maxEntries) }) observer.observe({ type: "layout-shift", buffered: true }) - } catch {} + observerReady = true + } catch (error) { + observerError = error instanceof Error ? error.message : String(error) + } } win.__pawwork_runtime_cls_probe = { start(nextAction, options) { + if (!observerReady) { + throw new Error(observerError ?? "Runtime CLS layout-shift observer did not start.") + } action = nextAction entries = [] startedAt = performance.now() + active = true capturePrimaryBeforeRects() const before = readSnapshot(options?.targetMessageID) snapshotBefore = { @@ -564,7 +581,7 @@ export async function installRuntimeClsProbe(page: Page) { }, stop() { const after = readSnapshot(snapshotBefore.targetMessageID) - return { + const result = { action, startedAt, endedAt: performance.now(), @@ -578,6 +595,11 @@ export async function installRuntimeClsProbe(page: Page) { scrollAfter: after.scrollAfter, }, } + active = false + startedAt = 0 + entries = [] + primaryBeforeRects = new WeakMap() + return result }, } }) From 125005acb9206bbeef89e8c020205a42ac4bc8ee Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 16:16:15 +0800 Subject: [PATCH 3/6] test(app): stabilize runtime CLS lifecycle self-test --- .../app/e2e/perf/runtime-cls-gate.spec.ts | 50 ++--------------- packages/app/e2e/perf/runtime-cls-probe.ts | 54 +++++++++++++++++-- 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts index d31d38875..fd2145386 100644 --- a/packages/app/e2e/perf/runtime-cls-gate.spec.ts +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -250,51 +250,9 @@ async function assertNoPrimaryRuntimeClsFailures(result: RuntimeClsResult) { ).toEqual([]) } -async function installMockRuntimeClsObserver(page: Page, mode: "ready" | "observe-error") { - await page.addInitScript((mode) => { - type MockEntry = PerformanceEntry & { - value?: number - hadRecentInput?: boolean - sources?: Array<{ node?: Node | null }> - } - - const callbacks: Array<(entries: MockEntry[]) => void> = [] - class MockPerformanceObserver { - private readonly callback: PerformanceObserverCallback - - constructor(callback: PerformanceObserverCallback) { - this.callback = callback - callbacks.push((entries) => { - this.callback({ getEntries: () => entries } as PerformanceObserverEntryList, this as PerformanceObserver) - }) - } - - observe() { - if (mode === "observe-error") throw new Error("mock layout-shift unsupported") - } - - disconnect() {} - takeRecords() { - return [] - } - - static supportedEntryTypes = ["layout-shift"] - } - - ;(window as typeof window & { PerformanceObserver: typeof PerformanceObserver }).PerformanceObserver = - MockPerformanceObserver as typeof PerformanceObserver - ;(window as typeof window & { __emitRuntimeClsEntry?: (entry: MockEntry) => void }).__emitRuntimeClsEntry = ( - entry, - ) => { - for (const callback of callbacks) callback([entry]) - } - }, mode) -} - test.describe("runtime CLS probe lifecycle", () => { test("fails instead of silently passing when layout-shift observer cannot start", async ({ page }) => { - await installMockRuntimeClsObserver(page, "observe-error") - await installRuntimeClsProbe(page) + await installRuntimeClsProbe(page, { mockObserver: "observe-error" }) await page.goto("about:blank") let errorMessage = "" @@ -307,8 +265,7 @@ test.describe("runtime CLS probe lifecycle", () => { }) test("ignores layout-shift entries after stop until the next measured window", async ({ page }) => { - await installMockRuntimeClsObserver(page, "ready") - await installRuntimeClsProbe(page) + await installRuntimeClsProbe(page, { mockObserver: "ready" }) await page.goto("about:blank") await page.setContent('
visible message
') @@ -338,8 +295,7 @@ test.describe("runtime CLS probe lifecycle", () => { }) test("classifies sources through the installed browser probe", async ({ page }) => { - await installMockRuntimeClsObserver(page, "ready") - await installRuntimeClsProbe(page) + await installRuntimeClsProbe(page, { mockObserver: "ready" }) await page.goto("about:blank") await page.setContent( [ diff --git a/packages/app/e2e/perf/runtime-cls-probe.ts b/packages/app/e2e/perf/runtime-cls-probe.ts index 73bf4783f..b7caca039 100644 --- a/packages/app/e2e/perf/runtime-cls-probe.ts +++ b/packages/app/e2e/perf/runtime-cls-probe.ts @@ -83,6 +83,10 @@ type RuntimeClsWindow = Window & { } } +type RuntimeClsProbeInstallOptions = { + mockObserver?: "ready" | "observe-error" +} + type PrimaryBeforeRectStore = Pick, "get"> const primarySelector = '[data-message-id], [data-component="session-turn"]' @@ -301,8 +305,8 @@ export function formatRuntimeClsFailure(input: { ].join("\n") } -export async function installRuntimeClsProbe(page: Page) { - await page.addInitScript(() => { +export async function installRuntimeClsProbe(page: Page, options?: RuntimeClsProbeInstallOptions) { + await page.addInitScript((options?: RuntimeClsProbeInstallOptions) => { type RuntimeClsRect = { x: number y: number @@ -370,11 +374,55 @@ export async function installRuntimeClsProbe(page: Page) { snapshot: RuntimeClsSnapshot } } + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + }, + ) => void } const win = window as RuntimeClsWindow if (win.__pawwork_runtime_cls_probe) return + if (options?.mockObserver) { + type MockEntry = PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + } + + const callbacks: Array<(entries: MockEntry[]) => void> = [] + class MockPerformanceObserver { + private readonly callback: PerformanceObserverCallback + + constructor(callback: PerformanceObserverCallback) { + this.callback = callback + callbacks.push((entries) => { + this.callback({ getEntries: () => entries } as PerformanceObserverEntryList, this as PerformanceObserver) + }) + } + + observe() { + if (options.mockObserver === "observe-error") throw new Error("mock layout-shift unsupported") + } + + disconnect() {} + takeRecords() { + return [] + } + + static supportedEntryTypes = ["layout-shift"] + } + + ;(window as typeof window & { PerformanceObserver: typeof PerformanceObserver }).PerformanceObserver = + MockPerformanceObserver as typeof PerformanceObserver + win.__emitRuntimeClsEntry = (entry) => { + for (const callback of callbacks) callback([entry]) + } + } + const primarySelector = '[data-message-id], [data-component="session-turn"]' const assistantResidualSelector = '[data-component="assistant-message"], [data-slot="session-turn-assistant-content"], [data-component="markdown"], [data-component="message-part"]' @@ -602,7 +650,7 @@ export async function installRuntimeClsProbe(page: Page) { return result }, } - }) + }, options) } export async function startRuntimeClsProbe(page: Page, action: string, options?: RuntimeClsStartOptions) { From 9b16cd29a7a3f1d59cbde1fe8ab35acc033778ee Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 16:32:37 +0800 Subject: [PATCH 4/6] test(app): install runtime CLS probe in current document --- .../app/e2e/perf/runtime-cls-gate.spec.ts | 10 + packages/app/e2e/perf/runtime-cls-probe.ts | 611 +++++++++--------- 2 files changed, 317 insertions(+), 304 deletions(-) diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts index fd2145386..3db1b334c 100644 --- a/packages/app/e2e/perf/runtime-cls-gate.spec.ts +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -251,6 +251,16 @@ async function assertNoPrimaryRuntimeClsFailures(result: RuntimeClsResult) { } test.describe("runtime CLS probe lifecycle", () => { + test("installs into the current document after it has already loaded", async ({ page }) => { + await page.goto("about:blank") + await installRuntimeClsProbe(page, { mockObserver: "ready" }) + + await startRuntimeClsProbe(page, "same-document-install") + const result = await stopRuntimeClsProbe(page) + + expect(result.action).toBe("same-document-install") + }) + test("fails instead of silently passing when layout-shift observer cannot start", async ({ page }) => { await installRuntimeClsProbe(page, { mockObserver: "observe-error" }) await page.goto("about:blank") diff --git a/packages/app/e2e/perf/runtime-cls-probe.ts b/packages/app/e2e/perf/runtime-cls-probe.ts index b7caca039..c652ab931 100644 --- a/packages/app/e2e/perf/runtime-cls-probe.ts +++ b/packages/app/e2e/perf/runtime-cls-probe.ts @@ -305,352 +305,355 @@ export function formatRuntimeClsFailure(input: { ].join("\n") } -export async function installRuntimeClsProbe(page: Page, options?: RuntimeClsProbeInstallOptions) { - await page.addInitScript((options?: RuntimeClsProbeInstallOptions) => { - type RuntimeClsRect = { - x: number - y: number - width: number - height: number - top: number - right: number - bottom: number - left: number - } +function runtimeClsProbeInitScript(options?: RuntimeClsProbeInstallOptions) { + type RuntimeClsRect = { + x: number + y: number + width: number + height: number + top: number + right: number + bottom: number + left: number + } - type RuntimeClsSourceKind = - | "primary-message-wrapper" - | "primary-turn" - | "primary-turn-descendant" - | "residual-assistant-message" - | "dock-or-scroll-recovery" - | "other" - - type RuntimeClsSourceClassification = { - kind: RuntimeClsSourceKind - source: { label: string; rect?: RuntimeClsRect; path: string[] } - primaryAncestor?: { - label: string - beforeRect?: RuntimeClsRect - afterRect?: RuntimeClsRect - visibleBefore: boolean - visibleAfter: boolean - } + type RuntimeClsSourceKind = + | "primary-message-wrapper" + | "primary-turn" + | "primary-turn-descendant" + | "residual-assistant-message" + | "dock-or-scroll-recovery" + | "other" + + type RuntimeClsSourceClassification = { + kind: RuntimeClsSourceKind + source: { label: string; rect?: RuntimeClsRect; path: string[] } + primaryAncestor?: { + label: string + beforeRect?: RuntimeClsRect + afterRect?: RuntimeClsRect + visibleBefore: boolean + visibleAfter: boolean } + } - type RuntimeClsEntry = { - at: number - value: number - hadRecentInput: boolean - sources: RuntimeClsSourceClassification[] - } + type RuntimeClsEntry = { + at: number + value: number + hadRecentInput: boolean + sources: RuntimeClsSourceClassification[] + } - type RuntimeClsScrollMetrics = { - scrollTop: number - scrollHeight: number - clientHeight: number - maxScrollTop: number - } + type RuntimeClsScrollMetrics = { + scrollTop: number + scrollHeight: number + clientHeight: number + maxScrollTop: number + } - type RuntimeClsSnapshot = { - targetMessageID?: string - targetBeforeRect?: RuntimeClsRect - targetAfterRect?: RuntimeClsRect - renderMode?: string - totalRows?: number - mountedRows?: number - scrollBefore?: RuntimeClsScrollMetrics - scrollAfter?: RuntimeClsScrollMetrics - } + type RuntimeClsSnapshot = { + targetMessageID?: string + targetBeforeRect?: RuntimeClsRect + targetAfterRect?: RuntimeClsRect + renderMode?: string + totalRows?: number + mountedRows?: number + scrollBefore?: RuntimeClsScrollMetrics + scrollAfter?: RuntimeClsScrollMetrics + } - type RuntimeClsWindow = Window & { - __pawwork_runtime_cls_probe?: { - start: (action: string, options?: { targetMessageID?: string }) => void - stop: () => { - action: string - startedAt: number - endedAt: number - entries: RuntimeClsEntry[] - snapshot: RuntimeClsSnapshot - } + type RuntimeClsWindow = Window & { + __pawwork_runtime_cls_probe?: { + start: (action: string, options?: { targetMessageID?: string }) => void + stop: () => { + action: string + startedAt: number + endedAt: number + entries: RuntimeClsEntry[] + snapshot: RuntimeClsSnapshot } - __emitRuntimeClsEntry?: ( - entry: PerformanceEntry & { - value?: number - hadRecentInput?: boolean - sources?: Array<{ node?: Node | null }> - }, - ) => void } - - const win = window as RuntimeClsWindow - if (win.__pawwork_runtime_cls_probe) return - - if (options?.mockObserver) { - type MockEntry = PerformanceEntry & { + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { value?: number hadRecentInput?: boolean sources?: Array<{ node?: Node | null }> - } + }, + ) => void + } - const callbacks: Array<(entries: MockEntry[]) => void> = [] - class MockPerformanceObserver { - private readonly callback: PerformanceObserverCallback + const win = window as RuntimeClsWindow + if (win.__pawwork_runtime_cls_probe) return - constructor(callback: PerformanceObserverCallback) { - this.callback = callback - callbacks.push((entries) => { - this.callback({ getEntries: () => entries } as PerformanceObserverEntryList, this as PerformanceObserver) - }) - } + if (options?.mockObserver) { + type MockEntry = PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + } - observe() { - if (options.mockObserver === "observe-error") throw new Error("mock layout-shift unsupported") - } + const callbacks: Array<(entries: MockEntry[]) => void> = [] + class MockPerformanceObserver { + private readonly callback: PerformanceObserverCallback - disconnect() {} - takeRecords() { - return [] - } + constructor(callback: PerformanceObserverCallback) { + this.callback = callback + callbacks.push((entries) => { + this.callback({ getEntries: () => entries } as PerformanceObserverEntryList, this as PerformanceObserver) + }) + } - static supportedEntryTypes = ["layout-shift"] + observe() { + if (options.mockObserver === "observe-error") throw new Error("mock layout-shift unsupported") } - ;(window as typeof window & { PerformanceObserver: typeof PerformanceObserver }).PerformanceObserver = - MockPerformanceObserver as typeof PerformanceObserver - win.__emitRuntimeClsEntry = (entry) => { - for (const callback of callbacks) callback([entry]) + disconnect() {} + takeRecords() { + return [] } - } - const primarySelector = '[data-message-id], [data-component="session-turn"]' - const assistantResidualSelector = - '[data-component="assistant-message"], [data-slot="session-turn-assistant-content"], [data-component="markdown"], [data-component="message-part"]' - const dockOrScrollSelector = - '[data-component="dock-prompt"], [data-component="session-prompt-dock"], [data-slot="question-options"], [data-slot="question-option"], [data-component="scroll-jump"], [data-action="scroll-to-bottom"]' - - const maxEntries = 256 - let action = "unknown" - let startedAt = 0 - let active = false - let entries: RuntimeClsEntry[] = [] - let primaryBeforeRects = new WeakMap() - let snapshotBefore: RuntimeClsSnapshot = {} - let observerReady = false - let observerError: string | undefined - - const rectFromDomRect = (input: DOMRect): RuntimeClsRect => ({ - x: input.x, - y: input.y, - width: input.width, - height: input.height, - top: input.top, - right: input.right, - bottom: input.bottom, - left: input.left, - }) - - const isVisibleRect = (rect: RuntimeClsRect | undefined) => { - if (!rect) return false - return rect.bottom > 0 && rect.top < window.innerHeight && rect.right > 0 && rect.left < window.innerWidth + static supportedEntryTypes = ["layout-shift"] } - const stableElementLabel = (element: Element) => { - const messageID = element.getAttribute("data-message-id") - if (messageID) return `[data-message-id="${messageID}"]` - const dataMessage = element.getAttribute("data-message") - if (dataMessage) return `[data-message="${dataMessage}"]` - const component = element.getAttribute("data-component") - if (component) return `[data-component="${component}"]` - const slot = element.getAttribute("data-slot") - if (slot) return `[data-slot="${slot}"]` - if (element.id) return `#${element.id}` - const tag = element.tagName.toLowerCase() - const classes = Array.from(element.classList).slice(0, 3) - return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag + ;(window as typeof window & { PerformanceObserver: typeof PerformanceObserver }).PerformanceObserver = + MockPerformanceObserver as typeof PerformanceObserver + win.__emitRuntimeClsEntry = (entry) => { + for (const callback of callbacks) callback([entry]) } + } - const elementPath = (element: Element) => { - const path: string[] = [] - let current: Element | null = element - while (current && path.length < 8) { - path.push(stableElementLabel(current)) - current = current.parentElement - } - return path - } + const primarySelector = '[data-message-id], [data-component="session-turn"]' + const assistantResidualSelector = + '[data-component="assistant-message"], [data-slot="session-turn-assistant-content"], [data-component="markdown"], [data-component="message-part"]' + const dockOrScrollSelector = + '[data-component="dock-prompt"], [data-component="session-prompt-dock"], [data-slot="question-options"], [data-slot="question-option"], [data-component="scroll-jump"], [data-action="scroll-to-bottom"]' + + const maxEntries = 256 + let action = "unknown" + let startedAt = 0 + let active = false + let entries: RuntimeClsEntry[] = [] + let primaryBeforeRects = new WeakMap() + let snapshotBefore: RuntimeClsSnapshot = {} + let observerReady = false + let observerError: string | undefined + + const rectFromDomRect = (input: DOMRect): RuntimeClsRect => ({ + x: input.x, + y: input.y, + width: input.width, + height: input.height, + top: input.top, + right: input.right, + bottom: input.bottom, + left: input.left, + }) - const sourceSnapshot = (source: Element) => ({ - label: stableElementLabel(source), - rect: rectFromDomRect(source.getBoundingClientRect()), - path: elementPath(source), - }) - - const findPrimaryBeforeRect = (primary: Element) => { - const direct = primaryBeforeRects.get(primary) - if (direct) return direct - const message = primary.closest("[data-message-id]") - if (message) { - const messageRect = primaryBeforeRects.get(message) - if (messageRect) return messageRect - } - const turn = primary.closest('[data-component="session-turn"]') - if (turn) return primaryBeforeRects.get(turn) + const isVisibleRect = (rect: RuntimeClsRect | undefined) => { + if (!rect) return false + return rect.bottom > 0 && rect.top < window.innerHeight && rect.right > 0 && rect.left < window.innerWidth + } + + const stableElementLabel = (element: Element) => { + const messageID = element.getAttribute("data-message-id") + if (messageID) return `[data-message-id="${messageID}"]` + const dataMessage = element.getAttribute("data-message") + if (dataMessage) return `[data-message="${dataMessage}"]` + const component = element.getAttribute("data-component") + if (component) return `[data-component="${component}"]` + const slot = element.getAttribute("data-slot") + if (slot) return `[data-slot="${slot}"]` + if (element.id) return `#${element.id}` + const tag = element.tagName.toLowerCase() + const classes = Array.from(element.classList).slice(0, 3) + return classes.length > 0 ? `${tag}.${classes.join(".")}` : tag + } + + const elementPath = (element: Element) => { + const path: string[] = [] + let current: Element | null = element + while (current && path.length < 8) { + path.push(stableElementLabel(current)) + current = current.parentElement } + return path + } - const classifyElement = (element: Element | null): RuntimeClsSourceClassification => { - if (!element) return { kind: "other", source: { label: "", path: [] } } - const primary = element.closest(primarySelector) - const primaryAncestor = primary - ? { - label: stableElementLabel(primary), - beforeRect: findPrimaryBeforeRect(primary), - afterRect: rectFromDomRect(primary.getBoundingClientRect()), - visibleBefore: false, - visibleAfter: false, - } - : undefined - if (primaryAncestor) { - primaryAncestor.visibleBefore = isVisibleRect(primaryAncestor.beforeRect) - primaryAncestor.visibleAfter = isVisibleRect(primaryAncestor.afterRect) - } - const source = sourceSnapshot(element) + const sourceSnapshot = (source: Element) => ({ + label: stableElementLabel(source), + rect: rectFromDomRect(source.getBoundingClientRect()), + path: elementPath(source), + }) - if (element.matches("[data-message-id]")) return { kind: "primary-message-wrapper", source, primaryAncestor } - if (element.matches('[data-component="session-turn"]')) return { kind: "primary-turn", source, primaryAncestor } - if (element.closest(dockOrScrollSelector)) return { kind: "dock-or-scroll-recovery", source, primaryAncestor } - if (primary && primaryAncestor?.visibleBefore && primaryAncestor.visibleAfter) { - return { kind: "primary-turn-descendant", source, primaryAncestor } - } - if (element.closest(assistantResidualSelector)) - return { kind: "residual-assistant-message", source, primaryAncestor } - return { kind: "other", source, primaryAncestor } + const findPrimaryBeforeRect = (primary: Element) => { + const direct = primaryBeforeRects.get(primary) + if (direct) return direct + const message = primary.closest("[data-message-id]") + if (message) { + const messageRect = primaryBeforeRects.get(message) + if (messageRect) return messageRect } + const turn = primary.closest('[data-component="session-turn"]') + if (turn) return primaryBeforeRects.get(turn) + } - const readScrollMetrics = (): RuntimeClsScrollMetrics | undefined => { - const list = document.querySelector('[data-slot="session-turn-list"]') - const viewport = list?.closest('[data-component="scroll-viewport"]') - if (!(viewport instanceof HTMLElement)) return undefined - const maxScrollTop = Math.max(0, viewport.scrollHeight - viewport.clientHeight) - return { - scrollTop: viewport.scrollTop, - scrollHeight: viewport.scrollHeight, - clientHeight: viewport.clientHeight, - maxScrollTop, - } + const classifyElement = (element: Element | null): RuntimeClsSourceClassification => { + if (!element) return { kind: "other", source: { label: "", path: [] } } + const primary = element.closest(primarySelector) + const primaryAncestor = primary + ? { + label: stableElementLabel(primary), + beforeRect: findPrimaryBeforeRect(primary), + afterRect: rectFromDomRect(primary.getBoundingClientRect()), + visibleBefore: false, + visibleAfter: false, + } + : undefined + if (primaryAncestor) { + primaryAncestor.visibleBefore = isVisibleRect(primaryAncestor.beforeRect) + primaryAncestor.visibleAfter = isVisibleRect(primaryAncestor.afterRect) } + const source = sourceSnapshot(element) - const messageByID = (id: string | undefined) => { - if (!id) return undefined - return Array.from(document.querySelectorAll("[data-message-id]")).find( - (node) => node.getAttribute("data-message-id") === id, - ) + if (element.matches("[data-message-id]")) return { kind: "primary-message-wrapper", source, primaryAncestor } + if (element.matches('[data-component="session-turn"]')) return { kind: "primary-turn", source, primaryAncestor } + if (element.closest(dockOrScrollSelector)) return { kind: "dock-or-scroll-recovery", source, primaryAncestor } + if (primary && primaryAncestor?.visibleBefore && primaryAncestor.visibleAfter) { + return { kind: "primary-turn-descendant", source, primaryAncestor } } + if (element.closest(assistantResidualSelector)) + return { kind: "residual-assistant-message", source, primaryAncestor } + return { kind: "other", source, primaryAncestor } + } - const readSnapshot = (targetMessageID?: string): RuntimeClsSnapshot => { - const list = document.querySelector('[data-slot="session-turn-list"]') as HTMLElement | null - const virtualRows = document.querySelectorAll('[data-component="session-virtual-row"]').length - const messages = document.querySelectorAll("[data-message-id]").length - const target = messageByID(targetMessageID) - return { - targetMessageID, - targetAfterRect: target instanceof Element ? rectFromDomRect(target.getBoundingClientRect()) : undefined, - renderMode: list?.dataset.renderMode, - totalRows: list?.dataset.totalRows ? Number(list.dataset.totalRows) : undefined, - mountedRows: virtualRows > 0 ? virtualRows : messages, - scrollAfter: readScrollMetrics(), - } + const readScrollMetrics = (): RuntimeClsScrollMetrics | undefined => { + const list = document.querySelector('[data-slot="session-turn-list"]') + const viewport = list?.closest('[data-component="scroll-viewport"]') + if (!(viewport instanceof HTMLElement)) return undefined + const maxScrollTop = Math.max(0, viewport.scrollHeight - viewport.clientHeight) + return { + scrollTop: viewport.scrollTop, + scrollHeight: viewport.scrollHeight, + clientHeight: viewport.clientHeight, + maxScrollTop, } + } - const capturePrimaryBeforeRects = () => { - primaryBeforeRects = new WeakMap() - for (const element of document.querySelectorAll(primarySelector)) { - primaryBeforeRects.set(element, rectFromDomRect(element.getBoundingClientRect())) - } + const messageByID = (id: string | undefined) => { + if (!id) return undefined + return Array.from(document.querySelectorAll("[data-message-id]")).find( + (node) => node.getAttribute("data-message-id") === id, + ) + } + + const readSnapshot = (targetMessageID?: string): RuntimeClsSnapshot => { + const list = document.querySelector('[data-slot="session-turn-list"]') as HTMLElement | null + const virtualRows = document.querySelectorAll('[data-component="session-virtual-row"]').length + const messages = document.querySelectorAll("[data-message-id]").length + const target = messageByID(targetMessageID) + return { + targetMessageID, + targetAfterRect: target instanceof Element ? rectFromDomRect(target.getBoundingClientRect()) : undefined, + renderMode: list?.dataset.renderMode, + totalRows: list?.dataset.totalRows ? Number(list.dataset.totalRows) : undefined, + mountedRows: virtualRows > 0 ? virtualRows : messages, + scrollAfter: readScrollMetrics(), } + } - if (typeof PerformanceObserver === "undefined") { - observerError = "PerformanceObserver is unavailable; runtime CLS gate cannot observe layout-shift entries." - } else if ( - Array.isArray(PerformanceObserver.supportedEntryTypes) && - !PerformanceObserver.supportedEntryTypes.includes("layout-shift") - ) { - observerError = "PerformanceObserver does not support layout-shift entries; runtime CLS gate cannot run." - } else { - try { - const observer = new PerformanceObserver((list) => { - for (const entry of list.getEntries() as Array< - PerformanceEntry & { - value?: number - hadRecentInput?: boolean - sources?: Array<{ node?: Node | null }> - } - >) { - if (!active || startedAt <= 0 || entry.startTime < startedAt) continue - if (typeof entry.value !== "number") continue - const sources = (entry.sources ?? []).map((source) => - classifyElement(source.node instanceof Element ? source.node : null), - ) - entries.push({ - at: entry.startTime, - value: entry.value, - hadRecentInput: entry.hadRecentInput === true, - sources, - }) - } - if (entries.length > maxEntries) entries = entries.slice(entries.length - maxEntries) - }) - observer.observe({ type: "layout-shift", buffered: true }) - observerReady = true - } catch (error) { - observerError = error instanceof Error ? error.message : String(error) - } + const capturePrimaryBeforeRects = () => { + primaryBeforeRects = new WeakMap() + for (const element of document.querySelectorAll(primarySelector)) { + primaryBeforeRects.set(element, rectFromDomRect(element.getBoundingClientRect())) } + } - win.__pawwork_runtime_cls_probe = { - start(nextAction, options) { - if (!observerReady) { - throw new Error(observerError ?? "Runtime CLS layout-shift observer did not start.") - } - action = nextAction - entries = [] - startedAt = performance.now() - active = true - capturePrimaryBeforeRects() - const before = readSnapshot(options?.targetMessageID) - snapshotBefore = { - targetMessageID: options?.targetMessageID, - targetBeforeRect: before.targetAfterRect, - renderMode: before.renderMode, - totalRows: before.totalRows, - mountedRows: before.mountedRows, - scrollBefore: before.scrollAfter, - } - }, - stop() { - const after = readSnapshot(snapshotBefore.targetMessageID) - const result = { - action, - startedAt, - endedAt: performance.now(), - entries: entries.slice(), - snapshot: { - ...snapshotBefore, - targetAfterRect: after.targetAfterRect, - renderMode: after.renderMode ?? snapshotBefore.renderMode, - totalRows: after.totalRows ?? snapshotBefore.totalRows, - mountedRows: after.mountedRows ?? snapshotBefore.mountedRows, - scrollAfter: after.scrollAfter, - }, + if (typeof PerformanceObserver === "undefined") { + observerError = "PerformanceObserver is unavailable; runtime CLS gate cannot observe layout-shift entries." + } else if ( + Array.isArray(PerformanceObserver.supportedEntryTypes) && + !PerformanceObserver.supportedEntryTypes.includes("layout-shift") + ) { + observerError = "PerformanceObserver does not support layout-shift entries; runtime CLS gate cannot run." + } else { + try { + const observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries() as Array< + PerformanceEntry & { + value?: number + hadRecentInput?: boolean + sources?: Array<{ node?: Node | null }> + } + >) { + if (!active || startedAt <= 0 || entry.startTime < startedAt) continue + if (typeof entry.value !== "number") continue + const sources = (entry.sources ?? []).map((source) => + classifyElement(source.node instanceof Element ? source.node : null), + ) + entries.push({ + at: entry.startTime, + value: entry.value, + hadRecentInput: entry.hadRecentInput === true, + sources, + }) } - active = false - startedAt = 0 - entries = [] - primaryBeforeRects = new WeakMap() - return result - }, + if (entries.length > maxEntries) entries = entries.slice(entries.length - maxEntries) + }) + observer.observe({ type: "layout-shift", buffered: true }) + observerReady = true + } catch (error) { + observerError = error instanceof Error ? error.message : String(error) } - }, options) + } + + win.__pawwork_runtime_cls_probe = { + start(nextAction, options) { + if (!observerReady) { + throw new Error(observerError ?? "Runtime CLS layout-shift observer did not start.") + } + action = nextAction + entries = [] + startedAt = performance.now() + active = true + capturePrimaryBeforeRects() + const before = readSnapshot(options?.targetMessageID) + snapshotBefore = { + targetMessageID: options?.targetMessageID, + targetBeforeRect: before.targetAfterRect, + renderMode: before.renderMode, + totalRows: before.totalRows, + mountedRows: before.mountedRows, + scrollBefore: before.scrollAfter, + } + }, + stop() { + const after = readSnapshot(snapshotBefore.targetMessageID) + const result = { + action, + startedAt, + endedAt: performance.now(), + entries: entries.slice(), + snapshot: { + ...snapshotBefore, + targetAfterRect: after.targetAfterRect, + renderMode: after.renderMode ?? snapshotBefore.renderMode, + totalRows: after.totalRows ?? snapshotBefore.totalRows, + mountedRows: after.mountedRows ?? snapshotBefore.mountedRows, + scrollAfter: after.scrollAfter, + }, + } + active = false + startedAt = 0 + entries = [] + primaryBeforeRects = new WeakMap() + return result + }, + } +} + +export async function installRuntimeClsProbe(page: Page, options?: RuntimeClsProbeInstallOptions) { + await page.addInitScript(runtimeClsProbeInitScript, options) + await page.evaluate(runtimeClsProbeInitScript, options) } export async function startRuntimeClsProbe(page: Page, action: string, options?: RuntimeClsStartOptions) { From 261b7671efbde529636367067019652413d9a827 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 16:39:38 +0800 Subject: [PATCH 5/6] test(app): address runtime CLS review nits --- .../app/e2e/perf/runtime-cls-gate.spec.ts | 70 +++++++++---------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts index 3db1b334c..972841f07 100644 --- a/packages/app/e2e/perf/runtime-cls-gate.spec.ts +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -1,6 +1,6 @@ import type { Page } from "@playwright/test" import { test, expect } from "../fixtures" -import { cleanupSession, seedSessionQuestion, withSession } from "../actions" +import { seedSessionQuestion, withSession } from "../actions" import { inputMatch } from "../prompt/mock" import { promptSelector, @@ -21,12 +21,12 @@ import { type RuntimeClsResult, } from "./runtime-cls-probe" -const runtimeClsSeedTurns = 60 -const runtimeClsMinimumRows = 52 -const runtimeClsMaximumMountedMessages = 48 -const composerGrowthText = Array.from({ length: 8 }, (_, index) => `composer growth line ${index + 1}`).join("\n") +const RUNTIME_CLS_SEED_TURNS = 60 +const RUNTIME_CLS_MINIMUM_ROWS = 52 +const RUNTIME_CLS_MAXIMUM_MOUNTED_MESSAGES = 48 +const COMPOSER_GROWTH_TEXT = Array.from({ length: 8 }, (_, index) => `composer growth line ${index + 1}`).join("\n") -const question = [ +const QUESTION = [ { header: "Runtime CLS check", question: "Pick one option to close the dock", @@ -76,7 +76,7 @@ async function settleFrames(page: Page, count = 4) { } async function seedRuntimeClsSession(project: RuntimeClsProject, sessionID: string) { - for (let turn = 0; turn < runtimeClsSeedTurns; turn += 1) { + for (let turn = 0; turn < RUNTIME_CLS_SEED_TURNS; turn += 1) { await project.sdk.session.promptAsync({ sessionID, noReply: true, @@ -155,7 +155,7 @@ async function revealRuntimeClsRows(page: Page) { for (let attempt = 0; attempt < 24; attempt += 1) { const budget = await readTimelineDomBudget(page) - if (budget.totalRows >= runtimeClsMinimumRows) return budget + if (budget.totalRows >= RUNTIME_CLS_MINIMUM_ROWS) return budget await test.step(`load earlier runtime CLS rows attempt ${attempt + 1}`, async () => { await scrollTimelineToRatio(page, 0) @@ -171,7 +171,7 @@ async function revealRuntimeClsRows(page: Page) { await expect .poll(async () => (await readTimelineDomBudget(page)).totalRows, { timeout: 1_000 }) - .toBeGreaterThanOrEqual(runtimeClsMinimumRows) + .toBeGreaterThanOrEqual(RUNTIME_CLS_MINIMUM_ROWS) return await readTimelineDomBudget(page) } @@ -209,9 +209,9 @@ async function prepareRuntimeClsWindow(page: Page, project: RuntimeClsProject, s const budget = await test.step("reveal enough timeline rows for virtualized runtime CLS coverage", async () => { return await revealRuntimeClsRows(page) }) - expect(budget.totalRows).toBeGreaterThanOrEqual(runtimeClsMinimumRows) + expect(budget.totalRows).toBeGreaterThanOrEqual(RUNTIME_CLS_MINIMUM_ROWS) expect(budget.hasVirtualizer).toBe(true) - expect(budget.mountedMessages).toBeLessThanOrEqual(runtimeClsMaximumMountedMessages) + expect(budget.mountedMessages).toBeLessThanOrEqual(RUNTIME_CLS_MAXIMUM_MOUNTED_MESSAGES) await test.step("position viewport away from top and bottom", async () => { await positionTimelineForMeasuredWindow(page) @@ -380,7 +380,7 @@ test.describe("runtime CLS source gate", () => { await settleFrames(page, 4) await startRuntimeClsProbe(page, "composer-growth", { targetMessageID }) - await prompt.fill(composerGrowthText) + await prompt.fill(COMPOSER_GROWTH_TEXT) await expect.poll(async () => readPromptHeight(page)).toBeGreaterThan(beforeHeight + 16) await settleFrames(page, 6) const result = await stopRuntimeClsProbe(page) @@ -398,7 +398,7 @@ test.describe("runtime CLS source gate", () => { const prompt = page.locator(promptSelector).first() await expect(prompt).toBeVisible() await prompt.click() - await prompt.fill(composerGrowthText) + await prompt.fill(COMPOSER_GROWTH_TEXT) const grownHeight = await expect .poll(async () => readPromptHeight(page)) .toBeGreaterThan(64) @@ -429,33 +429,29 @@ test.describe("runtime CLS source gate", () => { if (!child?.id) throw new Error("Child session create did not return an id") project.trackSession(child.id) const dock = page.locator(questionDockSelector) - try { - await test.step("seed child question dock outside the measured window", async () => { - await llm.toolMatch(inputMatch({ questions: question }), "question", { questions: question }) - await seedSessionQuestion(project.sdk, { sessionID: child.id, questions: question }) - }) - const targetMessageID = - await test.step("reveal a long visible parent timeline window with the dock open", async () => { - const target = await prepareRuntimeClsWindow(page, project, session.id) - await expect(dock).toBeVisible({ timeout: 30_000 }) - await settleFrames(page, 6) - return target - }) - - const result = await test.step("close the child question dock under the runtime CLS probe", async () => { - await startRuntimeClsProbe(page, "question-dock-close", { targetMessageID }) - await dock.getByRole("radio", { name: /Continue/i }).click() - await dock.getByRole("button", { name: /submit/i }).click() - await expect(dock).toHaveCount(0) - await expect(page.locator(promptSelector).first()).toBeVisible() + await test.step("seed child question dock outside the measured window", async () => { + await llm.toolMatch(inputMatch({ questions: QUESTION }), "question", { questions: QUESTION }) + await seedSessionQuestion(project.sdk, { sessionID: child.id, questions: QUESTION }) + }) + const targetMessageID = + await test.step("reveal a long visible parent timeline window with the dock open", async () => { + const target = await prepareRuntimeClsWindow(page, project, session.id) + await expect(dock).toBeVisible({ timeout: 30_000 }) await settleFrames(page, 6) - return await stopRuntimeClsProbe(page) + return target }) - await assertNoPrimaryRuntimeClsFailures(result) - } finally { - await cleanupSession({ sdk: project.sdk, sessionID: child.id }) - } + const result = await test.step("close the child question dock under the runtime CLS probe", async () => { + await startRuntimeClsProbe(page, "question-dock-close", { targetMessageID }) + await dock.getByRole("radio", { name: /Continue/i }).click() + await dock.getByRole("button", { name: /submit/i }).click() + await expect(dock).toHaveCount(0) + await expect(page.locator(promptSelector).first()).toBeVisible() + await settleFrames(page, 6) + return await stopRuntimeClsProbe(page) + }) + + await assertNoPrimaryRuntimeClsFailures(result) }) }) }) From 13e94f8367e00e4dc6836249a58a1ba1f598814a Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 21 May 2026 16:42:27 +0800 Subject: [PATCH 6/6] test(app): tighten runtime CLS source assertions --- .../app/e2e/perf/runtime-cls-gate.spec.ts | 22 ++++++++++++++++++- packages/app/e2e/perf/runtime-cls-probe.ts | 18 ++++++++++----- .../app/e2e/perf/runtime-cls-probe.unit.ts | 21 ++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/packages/app/e2e/perf/runtime-cls-gate.spec.ts b/packages/app/e2e/perf/runtime-cls-gate.spec.ts index 972841f07..4984d8702 100644 --- a/packages/app/e2e/perf/runtime-cls-gate.spec.ts +++ b/packages/app/e2e/perf/runtime-cls-gate.spec.ts @@ -280,7 +280,27 @@ test.describe("runtime CLS probe lifecycle", () => { await page.setContent('
visible message
') await startRuntimeClsProbe(page, "first-window", { targetMessageID: "msg-1" }) - await stopRuntimeClsProbe(page) + const firstStop = await page.evaluate(() => { + const win = window as typeof window & { + __emitRuntimeClsEntry?: ( + entry: PerformanceEntry & { value: number; sources: Array<{ node: Node | null }> }, + ) => void + __pawwork_runtime_cls_probe?: { stop: () => RuntimeClsResult } + } + const source = document.querySelector('[data-message-id="msg-1"]') + win.__emitRuntimeClsEntry?.({ + name: "layout-shift", + entryType: "layout-shift", + startTime: performance.now() + 1, + duration: 0, + toJSON: () => ({}), + value: 0.04, + sources: [{ node: source }], + }) + return win.__pawwork_runtime_cls_probe?.stop() + }) + expect(firstStop?.entries).toHaveLength(1) + const repeatedStop = await page.evaluate(() => { const win = window as typeof window & { __emitRuntimeClsEntry?: ( diff --git a/packages/app/e2e/perf/runtime-cls-probe.ts b/packages/app/e2e/perf/runtime-cls-probe.ts index c652ab931..9d40050c6 100644 --- a/packages/app/e2e/perf/runtime-cls-probe.ts +++ b/packages/app/e2e/perf/runtime-cls-probe.ts @@ -230,12 +230,13 @@ export function classifyRuntimeClsSource( primaryBeforeRects: input.primaryBeforeRects, }) const sourceSnapshot = sourceNodeSnapshot(source) + const primaryVisible = ancestor?.visibleBefore === true && ancestor.visibleAfter === true - if (source.matches("[data-message-id]")) { + if (source.matches("[data-message-id]") && primaryVisible) { return { kind: "primary-message-wrapper", source: sourceSnapshot, primaryAncestor: ancestor } } - if (source.matches('[data-component="session-turn"]')) { + if (source.matches('[data-component="session-turn"]') && primaryVisible) { return { kind: "primary-turn", source: sourceSnapshot, primaryAncestor: ancestor } } @@ -243,7 +244,7 @@ export function classifyRuntimeClsSource( return { kind: "dock-or-scroll-recovery", source: sourceSnapshot, primaryAncestor: ancestor } } - if (primary && ancestor?.visibleBefore && ancestor.visibleAfter) { + if (primary && primaryVisible) { return { kind: "primary-turn-descendant", source: sourceSnapshot, primaryAncestor: ancestor } } @@ -514,11 +515,16 @@ function runtimeClsProbeInitScript(options?: RuntimeClsProbeInstallOptions) { primaryAncestor.visibleAfter = isVisibleRect(primaryAncestor.afterRect) } const source = sourceSnapshot(element) + const primaryVisible = primaryAncestor?.visibleBefore === true && primaryAncestor.visibleAfter === true - if (element.matches("[data-message-id]")) return { kind: "primary-message-wrapper", source, primaryAncestor } - if (element.matches('[data-component="session-turn"]')) return { kind: "primary-turn", source, primaryAncestor } + if (element.matches("[data-message-id]") && primaryVisible) { + return { kind: "primary-message-wrapper", source, primaryAncestor } + } + if (element.matches('[data-component="session-turn"]') && primaryVisible) { + return { kind: "primary-turn", source, primaryAncestor } + } if (element.closest(dockOrScrollSelector)) return { kind: "dock-or-scroll-recovery", source, primaryAncestor } - if (primary && primaryAncestor?.visibleBefore && primaryAncestor.visibleAfter) { + if (primary && primaryVisible) { return { kind: "primary-turn-descendant", source, primaryAncestor } } if (element.closest(assistantResidualSelector)) diff --git a/packages/app/e2e/perf/runtime-cls-probe.unit.ts b/packages/app/e2e/perf/runtime-cls-probe.unit.ts index 82248bf16..9390d87e3 100644 --- a/packages/app/e2e/perf/runtime-cls-probe.unit.ts +++ b/packages/app/e2e/perf/runtime-cls-probe.unit.ts @@ -76,6 +76,27 @@ describe("runtime CLS source classifier", () => { expect(result.primaryAncestor?.label).toBe('[data-component="session-turn"]') }) + test("does not classify off-screen direct primary sources as primary failures", () => { + const before = rect({ y: -280, top: -280, bottom: -160 }) + const after = rect({ y: -260, top: -260, bottom: -140 }) + const { message, turn } = buildTurnFixture({ before, after }) + + const messageResult = classifyRuntimeClsSource(message, { + viewportHeight: 720, + primaryBeforeRects: new Map([[message, before]]), + }) + const turnResult = classifyRuntimeClsSource(turn, { + viewportHeight: 720, + primaryBeforeRects: new Map([[turn, before]]), + }) + + expect(messageResult.kind).toBe("other") + expect(turnResult.kind).toBe("other") + expect(collectRuntimeClsFailures([{ at: 1, value: 0.04, hadRecentInput: true, sources: [messageResult] }])).toEqual( + [], + ) + }) + test("promotes assistant descendants inside visible primary ancestors to primary-turn-descendant", () => { const { markdown, turn, before } = buildTurnFixture()