From c8d0917d64818731eca53f77984a146ff8d8e5c5 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 14 May 2026 01:02:06 +0800 Subject: [PATCH 1/3] test(app): add PR0.4 low-end perf profile --- .github/workflows/perf-probe-baseline.yml | 147 +++++++++++++++++- packages/app/e2e/perf/perf-probe.spec.ts | 56 ++++++- packages/app/e2e/perf/probe.ts | 4 +- packages/app/e2e/perf/profiles.ts | 34 ++++ packages/app/e2e/perf/timeline-fixture.ts | 27 ++++ packages/app/script/merge-perf-artifacts.ts | 40 +++++ packages/app/src/testing/perf-metrics.test.ts | 113 +++++++++++++- packages/app/src/testing/perf-metrics.ts | 107 +++++++++++-- .../app/src/testing/perf-workflow.test.ts | 16 ++ 9 files changed, 521 insertions(+), 23 deletions(-) create mode 100644 packages/app/e2e/perf/profiles.ts create mode 100644 packages/app/e2e/perf/timeline-fixture.ts create mode 100644 packages/app/script/merge-perf-artifacts.ts create mode 100644 packages/app/src/testing/perf-workflow.test.ts diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 6d1c2fded..5f1c79285 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -11,6 +11,7 @@ on: - "packages/app/package.json" - "packages/app/playwright.config.ts" - "packages/app/script/compare-perf.ts" + - "packages/app/script/merge-perf-artifacts.ts" - "packages/app/script/e2e-local.ts" - "packages/app/src/testing/perf-metrics*" workflow_dispatch: @@ -31,11 +32,13 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6 with: + fetch-depth: 0 path: head persist-credentials: false - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # actions/checkout@v6 with: + fetch-depth: 0 path: base persist-credentials: false ref: ${{ github.event.pull_request.base.sha || github.sha }} @@ -78,6 +81,56 @@ jobs: cp head/packages/app/playwright.config.ts base/packages/app/playwright.config.ts cp head/packages/app/src/testing/perf-metrics.ts base/packages/app/src/testing/perf-metrics.ts + - name: Detect low-end perf scope + id: low_end_scope + working-directory: head + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + + is_low_end_path() { + case "$1" in + packages/app/src/pages/session/*|packages/app/src/pages/session/**|packages/app/src/pages/session.tsx|packages/ui/src/components/message-part.tsx|packages/ui/src/components/session-turn.tsx|packages/ui/src/components/markdown.tsx|packages/app/e2e/perf/*|packages/app/e2e/perf/**|packages/app/src/testing/perf-metrics*|packages/app/script/compare-perf.ts|packages/app/script/merge-perf-artifacts.ts|.github/workflows/perf-probe-baseline.yml) + return 0 + ;; + *) + return 1 + ;; + esac + } + + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + echo "run_low_end=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then + BASE_SHA="$(git rev-list --max-parents=0 HEAD | tail -n 1)" + fi + + run_low_end=false + mapfile -t changes < <(git diff --name-status --find-renames --find-copies "$BASE_SHA" "$HEAD_SHA" --) + for change in "${changes[@]}"; do + IFS=$'\t' read -r status path1 path2 <<< "$change" + case "$status" in + A*|M*|T*|D*) + if is_low_end_path "$path1"; then run_low_end=true; break; fi + ;; + R*|C*) + if is_low_end_path "$path1" || is_low_end_path "$path2"; then run_low_end=true; break; fi + ;; + *) + run_low_end=true + break + ;; + esac + done + + echo "run_low_end=$run_low_end" >> "$GITHUB_OUTPUT" + - name: Run perf probe baseline (base) env: CI: "true" @@ -92,13 +145,45 @@ jobs: PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head.json run: bun --cwd head/packages/app test:e2e:local:perf + - name: Run low-end perf probe baseline (base) + if: steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: base + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-low-end.json + run: bun --cwd base/packages/app test:e2e:local:perf + + - name: Run low-end perf probe baseline (head) + if: steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: head + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-low-end.json + run: bun --cwd head/packages/app test:e2e:local:perf + + - name: Merge perf profile artifacts (base) + run: > + bun head/packages/app/script/merge-perf-artifacts.ts + --required "${PERF_ARTIFACT_DIR}/perf-base.json" + --optional "${PERF_ARTIFACT_DIR}/perf-base-low-end.json" + --output "${PERF_ARTIFACT_DIR}/perf-base-combined.json" + + - name: Merge perf profile artifacts (head) + run: > + bun head/packages/app/script/merge-perf-artifacts.ts + --required "${PERF_ARTIFACT_DIR}/perf-head.json" + --optional "${PERF_ARTIFACT_DIR}/perf-head-low-end.json" + --output "${PERF_ARTIFACT_DIR}/perf-head-combined.json" + - name: Compare base and head perf baselines id: compare continue-on-error: true run: > bun head/packages/app/script/compare-perf.ts - --base "${PERF_ARTIFACT_DIR}/perf-base.json" - --head "${PERF_ARTIFACT_DIR}/perf-head.json" + --base "${PERF_ARTIFACT_DIR}/perf-base-combined.json" + --head "${PERF_ARTIFACT_DIR}/perf-head-combined.json" --output "${PERF_ARTIFACT_DIR}/perf-compare.json" --comment-output "${PERF_ARTIFACT_DIR}/perf-comment.md" @@ -118,14 +203,48 @@ jobs: PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-confirm.json run: bun --cwd head/packages/app test:e2e:local:perf + - name: Confirm low-end perf regression (base) + if: steps.compare.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: base + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-low-end-confirm.json + run: bun --cwd base/packages/app test:e2e:local:perf + + - name: Confirm low-end perf regression (head) + if: steps.compare.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: head + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-low-end-confirm.json + run: bun --cwd head/packages/app test:e2e:local:perf + + - name: Merge confirmed perf profile artifacts (base) + if: steps.compare.outcome == 'failure' + run: > + bun head/packages/app/script/merge-perf-artifacts.ts + --required "${PERF_ARTIFACT_DIR}/perf-base-confirm.json" + --optional "${PERF_ARTIFACT_DIR}/perf-base-low-end-confirm.json" + --output "${PERF_ARTIFACT_DIR}/perf-base-confirm-combined.json" + + - name: Merge confirmed perf profile artifacts (head) + if: steps.compare.outcome == 'failure' + run: > + bun head/packages/app/script/merge-perf-artifacts.ts + --required "${PERF_ARTIFACT_DIR}/perf-head-confirm.json" + --optional "${PERF_ARTIFACT_DIR}/perf-head-low-end-confirm.json" + --output "${PERF_ARTIFACT_DIR}/perf-head-confirm-combined.json" + - name: Compare confirmed perf baselines id: compare_confirmed if: steps.compare.outcome == 'failure' continue-on-error: true run: > bun head/packages/app/script/compare-perf.ts - --base "${PERF_ARTIFACT_DIR}/perf-base-confirm.json" - --head "${PERF_ARTIFACT_DIR}/perf-head-confirm.json" + --base "${PERF_ARTIFACT_DIR}/perf-base-confirm-combined.json" + --head "${PERF_ARTIFACT_DIR}/perf-head-confirm-combined.json" --output "${PERF_ARTIFACT_DIR}/perf-compare-confirm.json" --comment-output "${PERF_ARTIFACT_DIR}/perf-comment.md" @@ -188,6 +307,26 @@ jobs: PAWWORK_PERF_TRACE: "1" run: bun --cwd head/packages/app test:e2e:local:perf + - name: Capture low-end perf diagnostic trace (base) + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: base + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-low-end-trace.json + PAWWORK_PERF_TRACE: "1" + run: bun --cwd base/packages/app test:e2e:local:perf + + - name: Capture low-end perf diagnostic trace (head) + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + env: + CI: "true" + PAWWORK_PERF_BRANCH: head + PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-low-end-trace.json + PAWWORK_PERF_TRACE: "1" + run: bun --cwd head/packages/app test:e2e:local:perf + - name: Fail job on comparator regression if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' run: exit 1 diff --git a/packages/app/e2e/perf/perf-probe.spec.ts b/packages/app/e2e/perf/perf-probe.spec.ts index a34f0f9fc..0e037da33 100644 --- a/packages/app/e2e/perf/perf-probe.spec.ts +++ b/packages/app/e2e/perf/perf-probe.spec.ts @@ -12,9 +12,12 @@ import { } from "../selectors" import { sessionPath, terminalToggleKey } from "../utils" import { installPerfProbe, resetPerfProbe, snapshotPerfProbe, summarizeScenarioRuns } from "./probe" +import { applyPerfProfile, readPerfProfile, shouldRunScenario, type PerfScenarioName } from "./profiles" +import { seedTimelineRecomputeSession } from "./timeline-fixture" const outputPath = process.env.PAWWORK_PERF_OUTPUT ?? path.join(process.cwd(), "e2e", "perf-results", "pr0.1-baseline.json") const perfBranch = process.env.PAWWORK_PERF_BRANCH ?? "dev" +const perfProfile = readPerfProfile() const longMarkdown = [ "# Baseline stream", @@ -146,6 +149,10 @@ async function scrollTimelineTo(page: Parameters[0], t expect(found).toBe(true) } +function skipUnlessScenario(name: PerfScenarioName) { + test.skip(!shouldRunScenario(perfProfile, name), `${perfProfile} profile does not run ${name}`) +} + test.describe("PR0.1 perf probe baseline", () => { test.describe.configure({ mode: "serial" }) @@ -155,7 +162,9 @@ test.describe("PR0.1 perf probe baseline", () => { }) test("homepage-cold emits a 3-run JSON baseline", async ({ page, project }) => { + skipUnlessScenario("homepage-cold") await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) await project.open() const runs = [] @@ -171,11 +180,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, scenario: "homepage-cold", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "homepage-cold", runs })) }) test("session-streaming-long emits a 3-run JSON baseline", async ({ page, project, llm }) => { + skipUnlessScenario("session-streaming-long") await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) await project.open() const runs = [] @@ -216,11 +227,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, scenario: "session-streaming-long", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-streaming-long", runs })) }) test("tool-call-expand emits a 3-run JSON baseline", async ({ page, project, llm }) => { + skipUnlessScenario("tool-call-expand") await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) await project.open() const runs = [] @@ -246,11 +259,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, scenario: "tool-call-expand", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "tool-call-expand", runs })) }) test("terminal-side-panel-open emits a 3-run JSON baseline", async ({ page, project }) => { + skipUnlessScenario("terminal-side-panel-open") await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) await project.open() const runs = [] @@ -273,11 +288,13 @@ test.describe("PR0.1 perf probe baseline", () => { }) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, scenario: "terminal-side-panel-open", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "terminal-side-panel-open", runs })) }) test("session-scroll-reading emits a 3-run JSON baseline", async ({ page, project }) => { + skipUnlessScenario("session-scroll-reading") await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) await project.open() const runs = [] @@ -312,6 +329,35 @@ test.describe("PR0.1 perf probe baseline", () => { }) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, scenario: "session-scroll-reading", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-scroll-reading", runs })) + }) + + test("session-timeline-recompute emits a 3-run low-end JSON baseline", async ({ page, project }) => { + skipUnlessScenario("session-timeline-recompute") + await installPerfProbe(page) + await applyPerfProfile(page, perfProfile) + await project.open() + + const runs = [] + for (let run = 0; run < 3; run += 1) { + await withSession(project.sdk, `perf timeline recompute ${Date.now()}-${run}`, async (session) => { + await seedTimelineRecomputeSession(project, session.id) + await page.goto(sessionPath(project.directory, session.id)) + await expect(page.locator(sessionMessageItemSelector).first()).toBeVisible({ timeout: 30_000 }) + await expect.poll(async () => page.locator(sessionMessageItemSelector).count()).toBeGreaterThanOrEqual(8) + await resetPerfProbe(page) + await page.locator(scrollViewportSelector).first().hover() + for (let index = 0; index < 4; index += 1) { + await page.mouse.wheel(0, index % 2 === 0 ? 2400 : -2400) + await settleFrames(page, 2) + await scrollTimelineTo(page, index % 2 === 0 ? 0 : 1200) + await settleFrames(page, 2) + } + runs.push(await snapshotPerfProbe(page)) + if (run < 2) await cooldownAfterRun(page) + }) + } + + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-timeline-recompute", runs })) }) }) diff --git a/packages/app/e2e/perf/probe.ts b/packages/app/e2e/perf/probe.ts index ffd52f235..a01cb9979 100644 --- a/packages/app/e2e/perf/probe.ts +++ b/packages/app/e2e/perf/probe.ts @@ -1,5 +1,5 @@ import type { Page } from "@playwright/test" -import { aggregatePerfRuns, summarizePerfRun, type PerfRunSummary } from "../../src/testing/perf-metrics" +import { aggregatePerfRuns, summarizePerfRun, type PerfProfile, type PerfRunSummary } from "../../src/testing/perf-metrics" type BrowserPerfSample = { startedAt: number @@ -158,6 +158,6 @@ export async function snapshotPerfProbe(page: Page) { return summarizePerfRun(sample) } -export function summarizeScenarioRuns(input: { branch: string; scenario: string; runs: PerfRunSummary[] }) { +export function summarizeScenarioRuns(input: { branch: string; profile?: PerfProfile; scenario: string; runs: PerfRunSummary[] }) { return aggregatePerfRuns(input) } diff --git a/packages/app/e2e/perf/profiles.ts b/packages/app/e2e/perf/profiles.ts new file mode 100644 index 000000000..cf06f1313 --- /dev/null +++ b/packages/app/e2e/perf/profiles.ts @@ -0,0 +1,34 @@ +import type { Page } from "@playwright/test" +import type { PerfProfile } from "../../src/testing/perf-metrics" + +export type PerfScenarioName = + | "homepage-cold" + | "session-streaming-long" + | "tool-call-expand" + | "terminal-side-panel-open" + | "session-scroll-reading" + | "session-timeline-recompute" + +const defaultScenarios = new Set([ + "homepage-cold", + "session-streaming-long", + "tool-call-expand", + "terminal-side-panel-open", + "session-scroll-reading", +]) + +const lowEndScenarios = new Set(["session-timeline-recompute"]) + +export function readPerfProfile(): PerfProfile { + return process.env.PAWWORK_PERF_PROFILE === "low-end" ? "low-end" : "default" +} + +export function shouldRunScenario(profile: PerfProfile, scenario: PerfScenarioName) { + return profile === "low-end" ? lowEndScenarios.has(scenario) : defaultScenarios.has(scenario) +} + +export async function applyPerfProfile(page: Page, profile: PerfProfile) { + if (profile !== "low-end") return + const client = await page.context().newCDPSession(page) + await client.send("Emulation.setCPUThrottlingRate", { rate: 4 }) +} diff --git a/packages/app/e2e/perf/timeline-fixture.ts b/packages/app/e2e/perf/timeline-fixture.ts new file mode 100644 index 000000000..6146f9ad0 --- /dev/null +++ b/packages/app/e2e/perf/timeline-fixture.ts @@ -0,0 +1,27 @@ +type TimelineProject = { + sdk: { + session: { + promptAsync(input: { sessionID: string; noReply: true; parts: Array<{ type: "text"; text: string }> }): Promise + } + } +} + +export async function seedTimelineRecomputeSession(project: TimelineProject, sessionID: string) { + for (let turn = 0; turn < 36; turn += 1) { + await project.sdk.session.promptAsync({ + sessionID, + noReply: true, + parts: [ + { + type: "text", + text: [ + `timeline recompute seed turn ${turn}`, + `owner user-${turn % 4}`, + `plain text payload ${"content ".repeat(18)}`, + `status line ${turn % 3}`, + ].join("\n"), + }, + ], + }) + } +} diff --git a/packages/app/script/merge-perf-artifacts.ts b/packages/app/script/merge-perf-artifacts.ts new file mode 100644 index 000000000..be1a6c216 --- /dev/null +++ b/packages/app/script/merge-perf-artifacts.ts @@ -0,0 +1,40 @@ +import fs from "node:fs/promises" +import path from "node:path" +import type { PerfScenarioSummary } from "../src/testing/perf-metrics" + +function readArg(flag: string) { + const index = process.argv.indexOf(flag) + if (index === -1) return undefined + const value = process.argv[index + 1] + if (!value || value.startsWith("--")) return undefined + return value +} + +async function readPerfFile(filePath: string, required: boolean) { + try { + const payload = JSON.parse(await fs.readFile(filePath, "utf8")) as PerfScenarioSummary[] + if (!Array.isArray(payload)) { + throw new Error(`Expected an array of perf scenarios in ${filePath}`) + } + return payload + } catch (error) { + if (!required && (error as NodeJS.ErrnoException).code === "ENOENT") return [] + throw error + } +} + +async function main() { + const outputPath = readArg("--output") + const requiredPath = readArg("--required") + const optionalPath = readArg("--optional") + + if (!outputPath || !requiredPath) { + throw new Error("Usage: bun script/merge-perf-artifacts.ts --required [--optional ] --output ") + } + + const merged = [...(await readPerfFile(requiredPath, true)), ...(optionalPath ? await readPerfFile(optionalPath, false) : [])] + await fs.mkdir(path.dirname(outputPath), { recursive: true }) + await fs.writeFile(outputPath, `${JSON.stringify(merged, null, 2)}\n`) +} + +await main() diff --git a/packages/app/src/testing/perf-metrics.test.ts b/packages/app/src/testing/perf-metrics.test.ts index c9d1502dd..010550930 100644 --- a/packages/app/src/testing/perf-metrics.test.ts +++ b/packages/app/src/testing/perf-metrics.test.ts @@ -1,6 +1,41 @@ import { describe, expect, test } from "bun:test" import { aggregatePerfRuns, comparePerfBaselines, comparePerfScenarioSummaries, PERF_COMMENT_MARKER, renderPerfBaselineComment, summarizePerfRun } from "./perf-metrics" +function scenario(input: { + branch?: string + profile?: "default" | "low-end" + scenario?: string + interaction?: number + worst?: number + longTask?: number + tbt?: number + frameP95?: number + frameMax?: number + jank?: number + cls?: number +}) { + const value = input.interaction ?? 40 + return { + branch: input.branch ?? "base", + profile: input.profile ?? "default", + scenario: input.scenario ?? "homepage-cold", + runs: 3, + interaction_ms_median: value, + interaction_ms_worst: input.worst ?? value, + interaction_ms: value, + interaction_delay_ms: 0, + long_task_count: input.longTask && input.longTask > 0 ? 1 : 0, + long_task_max_ms: input.longTask ?? 0, + tbt_ms: input.tbt ?? 0, + frame_gap_p95_ms: input.frameP95 ?? 16.8, + frame_gap_max_ms: input.frameMax ?? 16.8, + jank_count_50ms: input.jank ?? 0, + cls: input.cls ?? 0, + window_ms: 900, + run_details: [], + } +} + describe("perf metrics", () => { test("summarizes a perf sample window", () => { const summary = summarizePerfRun({ @@ -104,6 +139,7 @@ describe("perf metrics", () => { }) expect(summary.branch).toBe("dev") + expect(summary.profile).toBe("default") expect(summary.scenario).toBe("tool-call-expand") expect(summary.runs).toBe(3) expect(summary.interaction_ms_median).toBe(52) @@ -122,6 +158,7 @@ describe("perf metrics", () => { scenario: "session-streaming-long", base: { branch: "base", + profile: "default", scenario: "session-streaming-long", runs: 3, interaction_ms_median: 100, @@ -140,6 +177,7 @@ describe("perf metrics", () => { }, head: { branch: "head", + profile: "default", scenario: "session-streaming-long", runs: 3, interaction_ms_median: 116, @@ -168,6 +206,7 @@ describe("perf metrics", () => { scenario: "tool-call-expand", base: { branch: "base", + profile: "default", scenario: "tool-call-expand", runs: 3, interaction_ms_median: 120, @@ -186,6 +225,7 @@ describe("perf metrics", () => { }, head: { branch: "head", + profile: "default", scenario: "tool-call-expand", runs: 3, interaction_ms_median: 120, @@ -213,6 +253,7 @@ describe("perf metrics", () => { scenario: "homepage-cold", base: { branch: "base", + profile: "default", scenario: "homepage-cold", runs: 3, interaction_ms_median: 70, @@ -233,6 +274,7 @@ describe("perf metrics", () => { }, head: { branch: "head", + profile: "default", scenario: "homepage-cold", runs: 3, interaction_ms_median: 74, @@ -302,7 +344,72 @@ describe("perf metrics", () => { const result = comparePerfBaselines({ base, head }) expect(result.pass).toBe(false) - expect(result.failures).toContain("missing_head_scenario:tool-call-expand") + expect(result.failures).toContain("missing_head_scenario:default:tool-call-expand") + }) + + test("compares baseline collections by profile and scenario", () => { + const base = [ + scenario({ branch: "base", profile: "default", scenario: "session-timeline-recompute" }), + scenario({ branch: "base", profile: "low-end", scenario: "session-timeline-recompute" }), + ] + const head = [ + scenario({ branch: "head", profile: "default", scenario: "session-timeline-recompute" }), + scenario({ branch: "head", profile: "low-end", scenario: "session-timeline-recompute", interaction: 60 }), + ] + + const result = comparePerfBaselines({ base, head }) + + expect(result.scenarios.map((entry) => `${entry.profile}:${entry.scenario}`)).toEqual([ + "default:session-timeline-recompute", + "low-end:session-timeline-recompute", + ]) + }) + + test("fails when the matching profile and scenario is missing", () => { + const base = [ + scenario({ branch: "base", profile: "default", scenario: "session-timeline-recompute" }), + scenario({ branch: "base", profile: "low-end", scenario: "session-timeline-recompute" }), + ] + const head = [scenario({ branch: "head", profile: "default", scenario: "session-timeline-recompute" })] + + const result = comparePerfBaselines({ base, head }) + + expect(result.pass).toBe(false) + expect(result.failures).toContain("missing_head_scenario:low-end:session-timeline-recompute") + }) + + test("keeps low-end moderate regressions warning-only", () => { + const result = comparePerfScenarioSummaries({ + scenario: "session-timeline-recompute", + base: scenario({ branch: "base", profile: "low-end", interaction: 100, worst: 150 }), + head: scenario({ branch: "head", profile: "low-end", interaction: 126, worst: 190 }), + }) + + expect(result.pass).toBe(true) + expect(result.failures).toHaveLength(0) + expect(result.warnings).toContain("interaction_ms_median") + }) + + test("fails low-end catastrophic regressions", () => { + const result = comparePerfScenarioSummaries({ + scenario: "session-timeline-recompute", + base: scenario({ branch: "base", profile: "low-end", interaction: 120, worst: 240, longTask: 120 }), + head: scenario({ branch: "head", profile: "low-end", interaction: 180, worst: 760, longTask: 360 }), + }) + + expect(result.pass).toBe(false) + expect(result.failures).toEqual(expect.arrayContaining(["interaction_ms_worst", "long_task_max_ms"])) + }) + + test("does not fail low-end absolute slowness without regression", () => { + const result = comparePerfScenarioSummaries({ + scenario: "session-timeline-recompute", + base: scenario({ branch: "base", profile: "low-end", interaction: 120, worst: 760, longTask: 340 }), + head: scenario({ branch: "head", profile: "low-end", interaction: 121, worst: 761, longTask: 341 }), + }) + + expect(result.pass).toBe(true) + expect(result.failures).toHaveLength(0) }) test("renders a markdown comment with scenario deltas and fail or warn status", () => { @@ -391,8 +498,8 @@ describe("perf metrics", () => { expect(comment).toContain(PERF_COMMENT_MARKER) expect(comment).toContain("## Perf delta summary") - expect(comment).toContain("| homepage-cold |") - expect(comment).toContain("| session-scroll-reading |") + expect(comment).toContain("| default / homepage-cold |") + expect(comment).toContain("| default / session-scroll-reading |") expect(comment).toContain("warn: fcp_ms, lcp_ms") expect(comment).toContain("fail: interaction_ms_median") }) diff --git a/packages/app/src/testing/perf-metrics.ts b/packages/app/src/testing/perf-metrics.ts index bedb132cf..3ec9dbb0c 100644 --- a/packages/app/src/testing/perf-metrics.ts +++ b/packages/app/src/testing/perf-metrics.ts @@ -42,8 +42,11 @@ export type PerfRunSummary = { heap_used_mb?: number } +export type PerfProfile = "default" | "low-end" + export type PerfScenarioSummary = PerfRunSummary & { branch: string + profile: PerfProfile scenario: string runs: number interaction_ms_median: number @@ -52,6 +55,7 @@ export type PerfScenarioSummary = PerfRunSummary & { } export type PerfScenarioComparison = { + profile: PerfProfile scenario: string pass: boolean failures: string[] @@ -95,6 +99,24 @@ const perfCatastrophicThresholds = { longTaskMaxMs: 250, } as const +const lowEndWarningThresholds = { + interactionMedianMs: 20, + interactionMedianRatio: 1.15, + interactionWorstMs: 80, + longTaskMaxMs: 60, + tbtMs: 120, + frameGapP95Ms: 20, + frameGapMaxMs: 120, + jankCount: 4, + cls: 0.04, +} as const + +const lowEndCatastrophicThresholds = { + interactionMsWorst: 700, + frameGapMaxMs: 700, + longTaskMaxMs: 300, +} as const + function round(input: number) { return Math.round(input * 1000) / 1000 } @@ -160,6 +182,7 @@ export function summarizePerfRun(input: PerfRunSample): PerfRunSummary { export function aggregatePerfRuns(input: { branch: string + profile?: PerfProfile scenario: string runs: PerfRunSummary[] }): PerfScenarioSummary { @@ -170,6 +193,7 @@ export function aggregatePerfRuns(input: { return { branch: input.branch, + profile: input.profile ?? "default", scenario: input.scenario, runs: runs.length, interaction_ms_median: round(median(runs.map((run) => run.interaction_ms))), @@ -226,13 +250,13 @@ export function renderPerfBaselineComment(input: PerfBaselineComparison) { "", `Comparator: ${input.pass ? "pass" : "fail"}`, "", - "| Scenario | interaction median | interaction worst | long task max | tbt | frame gap p95 | frame gap max | jank count | cls | status |", + "| Profile / Scenario | interaction median | interaction worst | long task max | tbt | frame gap p95 | frame gap max | jank count | cls | status |", "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", ] for (const scenario of input.scenarios) { const columns = [ - scenario.scenario, + `${scenario.profile} / ${scenario.scenario}`, formatMetricDelta(scenario.base.interaction_ms_median, scenario.head.interaction_ms_median), formatMetricDelta(scenario.base.interaction_ms_worst, scenario.head.interaction_ms_worst), formatMetricDelta(scenario.base.long_task_max_ms, scenario.head.long_task_max_ms), @@ -261,6 +285,64 @@ export function comparePerfScenarioSummaries(input: { }): PerfScenarioComparison { const failures: string[] = [] const warnings: string[] = [] + const profile = input.head.profile ?? input.base.profile ?? "default" + + if (profile === "low-end") { + if ( + exceededByDelta({ + base: input.base.interaction_ms_median, + head: input.head.interaction_ms_median, + maxDelta: lowEndWarningThresholds.interactionMedianMs, + maxRatio: lowEndWarningThresholds.interactionMedianRatio, + }) + ) { + warnings.push("interaction_ms_median") + } + const interactionWorstRegressed = + input.head.interaction_ms_worst > input.base.interaction_ms_worst + lowEndWarningThresholds.interactionWorstMs + if (interactionWorstRegressed) { + warnings.push("interaction_ms_worst_delta") + } + if (interactionWorstRegressed && input.head.interaction_ms_worst >= lowEndCatastrophicThresholds.interactionMsWorst) { + failures.push("interaction_ms_worst") + } + const longTaskRegressed = input.head.long_task_max_ms > input.base.long_task_max_ms + lowEndWarningThresholds.longTaskMaxMs + if (longTaskRegressed) { + warnings.push("long_task_max_ms_delta") + } + if (longTaskRegressed && input.head.long_task_max_ms >= lowEndCatastrophicThresholds.longTaskMaxMs) { + failures.push("long_task_max_ms") + } + if (input.head.tbt_ms > input.base.tbt_ms + lowEndWarningThresholds.tbtMs) { + warnings.push("tbt_ms") + } + if (input.head.frame_gap_p95_ms > input.base.frame_gap_p95_ms + lowEndWarningThresholds.frameGapP95Ms) { + warnings.push("frame_gap_p95_ms") + } + const frameGapMaxRegressed = input.head.frame_gap_max_ms > input.base.frame_gap_max_ms + lowEndWarningThresholds.frameGapMaxMs + if (frameGapMaxRegressed) { + warnings.push("frame_gap_max_ms_delta") + } + if (frameGapMaxRegressed && input.head.frame_gap_max_ms >= lowEndCatastrophicThresholds.frameGapMaxMs) { + failures.push("frame_gap_max_ms") + } + if (input.head.jank_count_50ms > input.base.jank_count_50ms + lowEndWarningThresholds.jankCount) { + warnings.push("jank_count_50ms") + } + if (input.head.cls > input.base.cls + lowEndWarningThresholds.cls) { + warnings.push("cls_delta") + } + + return { + profile, + scenario: input.scenario, + pass: failures.length === 0, + failures: [...new Set(failures)], + warnings: [...new Set(warnings)], + base: input.base, + head: input.head, + } + } if ( exceededByDelta({ @@ -311,6 +393,7 @@ export function comparePerfScenarioSummaries(input: { addAbsoluteWarning(warnings, "lcp_ms", input.head.lcp_ms, perfAbsoluteWarnings.lcpMs) return { + profile, scenario: input.scenario, pass: failures.length === 0, failures: [...new Set(failures)], @@ -320,6 +403,10 @@ export function comparePerfScenarioSummaries(input: { } } +function scenarioKey(input: { profile?: PerfProfile; scenario: string }) { + return `${input.profile ?? "default"}:${input.scenario}` +} + export function comparePerfBaselines(input: { base: PerfScenarioSummary[] head: PerfScenarioSummary[] @@ -327,12 +414,13 @@ export function comparePerfBaselines(input: { const failures: string[] = [] const warnings: string[] = [] const scenarios: PerfScenarioComparison[] = [] - const headByScenario = new Map(input.head.map((scenario) => [scenario.scenario, scenario])) + const headByScenario = new Map(input.head.map((scenario) => [scenarioKey(scenario), scenario])) for (const baseScenario of input.base) { - const headScenario = headByScenario.get(baseScenario.scenario) + const key = scenarioKey(baseScenario) + const headScenario = headByScenario.get(key) if (!headScenario) { - failures.push(`missing_head_scenario:${baseScenario.scenario}`) + failures.push(`missing_head_scenario:${key}`) continue } const comparison = comparePerfScenarioSummaries({ @@ -341,13 +429,14 @@ export function comparePerfBaselines(input: { head: headScenario, }) scenarios.push(comparison) - for (const failure of comparison.failures) failures.push(`${baseScenario.scenario}:${failure}`) - for (const warning of comparison.warnings) warnings.push(`${baseScenario.scenario}:${warning}`) + for (const failure of comparison.failures) failures.push(`${key}:${failure}`) + for (const warning of comparison.warnings) warnings.push(`${key}:${warning}`) } for (const headScenario of input.head) { - if (!input.base.some((scenario) => scenario.scenario === headScenario.scenario)) { - failures.push(`missing_base_scenario:${headScenario.scenario}`) + const key = scenarioKey(headScenario) + if (!input.base.some((scenario) => scenarioKey(scenario) === key)) { + failures.push(`missing_base_scenario:${key}`) } } diff --git a/packages/app/src/testing/perf-workflow.test.ts b/packages/app/src/testing/perf-workflow.test.ts new file mode 100644 index 000000000..6cd5ee314 --- /dev/null +++ b/packages/app/src/testing/perf-workflow.test.ts @@ -0,0 +1,16 @@ +import fs from "node:fs/promises" +import { describe, expect, test } from "bun:test" + +describe("perf workflow contract", () => { + test("keeps default gate broad and low-end gate scoped", async () => { + const workflow = await fs.readFile(new URL("../../../../.github/workflows/perf-probe-baseline.yml", import.meta.url), "utf8") + + expect(workflow).toContain("fetch-depth: 0") + expect(workflow).toContain("Detect low-end perf scope") + expect(workflow).toContain("PAWWORK_PERF_PROFILE: low-end") + expect(workflow).toContain("perf-base-combined.json") + expect(workflow).toContain("perf-head-combined.json") + expect(workflow).toContain("perf-comment.md") + expect(workflow.match(/perf-comment\.md/g)?.length).toBe(3) + }) +}) From 6cff97396d5d899d4d01bb6996a7ceb9cc31691c Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 14 May 2026 01:18:38 +0800 Subject: [PATCH 2/3] test(app): address low-end perf review comments --- packages/app/e2e/perf/perf-probe.spec.ts | 28 ++++++++++++------------ packages/app/e2e/perf/probe.ts | 8 ++++++- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/app/e2e/perf/perf-probe.spec.ts b/packages/app/e2e/perf/perf-probe.spec.ts index 0e037da33..027a41d92 100644 --- a/packages/app/e2e/perf/perf-probe.spec.ts +++ b/packages/app/e2e/perf/perf-probe.spec.ts @@ -17,7 +17,7 @@ import { seedTimelineRecomputeSession } from "./timeline-fixture" const outputPath = process.env.PAWWORK_PERF_OUTPUT ?? path.join(process.cwd(), "e2e", "perf-results", "pr0.1-baseline.json") const perfBranch = process.env.PAWWORK_PERF_BRANCH ?? "dev" -const perfProfile = readPerfProfile() +const PERF_PROFILE = readPerfProfile() const longMarkdown = [ "# Baseline stream", @@ -150,7 +150,7 @@ async function scrollTimelineTo(page: Parameters[0], t } function skipUnlessScenario(name: PerfScenarioName) { - test.skip(!shouldRunScenario(perfProfile, name), `${perfProfile} profile does not run ${name}`) + test.skip(!shouldRunScenario(PERF_PROFILE, name), `${PERF_PROFILE} profile does not run ${name}`) } test.describe("PR0.1 perf probe baseline", () => { @@ -164,7 +164,7 @@ test.describe("PR0.1 perf probe baseline", () => { test("homepage-cold emits a 3-run JSON baseline", async ({ page, project }) => { skipUnlessScenario("homepage-cold") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -180,13 +180,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "homepage-cold", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "homepage-cold", runs })) }) test("session-streaming-long emits a 3-run JSON baseline", async ({ page, project, llm }) => { skipUnlessScenario("session-streaming-long") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -227,13 +227,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-streaming-long", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "session-streaming-long", runs })) }) test("tool-call-expand emits a 3-run JSON baseline", async ({ page, project, llm }) => { skipUnlessScenario("tool-call-expand") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -259,13 +259,13 @@ test.describe("PR0.1 perf probe baseline", () => { if (run < 2) await cooldownAfterRun(page) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "tool-call-expand", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "tool-call-expand", runs })) }) test("terminal-side-panel-open emits a 3-run JSON baseline", async ({ page, project }) => { skipUnlessScenario("terminal-side-panel-open") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -288,13 +288,13 @@ test.describe("PR0.1 perf probe baseline", () => { }) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "terminal-side-panel-open", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "terminal-side-panel-open", runs })) }) test("session-scroll-reading emits a 3-run JSON baseline", async ({ page, project }) => { skipUnlessScenario("session-scroll-reading") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -329,13 +329,13 @@ test.describe("PR0.1 perf probe baseline", () => { }) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-scroll-reading", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "session-scroll-reading", runs })) }) test("session-timeline-recompute emits a 3-run low-end JSON baseline", async ({ page, project }) => { skipUnlessScenario("session-timeline-recompute") await installPerfProbe(page) - await applyPerfProfile(page, perfProfile) + await applyPerfProfile(page, PERF_PROFILE) await project.open() const runs = [] @@ -358,6 +358,6 @@ test.describe("PR0.1 perf probe baseline", () => { }) } - scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: perfProfile, scenario: "session-timeline-recompute", runs })) + scenarioResults.push(summarizeScenarioRuns({ branch: perfBranch, profile: PERF_PROFILE, scenario: "session-timeline-recompute", runs })) }) }) diff --git a/packages/app/e2e/perf/probe.ts b/packages/app/e2e/perf/probe.ts index a01cb9979..3a69ed329 100644 --- a/packages/app/e2e/perf/probe.ts +++ b/packages/app/e2e/perf/probe.ts @@ -1,5 +1,6 @@ import type { Page } from "@playwright/test" import { aggregatePerfRuns, summarizePerfRun, type PerfProfile, type PerfRunSummary } from "../../src/testing/perf-metrics" +import type { PerfScenarioName } from "./profiles" type BrowserPerfSample = { startedAt: number @@ -158,6 +159,11 @@ export async function snapshotPerfProbe(page: Page) { return summarizePerfRun(sample) } -export function summarizeScenarioRuns(input: { branch: string; profile?: PerfProfile; scenario: string; runs: PerfRunSummary[] }) { +export function summarizeScenarioRuns(input: { + branch: string + profile?: PerfProfile + scenario: PerfScenarioName + runs: PerfRunSummary[] +}) { return aggregatePerfRuns(input) } From 10993684f871c4865f851470ebb81070fe45e71d Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Thu, 14 May 2026 01:24:07 +0800 Subject: [PATCH 3/3] ci(app): restore Playwright browser cache fallback --- .github/workflows/perf-probe-baseline.yml | 2 ++ packages/app/src/testing/perf-workflow.test.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 5f1c79285..dc497f36a 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -62,6 +62,8 @@ jobs: with: path: ${{ github.workspace }}/.playwright-browsers key: playwright-${{ runner.os }}-${{ hashFiles('head/packages/app/package.json', 'head/bun.lock', 'base/packages/app/package.json', 'base/bun.lock') }} + restore-keys: | + playwright-${{ runner.os }}- - name: Install head dependencies working-directory: head diff --git a/packages/app/src/testing/perf-workflow.test.ts b/packages/app/src/testing/perf-workflow.test.ts index 6cd5ee314..359024c15 100644 --- a/packages/app/src/testing/perf-workflow.test.ts +++ b/packages/app/src/testing/perf-workflow.test.ts @@ -8,6 +8,7 @@ describe("perf workflow contract", () => { expect(workflow).toContain("fetch-depth: 0") expect(workflow).toContain("Detect low-end perf scope") expect(workflow).toContain("PAWWORK_PERF_PROFILE: low-end") + expect(workflow).toContain("restore-keys: |\n playwright-${{ runner.os }}-") expect(workflow).toContain("perf-base-combined.json") expect(workflow).toContain("perf-head-combined.json") expect(workflow).toContain("perf-comment.md")