From d4b34c534015d9a0dff7092c8ea645eb15d7e00e Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Mon, 18 May 2026 15:03:33 +0800 Subject: [PATCH 1/4] ci(perf): upload trace-only artifact on perf-probe failure Add perf-probe-trace-{attempt} artifact containing only Playwright trace.zip files from the diagnostic re-run, plus a step summary and ::notice pointing PR authors at it. The existing perf-probe-baseline artifact is unchanged. The 90 MB perf-probe-baseline bundle is impractical to download from China (~40 KB/s sustained, both via mihomo and direct to Azure blob), so authors cannot inspect traces when a regression fires. A trace-only split brings the typical download to a few minutes and gives the on-failure pointer a single named target. Refs #698 --- .github/workflows/perf-probe-baseline.yml | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 19edd33f3..9547ab895 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -329,6 +329,33 @@ jobs: PAWWORK_PERF_TRACE: "1" run: bun --cwd head/packages/app test:e2e:local:perf + - name: Upload perf trace artifact + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # actions/upload-artifact@v7 + with: + name: perf-probe-trace-${{ github.run_attempt }} + if-no-files-found: warn + retention-days: 7 + path: | + head/packages/app/e2e/test-results/**/trace.zip + base/packages/app/e2e/test-results/**/trace.zip + + - name: Print perf trace artifact pointer + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + run: | + cat >> "$GITHUB_STEP_SUMMARY" <<'EOF' + ## Perf trace artifact + + The comparator failed twice. Playwright traces from the diagnostic re-run are uploaded as a separate small artifact so you do not have to download the full perf bundle. + + Download **perf-probe-trace-${{ github.run_attempt }}** from the "Artifacts" section of this run summary, then open a trace with: + + ``` + npx playwright show-trace path/to/trace.zip + ``` + EOF + echo "::notice::Perf comparator failed. Download artifact 'perf-probe-trace-${{ github.run_attempt }}' from this run and open trace.zip with 'npx playwright show-trace'." + - name: Fail job on comparator regression if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' run: exit 1 From cc9197312ef3689d5ee22906f963d012ac770287 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Mon, 18 May 2026 16:04:48 +0800 Subject: [PATCH 2/4] ci(perf): only capture traces for failing scenarios Add a PAWWORK_PERF_SCENARIOS env filter on top of the existing profile gate in shouldRunScenario, and resolve the failing scenario list from perf-compare-confirm.json before the diagnostic trace steps. Each of the four Capture trace steps now passes only its profile's failing scenarios to the spec, and skips entirely when that profile has no failure. Today the four trace steps re-run the full perf spec (all scenarios in the profile), even when a regression touched only one scenario. Measured on a forced-fail run, baseline + confirm + trace together cost ~34 min on the 30-min job ceiling, so any real perf regression risks being cut off before the trace artifact is produced. Filtering trace to the failing scenarios brings the worst-case fail path to ~22-26 min, leaving headroom for setup and future scenario growth. Confirm steps still run the whole profile to preserve the workflow's false-positive guard: a scenario must fail twice across two independent runs before its trace is recorded. Refs #698 --- .github/workflows/perf-probe-baseline.yml | 17 ++++-- packages/app/e2e/perf/profiles.ts | 17 +++++- packages/app/e2e/perf/profiles.unit.ts | 52 ++++++++++++++++++- packages/app/script/list-failing-scenarios.ts | 27 ++++++++++ 4 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 packages/app/script/list-failing-scenarios.ts diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 9547ab895..4a15ac9e6 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -291,40 +291,49 @@ jobs: }) core.info("Created perf delta comment") - - name: Capture perf diagnostic trace (base) + - name: Resolve failing scenarios for trace capture + id: failing_scenarios if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + run: bun head/packages/app/script/list-failing-scenarios.ts "${PERF_ARTIFACT_DIR}/perf-compare-confirm.json" >> "$GITHUB_OUTPUT" + + - name: Capture perf diagnostic trace (base) + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.failing_scenarios.outputs.default != '' env: CI: "true" PAWWORK_PERF_BRANCH: base + PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.default }} PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-trace.json PAWWORK_PERF_TRACE: "1" run: bun --cwd base/packages/app test:e2e:local:perf - name: Capture perf diagnostic trace (head) - if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.failing_scenarios.outputs.default != '' env: CI: "true" PAWWORK_PERF_BRANCH: head + PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.default }} PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-trace.json PAWWORK_PERF_TRACE: "1" run: bun --cwd head/packages/app test:e2e:local:perf - name: Capture low-end perf diagnostic trace (base) - if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' && steps.failing_scenarios.outputs.low_end != '' env: CI: "true" PAWWORK_PERF_BRANCH: base PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.low_end }} PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-low-end-trace.json PAWWORK_PERF_TRACE: "1" run: bun --cwd base/packages/app test:e2e:local:perf - name: Capture low-end perf diagnostic trace (head) - if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' + if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' && steps.failing_scenarios.outputs.low_end != '' env: CI: "true" PAWWORK_PERF_BRANCH: head PAWWORK_PERF_PROFILE: low-end + PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.low_end }} PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-low-end-trace.json PAWWORK_PERF_TRACE: "1" run: bun --cwd head/packages/app test:e2e:local:perf diff --git a/packages/app/e2e/perf/profiles.ts b/packages/app/e2e/perf/profiles.ts index b2cb850af..52094d47e 100644 --- a/packages/app/e2e/perf/profiles.ts +++ b/packages/app/e2e/perf/profiles.ts @@ -33,8 +33,23 @@ export function readPerfProfile(): PerfProfile { return process.env.PAWWORK_PERF_PROFILE === "low-end" ? "low-end" : "default" } +function readScenarioFilter(): ReadonlySet | null { + const raw = process.env.PAWWORK_PERF_SCENARIOS + if (!raw) return null + const items = raw + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0) as PerfScenarioName[] + if (items.length === 0) return null + return new Set(items) +} + export function shouldRunScenario(profile: PerfProfile, scenario: PerfScenarioName) { - return profile === "low-end" ? lowEndScenarios.has(scenario) : defaultScenarios.has(scenario) + const inProfile = profile === "low-end" ? lowEndScenarios.has(scenario) : defaultScenarios.has(scenario) + if (!inProfile) return false + const filter = readScenarioFilter() + if (filter && !filter.has(scenario)) return false + return true } export async function applyPerfProfile(page: Page, profile: PerfProfile) { diff --git a/packages/app/e2e/perf/profiles.unit.ts b/packages/app/e2e/perf/profiles.unit.ts index a17a3dec2..fd2bb5cc8 100644 --- a/packages/app/e2e/perf/profiles.unit.ts +++ b/packages/app/e2e/perf/profiles.unit.ts @@ -1,4 +1,4 @@ -import { expect, test } from "bun:test" +import { afterEach, beforeEach, describe, expect, test } from "bun:test" import { shouldRunScenario, type PerfScenarioName } from "./profiles" test("default profile runs heavy default-open bash perf coverage", () => { @@ -28,3 +28,53 @@ test("low-end profile gates concurrent-shimmer-extreme guard", () => { expect(shouldRunScenario("default", scenario)).toBe(false) expect(shouldRunScenario("low-end", scenario)).toBe(true) }) + +describe("PAWWORK_PERF_SCENARIOS env filter", () => { + const previous = process.env.PAWWORK_PERF_SCENARIOS + + beforeEach(() => { + delete process.env.PAWWORK_PERF_SCENARIOS + }) + + afterEach(() => { + if (previous === undefined) { + delete process.env.PAWWORK_PERF_SCENARIOS + } else { + process.env.PAWWORK_PERF_SCENARIOS = previous + } + }) + + test("absent env keeps profile membership as the only gate", () => { + expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true) + expect(shouldRunScenario("low-end", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(true) + }) + + test("env filter restricts to the listed scenarios within a profile", () => { + process.env.PAWWORK_PERF_SCENARIOS = "homepage-cold,long-session-input-lag" + + expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true) + expect(shouldRunScenario("default", "long-session-input-lag" as PerfScenarioName)).toBe(true) + expect(shouldRunScenario("default", "tool-call-expand" as PerfScenarioName)).toBe(false) + }) + + test("env filter does not lift profile membership", () => { + process.env.PAWWORK_PERF_SCENARIOS = "concurrent-shimmer-extreme" + + expect(shouldRunScenario("default", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(false) + expect(shouldRunScenario("low-end", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(true) + }) + + test("empty env is treated as no filter", () => { + process.env.PAWWORK_PERF_SCENARIOS = "" + + expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true) + }) + + test("whitespace and empty entries are tolerated", () => { + process.env.PAWWORK_PERF_SCENARIOS = " homepage-cold , , long-session-input-lag " + + expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true) + expect(shouldRunScenario("default", "long-session-input-lag" as PerfScenarioName)).toBe(true) + expect(shouldRunScenario("default", "tool-call-expand" as PerfScenarioName)).toBe(false) + }) +}) diff --git a/packages/app/script/list-failing-scenarios.ts b/packages/app/script/list-failing-scenarios.ts new file mode 100644 index 000000000..1e74de106 --- /dev/null +++ b/packages/app/script/list-failing-scenarios.ts @@ -0,0 +1,27 @@ +import fs from "node:fs/promises" +import type { PerfBaselineComparison } from "../src/testing/perf-metrics" + +async function main() { + const inputPath = process.argv[2] + if (!inputPath) { + throw new Error("Usage: bun script/list-failing-scenarios.ts ") + } + + const payload = JSON.parse(await fs.readFile(inputPath, "utf8")) as PerfBaselineComparison + const failingByProfile = new Map<"default" | "low_end", string[]>([ + ["default", []], + ["low_end", []], + ]) + + for (const scenario of payload.scenarios) { + if (scenario.failures.length === 0) continue + const key = scenario.profile === "low-end" ? "low_end" : "default" + failingByProfile.get(key)!.push(scenario.scenario) + } + + for (const [key, names] of failingByProfile) { + process.stdout.write(`${key}=${names.join(",")}\n`) + } +} + +await main() From 14910d6e5647c8d7cf8306d178354fb2c35321ef Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Mon, 18 May 2026 16:42:44 +0800 Subject: [PATCH 3/4] ci(perf): harden trace artifact gating and script error paths Address crosscheck findings on the perf trace artifact pipeline: - Gate Upload perf trace artifact and Print perf trace artifact pointer on steps.failing_scenarios.outputs being non-empty, so a top-level missing_* comparator failure (or any other path where no per-scenario regression is detected) no longer prints a pointer to an empty artifact. - Wrap both gates with always() so a trace capture step that exits non-zero partway still uploads whatever trace.zip files were produced before the failure, instead of being skipped by the default cancel-on-prior-failure behavior. - Have list-failing-scenarios.ts try/catch the JSON parse and write directly to process.env.GITHUB_OUTPUT instead of relying on shell stdout redirection, so stray script output or a malformed input cannot silently corrupt the step outputs that downstream conditions depend on. A ::warning:: surfaces the parse failure for the run author. - Add packages/app/script/list-failing-scenarios.ts to the workflow paths filter so future edits to the selector script actually trigger this workflow. Refs #698 --- .github/workflows/perf-probe-baseline.yml | 7 +++--- packages/app/script/list-failing-scenarios.ts | 23 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 4a15ac9e6..9c516ed3e 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -11,6 +11,7 @@ on: - "packages/app/package.json" - "packages/app/playwright.config.ts" - "packages/app/script/compare-perf.ts" + - "packages/app/script/list-failing-scenarios.ts" - "packages/app/script/merge-perf-artifacts.ts" - "packages/app/script/e2e-local.ts" - "packages/app/src/testing/perf-metrics*" @@ -294,7 +295,7 @@ jobs: - name: Resolve failing scenarios for trace capture id: failing_scenarios if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' - run: bun head/packages/app/script/list-failing-scenarios.ts "${PERF_ARTIFACT_DIR}/perf-compare-confirm.json" >> "$GITHUB_OUTPUT" + run: bun head/packages/app/script/list-failing-scenarios.ts "${PERF_ARTIFACT_DIR}/perf-compare-confirm.json" - name: Capture perf diagnostic trace (base) if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.failing_scenarios.outputs.default != '' @@ -339,7 +340,7 @@ jobs: run: bun --cwd head/packages/app test:e2e:local:perf - name: Upload perf trace artifact - if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + if: always() && steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && (steps.failing_scenarios.outputs.default != '' || steps.failing_scenarios.outputs.low_end != '') uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # actions/upload-artifact@v7 with: name: perf-probe-trace-${{ github.run_attempt }} @@ -350,7 +351,7 @@ jobs: base/packages/app/e2e/test-results/**/trace.zip - name: Print perf trace artifact pointer - if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' + if: always() && steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && (steps.failing_scenarios.outputs.default != '' || steps.failing_scenarios.outputs.low_end != '') run: | cat >> "$GITHUB_STEP_SUMMARY" <<'EOF' ## Perf trace artifact diff --git a/packages/app/script/list-failing-scenarios.ts b/packages/app/script/list-failing-scenarios.ts index 1e74de106..00491dd3f 100644 --- a/packages/app/script/list-failing-scenarios.ts +++ b/packages/app/script/list-failing-scenarios.ts @@ -7,20 +7,29 @@ async function main() { throw new Error("Usage: bun script/list-failing-scenarios.ts ") } - const payload = JSON.parse(await fs.readFile(inputPath, "utf8")) as PerfBaselineComparison const failingByProfile = new Map<"default" | "low_end", string[]>([ ["default", []], ["low_end", []], ]) - for (const scenario of payload.scenarios) { - if (scenario.failures.length === 0) continue - const key = scenario.profile === "low-end" ? "low_end" : "default" - failingByProfile.get(key)!.push(scenario.scenario) + try { + const payload = JSON.parse(await fs.readFile(inputPath, "utf8")) as PerfBaselineComparison + for (const scenario of payload.scenarios) { + if (scenario.failures.length === 0) continue + const key = scenario.profile === "low-end" ? "low_end" : "default" + failingByProfile.get(key)!.push(scenario.scenario) + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + process.stderr.write(`::warning::Failed to read failing scenarios from ${inputPath}: ${message}\n`) } - for (const [key, names] of failingByProfile) { - process.stdout.write(`${key}=${names.join(",")}\n`) + const output = Array.from(failingByProfile, ([key, names]) => `${key}=${names.join(",")}`).join("\n") + "\n" + const githubOutput = process.env.GITHUB_OUTPUT + if (githubOutput) { + await fs.appendFile(githubOutput, output) + } else { + process.stdout.write(output) } } From 05655029bfba30cc21e5b1aa9a2c92e77a35b97f Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Mon, 18 May 2026 17:01:48 +0800 Subject: [PATCH 4/4] ci(perf): trigger low-end scope when only list-failing-scenarios.ts changes Add packages/app/script/list-failing-scenarios.ts to is_low_end_path() so a PR that touches only this selector script still exercises the low_end output contract that the script writes, mirroring how compare-perf.ts and merge-perf-artifacts.ts already gate the low-end profile. Refs #698 --- .github/workflows/perf-probe-baseline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perf-probe-baseline.yml b/.github/workflows/perf-probe-baseline.yml index 9c516ed3e..d2f60abe9 100644 --- a/.github/workflows/perf-probe-baseline.yml +++ b/.github/workflows/perf-probe-baseline.yml @@ -96,7 +96,7 @@ jobs: is_low_end_path() { case "$1" in - packages/app/src/pages/session/*|packages/app/src/pages/session/**|packages/app/src/pages/session.tsx|packages/ui/src/components/message-part.tsx|packages/ui/src/components/session-turn.tsx|packages/ui/src/components/markdown.tsx|packages/ui/src/components/text-shimmer.css|packages/ui/src/components/text-shimmer.tsx|packages/ui/src/components/basic-tool.tsx|packages/ui/src/components/tool-status-title.tsx|packages/ui/src/styles/animations.css|packages/app/e2e/perf/*|packages/app/e2e/perf/**|packages/app/src/testing/perf-metrics*|packages/app/script/compare-perf.ts|packages/app/script/merge-perf-artifacts.ts|.github/workflows/perf-probe-baseline.yml) + packages/app/src/pages/session/*|packages/app/src/pages/session/**|packages/app/src/pages/session.tsx|packages/ui/src/components/message-part.tsx|packages/ui/src/components/session-turn.tsx|packages/ui/src/components/markdown.tsx|packages/ui/src/components/text-shimmer.css|packages/ui/src/components/text-shimmer.tsx|packages/ui/src/components/basic-tool.tsx|packages/ui/src/components/tool-status-title.tsx|packages/ui/src/styles/animations.css|packages/app/e2e/perf/*|packages/app/e2e/perf/**|packages/app/src/testing/perf-metrics*|packages/app/script/compare-perf.ts|packages/app/script/list-failing-scenarios.ts|packages/app/script/merge-perf-artifacts.ts|.github/workflows/perf-probe-baseline.yml) return 0 ;; *)