ci(perf): upload trace-only artifact on perf-probe failure

.github/workflows/perf-probe-baseline.yml

@@ -11,6 +11,7 @@ on:
       - "packages/app/package.json"
       - "packages/app/playwright.config.ts"
       - "packages/app/script/compare-perf.ts"
+      - "packages/app/script/list-failing-scenarios.ts"
       - "packages/app/script/merge-perf-artifacts.ts"
       - "packages/app/script/e2e-local.ts"
       - "packages/app/src/testing/perf-metrics*"
@@ -95,7 +96,7 @@ jobs:
 
           is_low_end_path() {
             case "$1" in
-              packages/app/src/pages/session/*|packages/app/src/pages/session/**|packages/app/src/pages/session.tsx|packages/ui/src/components/message-part.tsx|packages/ui/src/components/session-turn.tsx|packages/ui/src/components/markdown.tsx|packages/ui/src/components/text-shimmer.css|packages/ui/src/components/text-shimmer.tsx|packages/ui/src/components/basic-tool.tsx|packages/ui/src/components/tool-status-title.tsx|packages/ui/src/styles/animations.css|packages/app/e2e/perf/*|packages/app/e2e/perf/**|packages/app/src/testing/perf-metrics*|packages/app/script/compare-perf.ts|packages/app/script/merge-perf-artifacts.ts|.github/workflows/perf-probe-baseline.yml)
+              packages/app/src/pages/session/*|packages/app/src/pages/session/**|packages/app/src/pages/session.tsx|packages/ui/src/components/message-part.tsx|packages/ui/src/components/session-turn.tsx|packages/ui/src/components/markdown.tsx|packages/ui/src/components/text-shimmer.css|packages/ui/src/components/text-shimmer.tsx|packages/ui/src/components/basic-tool.tsx|packages/ui/src/components/tool-status-title.tsx|packages/ui/src/styles/animations.css|packages/app/e2e/perf/*|packages/app/e2e/perf/**|packages/app/src/testing/perf-metrics*|packages/app/script/compare-perf.ts|packages/app/script/list-failing-scenarios.ts|packages/app/script/merge-perf-artifacts.ts|.github/workflows/perf-probe-baseline.yml)
                 return 0
                 ;;
               *)
@@ -291,44 +292,80 @@ jobs:
             })
             core.info("Created perf delta comment")
 
-      - name: Capture perf diagnostic trace (base)
+      - name: Resolve failing scenarios for trace capture
+        id: failing_scenarios
         if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure'
+        run: bun head/packages/app/script/list-failing-scenarios.ts "${PERF_ARTIFACT_DIR}/perf-compare-confirm.json"
+
+      - name: Capture perf diagnostic trace (base)
+        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.failing_scenarios.outputs.default != ''
         env:
           CI: "true"
           PAWWORK_PERF_BRANCH: base
+          PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.default }}
           PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-trace.json
           PAWWORK_PERF_TRACE: "1"
         run: bun --cwd base/packages/app test:e2e:local:perf
 
       - name: Capture perf diagnostic trace (head)
-        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure'
+        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.failing_scenarios.outputs.default != ''
         env:
           CI: "true"
           PAWWORK_PERF_BRANCH: head
+          PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.default }}
           PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-trace.json
           PAWWORK_PERF_TRACE: "1"
         run: bun --cwd head/packages/app test:e2e:local:perf
 
       - name: Capture low-end perf diagnostic trace (base)
-        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true'
+        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' && steps.failing_scenarios.outputs.low_end != ''
         env:
           CI: "true"
           PAWWORK_PERF_BRANCH: base
           PAWWORK_PERF_PROFILE: low-end
+          PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.low_end }}
           PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-base-low-end-trace.json
           PAWWORK_PERF_TRACE: "1"
         run: bun --cwd base/packages/app test:e2e:local:perf
 
       - name: Capture low-end perf diagnostic trace (head)
-        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true'
+        if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && steps.low_end_scope.outputs.run_low_end == 'true' && steps.failing_scenarios.outputs.low_end != ''
         env:
           CI: "true"
           PAWWORK_PERF_BRANCH: head
           PAWWORK_PERF_PROFILE: low-end
+          PAWWORK_PERF_SCENARIOS: ${{ steps.failing_scenarios.outputs.low_end }}
           PAWWORK_PERF_OUTPUT: ${{ github.workspace }}/perf-artifacts/perf-head-low-end-trace.json
           PAWWORK_PERF_TRACE: "1"
         run: bun --cwd head/packages/app test:e2e:local:perf
 
+      - name: Upload perf trace artifact
+        if: always() && steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && (steps.failing_scenarios.outputs.default != '' || steps.failing_scenarios.outputs.low_end != '')
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # actions/upload-artifact@v7
+        with:
+          name: perf-probe-trace-${{ github.run_attempt }}
+          if-no-files-found: warn
+          retention-days: 7
+          path: |
+            head/packages/app/e2e/test-results/**/trace.zip
+            base/packages/app/e2e/test-results/**/trace.zip
+
+      - name: Print perf trace artifact pointer
+        if: always() && steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure' && (steps.failing_scenarios.outputs.default != '' || steps.failing_scenarios.outputs.low_end != '')
+        run: |
+          cat >> "$GITHUB_STEP_SUMMARY" <<'EOF'
+          ## Perf trace artifact
+
+          The comparator failed twice. Playwright traces from the diagnostic re-run are uploaded as a separate small artifact so you do not have to download the full perf bundle.
+
+          Download **perf-probe-trace-${{ github.run_attempt }}** from the "Artifacts" section of this run summary, then open a trace with:
+
+          ```
+          npx playwright show-trace path/to/trace.zip
+          ```
+          EOF
+          echo "::notice::Perf comparator failed. Download artifact 'perf-probe-trace-${{ github.run_attempt }}' from this run and open trace.zip with 'npx playwright show-trace'."
+
       - name: Fail job on comparator regression
         if: steps.compare.outcome == 'failure' && steps.compare_confirmed.outcome == 'failure'
         run: exit 1

packages/app/e2e/perf/profiles.ts

@@ -33,8 +33,23 @@ export function readPerfProfile(): PerfProfile {
   return process.env.PAWWORK_PERF_PROFILE === "low-end" ? "low-end" : "default"
 }
 
+function readScenarioFilter(): ReadonlySet<PerfScenarioName> | null {
+  const raw = process.env.PAWWORK_PERF_SCENARIOS
+  if (!raw) return null
+  const items = raw
+    .split(",")
+    .map((item) => item.trim())
+    .filter((item) => item.length > 0) as PerfScenarioName[]
+  if (items.length === 0) return null
+  return new Set(items)
+}
+
 export function shouldRunScenario(profile: PerfProfile, scenario: PerfScenarioName) {
-  return profile === "low-end" ? lowEndScenarios.has(scenario) : defaultScenarios.has(scenario)
+  const inProfile = profile === "low-end" ? lowEndScenarios.has(scenario) : defaultScenarios.has(scenario)
+  if (!inProfile) return false
+  const filter = readScenarioFilter()
+  if (filter && !filter.has(scenario)) return false
+  return true
 }
 
 export async function applyPerfProfile(page: Page, profile: PerfProfile) {

packages/app/e2e/perf/profiles.unit.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "bun:test"
+import { afterEach, beforeEach, describe, expect, test } from "bun:test"
 import { shouldRunScenario, type PerfScenarioName } from "./profiles"
 
 test("default profile runs heavy default-open bash perf coverage", () => {
@@ -28,3 +28,53 @@ test("low-end profile gates concurrent-shimmer-extreme guard", () => {
   expect(shouldRunScenario("default", scenario)).toBe(false)
   expect(shouldRunScenario("low-end", scenario)).toBe(true)
 })
+
+describe("PAWWORK_PERF_SCENARIOS env filter", () => {
+  const previous = process.env.PAWWORK_PERF_SCENARIOS
+
+  beforeEach(() => {
+    delete process.env.PAWWORK_PERF_SCENARIOS
+  })
+
+  afterEach(() => {
+    if (previous === undefined) {
+      delete process.env.PAWWORK_PERF_SCENARIOS
+    } else {
+      process.env.PAWWORK_PERF_SCENARIOS = previous
+    }
+  })
+
+  test("absent env keeps profile membership as the only gate", () => {
+    expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true)
+    expect(shouldRunScenario("low-end", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(true)
+  })
+
+  test("env filter restricts to the listed scenarios within a profile", () => {
+    process.env.PAWWORK_PERF_SCENARIOS = "homepage-cold,long-session-input-lag"
+
+    expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true)
+    expect(shouldRunScenario("default", "long-session-input-lag" as PerfScenarioName)).toBe(true)
+    expect(shouldRunScenario("default", "tool-call-expand" as PerfScenarioName)).toBe(false)
+  })
+
+  test("env filter does not lift profile membership", () => {
+    process.env.PAWWORK_PERF_SCENARIOS = "concurrent-shimmer-extreme"
+
+    expect(shouldRunScenario("default", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(false)
+    expect(shouldRunScenario("low-end", "concurrent-shimmer-extreme" as PerfScenarioName)).toBe(true)
+  })
+
+  test("empty env is treated as no filter", () => {
+    process.env.PAWWORK_PERF_SCENARIOS = ""
+
+    expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true)
+  })
+
+  test("whitespace and empty entries are tolerated", () => {
+    process.env.PAWWORK_PERF_SCENARIOS = " homepage-cold , , long-session-input-lag "
+
+    expect(shouldRunScenario("default", "homepage-cold" as PerfScenarioName)).toBe(true)
+    expect(shouldRunScenario("default", "long-session-input-lag" as PerfScenarioName)).toBe(true)
+    expect(shouldRunScenario("default", "tool-call-expand" as PerfScenarioName)).toBe(false)
+  })
+})

packages/app/script/list-failing-scenarios.ts

@@ -0,0 +1,36 @@
+import fs from "node:fs/promises"
+import type { PerfBaselineComparison } from "../src/testing/perf-metrics"
+
+async function main() {
+  const inputPath = process.argv[2]
+  if (!inputPath) {
+    throw new Error("Usage: bun script/list-failing-scenarios.ts <perf-compare.json>")
+  }
+
+  const failingByProfile = new Map<"default" | "low_end", string[]>([
+    ["default", []],
+    ["low_end", []],
+  ])
+
+  try {
+    const payload = JSON.parse(await fs.readFile(inputPath, "utf8")) as PerfBaselineComparison
+    for (const scenario of payload.scenarios) {
+      if (scenario.failures.length === 0) continue
+      const key = scenario.profile === "low-end" ? "low_end" : "default"
+      failingByProfile.get(key)!.push(scenario.scenario)
+    }
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error)
+    process.stderr.write(`::warning::Failed to read failing scenarios from ${inputPath}: ${message}\n`)
+  }
+
+  const output = Array.from(failingByProfile, ([key, names]) => `${key}=${names.join(",")}`).join("\n") + "\n"
+  const githubOutput = process.env.GITHUB_OUTPUT
+  if (githubOutput) {
+    await fs.appendFile(githubOutput, output)
+  } else {
+    process.stdout.write(output)
+  }
+}
+
+await main()