From a19d92267d9021f6229b1e6992576638e63f298d Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Sat, 23 May 2026 16:11:14 +0800 Subject: [PATCH 1/5] fix: raise perf interaction median floor --- packages/app/src/testing/perf-metrics.test.ts | 39 +++++++++++++++++-- packages/app/src/testing/perf-metrics.ts | 2 +- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/packages/app/src/testing/perf-metrics.test.ts b/packages/app/src/testing/perf-metrics.test.ts index 4d968b0f2..6085b1872 100644 --- a/packages/app/src/testing/perf-metrics.test.ts +++ b/packages/app/src/testing/perf-metrics.test.ts @@ -160,6 +160,39 @@ describe("perf metrics", () => { expect(summary.run_details).toHaveLength(3) }) + test("does not fail default interaction median on a single-frame delta", () => { + const result = comparePerfScenarioSummaries({ + scenario: "long-session-input-lag", + base: scenario({ branch: "base", scenario: "long-session-input-lag", interaction: 48 }), + head: scenario({ branch: "head", scenario: "long-session-input-lag", interaction: 64 }), + }) + + expect(result.pass).toBe(true) + expect(result.failures).not.toContain("interaction_ms_median") + }) + + test("keeps default interaction median at the 20ms floor passing", () => { + const result = comparePerfScenarioSummaries({ + scenario: "long-session-input-lag", + base: scenario({ branch: "base", scenario: "long-session-input-lag", interaction: 100 }), + head: scenario({ branch: "head", scenario: "long-session-input-lag", interaction: 120 }), + }) + + expect(result.pass).toBe(true) + expect(result.failures).not.toContain("interaction_ms_median") + }) + + test("fails default interaction median above the 20ms floor", () => { + const result = comparePerfScenarioSummaries({ + scenario: "long-session-input-lag", + base: scenario({ branch: "base", scenario: "long-session-input-lag", interaction: 100 }), + head: scenario({ branch: "head", scenario: "long-session-input-lag", interaction: 121 }), + }) + + expect(result.pass).toBe(false) + expect(result.failures).toContain("interaction_ms_median") + }) + test("fails a scenario when median regression breaks both the ms and percentage budgets", () => { const result = comparePerfScenarioSummaries({ scenario: "session-streaming-long", @@ -187,9 +220,9 @@ describe("perf metrics", () => { profile: "default", scenario: "session-streaming-long", runs: 3, - interaction_ms_median: 116, + interaction_ms_median: 125, interaction_ms_worst: 168, - interaction_ms: 116, + interaction_ms: 125, interaction_delay_ms: 14, long_task_count: 1, long_task_max_ms: 88, @@ -501,7 +534,7 @@ describe("perf metrics", () => { scenario: "session-scroll-reading", runs: [ { - interaction_ms: 32, + interaction_ms: 40, interaction_delay_ms: 1, long_task_count: 0, long_task_max_ms: 0, diff --git a/packages/app/src/testing/perf-metrics.ts b/packages/app/src/testing/perf-metrics.ts index ca83c09a6..635fdee8f 100644 --- a/packages/app/src/testing/perf-metrics.ts +++ b/packages/app/src/testing/perf-metrics.ts @@ -74,7 +74,7 @@ export type PerfBaselineComparison = { export const PERF_COMMENT_MARKER = "" const perfDeltaThresholds = { - interactionMedianMs: 10, + interactionMedianMs: 20, interactionMedianRatio: 1.05, interactionWorstMs: 50, longTaskMaxMs: 25, From d382488cfc84cde338b8eaf8354d703ecef17da7 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Sat, 23 May 2026 16:13:05 +0800 Subject: [PATCH 2/5] fix: confirm perf failures by metric key --- packages/app/src/testing/perf-metrics.test.ts | 67 ++++++++++++++ packages/app/src/testing/perf-metrics.ts | 89 ++++++++++++++++--- 2 files changed, 143 insertions(+), 13 deletions(-) diff --git a/packages/app/src/testing/perf-metrics.test.ts b/packages/app/src/testing/perf-metrics.test.ts index 6085b1872..586918517 100644 --- a/packages/app/src/testing/perf-metrics.test.ts +++ b/packages/app/src/testing/perf-metrics.test.ts @@ -434,6 +434,73 @@ describe("perf metrics", () => { ]) }) + test("does not confirm a different metric failure from the same scenario", () => { + const result = comparePerfBaselines({ + base: [scenario({ branch: "base", scenario: "session-scroll-reading", interaction: 32, frameMax: 16 })], + head: [scenario({ branch: "head", scenario: "session-scroll-reading", interaction: 40, frameMax: 120 })], + scenarioKeys: ["default:session-scroll-reading"], + failureKeys: ["default:session-scroll-reading:interaction_ms_median"], + }) + + expect(result.pass).toBe(true) + expect(result.failures).toHaveLength(0) + expect(result.scenarios[0].failures).toHaveLength(0) + expect(result.confirmation).toEqual({ + initialFailureKeys: ["default:session-scroll-reading:interaction_ms_median"], + rawConfirmedFailures: ["default:session-scroll-reading:frame_gap_max_ms_delta"], + intersectedFailures: [], + }) + }) + + test("confirms the same metric failure from the same scenario", () => { + const result = comparePerfBaselines({ + base: [scenario({ branch: "base", scenario: "session-scroll-reading", interaction: 48 })], + head: [scenario({ branch: "head", scenario: "session-scroll-reading", interaction: 76 })], + scenarioKeys: ["default:session-scroll-reading"], + failureKeys: ["default:session-scroll-reading:interaction_ms_median"], + }) + + expect(result.pass).toBe(false) + expect(result.failures).toEqual(["default:session-scroll-reading:interaction_ms_median"]) + expect(result.scenarios[0].failures).toEqual(["interaction_ms_median"]) + expect(result.confirmation).toEqual({ + initialFailureKeys: ["default:session-scroll-reading:interaction_ms_median"], + rawConfirmedFailures: ["default:session-scroll-reading:interaction_ms_median"], + intersectedFailures: ["default:session-scroll-reading:interaction_ms_median"], + }) + }) + + test("keeps missing requested confirmation scenarios as hard failures", () => { + const missingHead = comparePerfBaselines({ + base: [scenario({ branch: "base", scenario: "session-scroll-reading" })], + head: [], + scenarioKeys: ["default:session-scroll-reading"], + failureKeys: ["default:session-scroll-reading:interaction_ms_median"], + }) + const missingBase = comparePerfBaselines({ + base: [], + head: [scenario({ branch: "head", scenario: "session-scroll-reading" })], + scenarioKeys: ["default:session-scroll-reading"], + failureKeys: ["default:session-scroll-reading:interaction_ms_median"], + }) + const missingBoth = comparePerfBaselines({ + base: [], + head: [], + scenarioKeys: ["default:session-scroll-reading"], + failureKeys: ["default:session-scroll-reading:interaction_ms_median"], + }) + + expect(missingHead.pass).toBe(false) + expect(missingHead.failures).toEqual(["missing_head_scenario:default:session-scroll-reading"]) + expect(missingBase.pass).toBe(false) + expect(missingBase.failures).toEqual(["missing_base_scenario:default:session-scroll-reading"]) + expect(missingBoth.pass).toBe(false) + expect(missingBoth.failures).toEqual([ + "missing_base_scenario:default:session-scroll-reading", + "missing_head_scenario:default:session-scroll-reading", + ]) + }) + test("keeps low-end moderate regressions warning-only", () => { const result = comparePerfScenarioSummaries({ scenario: "session-timeline-recompute", diff --git a/packages/app/src/testing/perf-metrics.ts b/packages/app/src/testing/perf-metrics.ts index 635fdee8f..598b1c5ef 100644 --- a/packages/app/src/testing/perf-metrics.ts +++ b/packages/app/src/testing/perf-metrics.ts @@ -64,11 +64,20 @@ export type PerfScenarioComparison = { head: PerfScenarioSummary } +export type PerfFailureKey = string + +export type PerfBaselineConfirmation = { + initialFailureKeys: PerfFailureKey[] + rawConfirmedFailures: string[] + intersectedFailures: string[] +} + export type PerfBaselineComparison = { pass: boolean failures: string[] warnings: string[] scenarios: PerfScenarioComparison[] + confirmation?: PerfBaselineConfirmation } export const PERF_COMMENT_MARKER = "" @@ -420,40 +429,93 @@ function scenarioKey(input: { profile?: PerfProfile; scenario: string }) { return `${input.profile ?? "default"}:${input.scenario}` } +export function perfFailureKey(input: { profile?: PerfProfile; scenario: string; metric: string }): PerfFailureKey { + return `${scenarioKey(input)}:${input.metric}` +} + +function pushUnique(list: string[], value: string) { + if (!list.includes(value)) list.push(value) +} + export function comparePerfBaselines(input: { base: PerfScenarioSummary[] head: PerfScenarioSummary[] scenarioKeys?: string[] + failureKeys?: PerfFailureKey[] }): PerfBaselineComparison { const failures: string[] = [] const warnings: string[] = [] const scenarios: PerfScenarioComparison[] = [] + const baseByScenario = new Map(input.base.map((scenario) => [scenarioKey(scenario), scenario])) const headByScenario = new Map(input.head.map((scenario) => [scenarioKey(scenario), scenario])) - const requestedScenarioKeys = input.scenarioKeys ? new Set(input.scenarioKeys) : undefined + const requestedScenarioKeys = input.scenarioKeys ? [...new Set(input.scenarioKeys)] : undefined + const requestedFailureKeys = input.failureKeys ? new Set(input.failureKeys) : undefined + const confirmation: PerfBaselineConfirmation | undefined = input.failureKeys + ? { + initialFailureKeys: [...new Set(input.failureKeys)], + rawConfirmedFailures: [], + intersectedFailures: [], + } + : undefined + + function addHardFailure(failure: string) { + pushUnique(failures, failure) + if (!confirmation) return + pushUnique(confirmation.rawConfirmedFailures, failure) + pushUnique(confirmation.intersectedFailures, failure) + } - for (const baseScenario of input.base) { - const key = scenarioKey(baseScenario) - if (requestedScenarioKeys && !requestedScenarioKeys.has(key)) continue + const comparisonKeys = requestedScenarioKeys ?? input.base.map((scenario) => scenarioKey(scenario)) + + for (const key of comparisonKeys) { + const baseScenario = baseByScenario.get(key) const headScenario = headByScenario.get(key) + if (!baseScenario) { + addHardFailure(`missing_base_scenario:${key}`) + } if (!headScenario) { - failures.push(`missing_head_scenario:${key}`) + addHardFailure(`missing_head_scenario:${key}`) + } + if (!baseScenario || !headScenario) { continue } - const comparison = comparePerfScenarioSummaries({ + + const rawComparison = comparePerfScenarioSummaries({ scenario: baseScenario.scenario, base: baseScenario, head: headScenario, }) + const filteredFailures: string[] = [] + for (const failure of rawComparison.failures) { + const fullFailureKey = perfFailureKey({ + profile: rawComparison.profile, + scenario: rawComparison.scenario, + metric: failure, + }) + if (confirmation) pushUnique(confirmation.rawConfirmedFailures, fullFailureKey) + if (requestedFailureKeys && !requestedFailureKeys.has(fullFailureKey)) continue + pushUnique(filteredFailures, failure) + pushUnique(failures, fullFailureKey) + if (confirmation) pushUnique(confirmation.intersectedFailures, fullFailureKey) + } + + const comparison = requestedFailureKeys + ? { + ...rawComparison, + pass: filteredFailures.length === 0, + failures: filteredFailures, + } + : rawComparison scenarios.push(comparison) - for (const failure of comparison.failures) failures.push(`${key}:${failure}`) - for (const warning of comparison.warnings) warnings.push(`${key}:${warning}`) + for (const warning of comparison.warnings) pushUnique(warnings, `${key}:${warning}`) } - for (const headScenario of input.head) { - const key = scenarioKey(headScenario) - if (requestedScenarioKeys && !requestedScenarioKeys.has(key)) continue - if (!input.base.some((scenario) => scenarioKey(scenario) === key)) { - failures.push(`missing_base_scenario:${key}`) + if (!requestedScenarioKeys) { + for (const headScenario of input.head) { + const key = scenarioKey(headScenario) + if (!baseByScenario.has(key)) { + addHardFailure(`missing_base_scenario:${key}`) + } } } @@ -462,5 +524,6 @@ export function comparePerfBaselines(input: { failures, warnings, scenarios, + confirmation, } } From 7d19f60231eed41718b8626d8e21f6294a16817b Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Sat, 23 May 2026 16:14:21 +0800 Subject: [PATCH 3/5] test: cover perf confirm script scope --- packages/app/script/compare-perf.ts | 34 +++-- .../src/testing/compare-perf-script.test.ts | 119 ++++++++++++++++++ 2 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 packages/app/src/testing/compare-perf-script.test.ts diff --git a/packages/app/script/compare-perf.ts b/packages/app/script/compare-perf.ts index 20c42d3c1..2ce6f1ce7 100644 --- a/packages/app/script/compare-perf.ts +++ b/packages/app/script/compare-perf.ts @@ -2,8 +2,10 @@ import fs from "node:fs/promises" import path from "node:path" import { comparePerfBaselines, + perfFailureKey, renderPerfBaselineComment, type PerfBaselineComparison, + type PerfFailureKey, type PerfScenarioSummary, } from "../src/testing/perf-metrics" @@ -23,14 +25,26 @@ async function readPerfFile(filePath: string) { return payload } -async function readFailureScenarioKeys(filePath: string) { +async function readFailureScope(filePath: string): Promise<{ scenarioKeys: string[]; failureKeys: PerfFailureKey[] }> { const payload = JSON.parse(await fs.readFile(filePath, "utf8")) as PerfBaselineComparison if (!Array.isArray(payload.scenarios)) { throw new Error(`Expected a perf comparison with scenarios in ${filePath}`) } - return payload.scenarios - .filter((scenario) => scenario.failures.length > 0) - .map((scenario) => `${scenario.profile}:${scenario.scenario}`) + const scenarioKeys = new Set() + const failureKeys = new Set() + + for (const scenario of payload.scenarios) { + if (scenario.failures.length === 0) continue + scenarioKeys.add(`${scenario.profile}:${scenario.scenario}`) + for (const failure of scenario.failures) { + failureKeys.add(perfFailureKey({ profile: scenario.profile, scenario: scenario.scenario, metric: failure })) + } + } + + return { + scenarioKeys: [...scenarioKeys], + failureKeys: [...failureKeys], + } } async function inferFailureScenarioSource(input: { outputPath?: string; failuresFromPath?: string }) { @@ -59,12 +73,17 @@ async function main() { } const failuresSourcePath = await inferFailureScenarioSource({ outputPath, failuresFromPath }) - const [base, head, scenarioKeys] = await Promise.all([ + const [base, head, failureScope] = await Promise.all([ readPerfFile(basePath), readPerfFile(headPath), - failuresSourcePath ? readFailureScenarioKeys(failuresSourcePath) : undefined, + failuresSourcePath ? readFailureScope(failuresSourcePath) : undefined, ]) - const comparison = comparePerfBaselines({ base, head, scenarioKeys }) + const comparison = comparePerfBaselines({ + base, + head, + scenarioKeys: failureScope?.scenarioKeys, + failureKeys: failureScope?.failureKeys, + }) if (outputPath) { await fs.mkdir(path.dirname(outputPath), { recursive: true }) @@ -79,6 +98,7 @@ async function main() { pass: comparison.pass, failures: comparison.failures, warnings: comparison.warnings, + confirmation: comparison.confirmation, } console.log(JSON.stringify(summary, null, 2)) diff --git a/packages/app/src/testing/compare-perf-script.test.ts b/packages/app/src/testing/compare-perf-script.test.ts new file mode 100644 index 000000000..2d5c35a34 --- /dev/null +++ b/packages/app/src/testing/compare-perf-script.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import type { PerfBaselineComparison, PerfProfile, PerfScenarioSummary } from "./perf-metrics" + +const tempRoots: string[] = [] + +afterEach(async () => { + await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +function scenario(input: { + branch: string + profile?: PerfProfile + scenario?: string + interaction?: number + frameMax?: number +}): PerfScenarioSummary { + const interaction = input.interaction ?? 40 + return { + branch: input.branch, + profile: input.profile ?? "default", + scenario: input.scenario ?? "session-scroll-reading", + runs: 3, + interaction_ms_median: interaction, + interaction_ms_worst: interaction, + interaction_ms: interaction, + interaction_delay_ms: 0, + long_task_count: 0, + long_task_max_ms: 0, + tbt_ms: 0, + frame_gap_p95_ms: 16, + frame_gap_max_ms: input.frameMax ?? 16, + jank_count_50ms: 0, + cls: 0, + window_ms: 900, + run_details: [], + } +} + +async function writeJson(filePath: string, payload: unknown) { + await writeFile(filePath, `${JSON.stringify(payload, null, 2)}\n`) +} + +describe("compare-perf script", () => { + test("confirms only the initial metric failure key from failures-from", async () => { + const root = await mkdtemp(join(tmpdir(), "compare-perf-script-")) + tempRoots.push(root) + const basePath = join(root, "base.json") + const headPath = join(root, "head.json") + const initialPath = join(root, "perf-compare.json") + const outputPath = join(root, "perf-compare-confirm.json") + const initialComparison: PerfBaselineComparison = { + pass: false, + failures: ["default:session-scroll-reading:interaction_ms_median"], + warnings: [], + scenarios: [ + { + profile: "default", + scenario: "session-scroll-reading", + pass: false, + failures: ["interaction_ms_median"], + warnings: [], + base: scenario({ branch: "base", interaction: 48 }), + head: scenario({ branch: "head", interaction: 76 }), + }, + ], + } + + await writeJson(basePath, [scenario({ branch: "base", interaction: 32, frameMax: 16 })]) + await writeJson(headPath, [scenario({ branch: "head", interaction: 40, frameMax: 120 })]) + await writeJson(initialPath, initialComparison) + + const child = Bun.spawn( + [ + process.execPath, + "script/compare-perf.ts", + "--base", + basePath, + "--head", + headPath, + "--output", + outputPath, + "--failures-from", + initialPath, + ], + { + cwd: process.cwd(), + stderr: "pipe", + stdout: "pipe", + }, + ) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + + expect(stderr).toBe("") + expect(exitCode).toBe(0) + const summary = JSON.parse(stdout) as Pick + expect(summary.pass).toBe(true) + expect(summary.failures).toHaveLength(0) + expect(summary.confirmation).toEqual({ + initialFailureKeys: ["default:session-scroll-reading:interaction_ms_median"], + rawConfirmedFailures: ["default:session-scroll-reading:frame_gap_max_ms_delta"], + intersectedFailures: [], + }) + const comparison = JSON.parse(await readFile(outputPath, "utf8")) as PerfBaselineComparison + expect(comparison.pass).toBe(true) + expect(comparison.failures).toHaveLength(0) + expect(comparison.confirmation).toEqual({ + initialFailureKeys: ["default:session-scroll-reading:interaction_ms_median"], + rawConfirmedFailures: ["default:session-scroll-reading:frame_gap_max_ms_delta"], + intersectedFailures: [], + }) + }) +}) From 3ba01b956ef492661667a1da78da882802cd7763 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Sat, 23 May 2026 17:23:02 +0800 Subject: [PATCH 4/5] fix: keep missing perf scenarios in confirm scope --- packages/app/script/compare-perf.ts | 11 ++++ .../src/testing/compare-perf-script.test.ts | 53 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/packages/app/script/compare-perf.ts b/packages/app/script/compare-perf.ts index 2ce6f1ce7..687edd6a1 100644 --- a/packages/app/script/compare-perf.ts +++ b/packages/app/script/compare-perf.ts @@ -30,9 +30,20 @@ async function readFailureScope(filePath: string): Promise<{ scenarioKeys: strin if (!Array.isArray(payload.scenarios)) { throw new Error(`Expected a perf comparison with scenarios in ${filePath}`) } + if (!Array.isArray(payload.failures)) { + throw new Error(`Expected a perf comparison with failures in ${filePath}`) + } const scenarioKeys = new Set() const failureKeys = new Set() + for (const failure of payload.failures) { + if (failure.startsWith("missing_base_scenario:")) { + scenarioKeys.add(failure.slice("missing_base_scenario:".length)) + } else if (failure.startsWith("missing_head_scenario:")) { + scenarioKeys.add(failure.slice("missing_head_scenario:".length)) + } + } + for (const scenario of payload.scenarios) { if (scenario.failures.length === 0) continue scenarioKeys.add(`${scenario.profile}:${scenario.scenario}`) diff --git a/packages/app/src/testing/compare-perf-script.test.ts b/packages/app/src/testing/compare-perf-script.test.ts index 2d5c35a34..e47aa1cb5 100644 --- a/packages/app/src/testing/compare-perf-script.test.ts +++ b/packages/app/src/testing/compare-perf-script.test.ts @@ -116,4 +116,57 @@ describe("compare-perf script", () => { intersectedFailures: [], }) }) + + test("keeps top-level missing scenario failures in the confirmation scope", async () => { + const root = await mkdtemp(join(tmpdir(), "compare-perf-script-")) + tempRoots.push(root) + const basePath = join(root, "base.json") + const headPath = join(root, "head.json") + const initialPath = join(root, "perf-compare.json") + const outputPath = join(root, "perf-compare-confirm.json") + const initialComparison: PerfBaselineComparison = { + pass: false, + failures: ["missing_head_scenario:default:session-scroll-reading"], + warnings: [], + scenarios: [], + } + + await writeJson(basePath, [scenario({ branch: "base" })]) + await writeJson(headPath, []) + await writeJson(initialPath, initialComparison) + + const child = Bun.spawn( + [ + process.execPath, + "script/compare-perf.ts", + "--base", + basePath, + "--head", + headPath, + "--output", + outputPath, + "--failures-from", + initialPath, + ], + { + cwd: process.cwd(), + stderr: "pipe", + stdout: "pipe", + }, + ) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + + expect(stderr).toBe("") + expect(exitCode).toBe(1) + const summary = JSON.parse(stdout) as Pick + expect(summary.pass).toBe(false) + expect(summary.failures).toEqual(["missing_head_scenario:default:session-scroll-reading"]) + const comparison = JSON.parse(await readFile(outputPath, "utf8")) as PerfBaselineComparison + expect(comparison.pass).toBe(false) + expect(comparison.failures).toEqual(["missing_head_scenario:default:session-scroll-reading"]) + }) }) From b53932eea9425a94c0a8dbdbbbd0b6279aaf6625 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Sat, 23 May 2026 17:43:48 +0800 Subject: [PATCH 5/5] fix: dedupe perf baseline comparison keys --- packages/app/src/testing/perf-metrics.test.ts | 14 ++++++++++++++ packages/app/src/testing/perf-metrics.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/app/src/testing/perf-metrics.test.ts b/packages/app/src/testing/perf-metrics.test.ts index 586918517..ed6798d4d 100644 --- a/packages/app/src/testing/perf-metrics.test.ts +++ b/packages/app/src/testing/perf-metrics.test.ts @@ -405,6 +405,20 @@ describe("perf metrics", () => { ]) }) + test("deduplicates fallback comparison keys from baseline scenarios", () => { + const result = comparePerfBaselines({ + base: [ + scenario({ branch: "base", profile: "default", scenario: "session-scroll-reading" }), + scenario({ branch: "base", profile: "default", scenario: "session-scroll-reading" }), + ], + head: [scenario({ branch: "head", profile: "default", scenario: "session-scroll-reading" })], + }) + + expect(result.scenarios.map((entry) => `${entry.profile}:${entry.scenario}`)).toEqual([ + "default:session-scroll-reading", + ]) + }) + test("fails when the matching profile and scenario is missing", () => { const base = [ scenario({ branch: "base", profile: "default", scenario: "session-timeline-recompute" }), diff --git a/packages/app/src/testing/perf-metrics.ts b/packages/app/src/testing/perf-metrics.ts index 598b1c5ef..d6c3017e0 100644 --- a/packages/app/src/testing/perf-metrics.ts +++ b/packages/app/src/testing/perf-metrics.ts @@ -465,7 +465,7 @@ export function comparePerfBaselines(input: { pushUnique(confirmation.intersectedFailures, failure) } - const comparisonKeys = requestedScenarioKeys ?? input.base.map((scenario) => scenarioKey(scenario)) + const comparisonKeys = requestedScenarioKeys ?? [...new Set(input.base.map((scenario) => scenarioKey(scenario)))] for (const key of comparisonKeys) { const baseScenario = baseByScenario.get(key)