feat(benchmark): support comparing benchmark result

packages/vitest/src/defaults.ts

@@ -4,7 +4,7 @@ import { isCI } from './utils/env'
 
 export const defaultInclude = ['**/*.{test,spec}.?(c|m)[jt]s?(x)']
 export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**', '**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*']
-export const benchmarkConfigDefaults: Required<Omit<BenchmarkUserOptions, 'outputFile'>> = {
+export const benchmarkConfigDefaults: Required<Omit<BenchmarkUserOptions, 'outputFile' | 'compare'>> = {
   include: ['**/*.{bench,benchmark}.?(c|m)[jt]s?(x)'],
   exclude: defaultExclude,
   includeSource: [],

packages/vitest/src/node/cli/cli-config.ts

@@ -582,6 +582,10 @@ export const cliOptionsConfig: VitestCLIOptions = {
   clearScreen: {
     description: 'Clear terminal screen when re-running tests during watch mode (default: true)',
   },
+  compare: {
+    description: 'benchmark output file to compare against',
+    argument: '<filename>',
+  },
 
   // disable CLI options
   cliExclude: null,

packages/vitest/src/node/config.ts

@@ -349,6 +349,10 @@ export function resolveConfig(
 
     if (options.outputFile)
       resolved.benchmark.outputFile = options.outputFile
+
+    // --compare from cli
+    if (options.compare)
+      resolved.benchmark.compare = options.compare
   }
 
   resolved.setupFiles = toArray(resolved.setupFiles || []).map(file =>

packages/vitest/src/node/reporters/benchmark/table/index.ts

@@ -1,6 +1,11 @@
+import fs from 'node:fs'
 import c from 'picocolors'
+import * as pathe from 'pathe'
 import type { UserConsoleLog } from '../../../../types/general'
 import { BaseReporter } from '../../base'
+import type { BenchmarkResult, File } from '../../../../types'
+import { getTasks } from '../../../../utils'
+import { getOutputFile } from '../../../../utils/config-helpers'
 import { type TableRendererOptions, createTableRenderer } from './tableRender'
 
 export class TableReporter extends BaseReporter {
@@ -17,11 +22,22 @@ export class TableReporter extends BaseReporter {
     super.onWatcherStart()
   }
 
-  onCollected() {
+  async onCollected() {
     if (this.isTTY) {
       this.rendererOptions.logger = this.ctx.logger
       this.rendererOptions.showHeap = this.ctx.config.logHeapUsage
       this.rendererOptions.slowTestThreshold = this.ctx.config.slowTestThreshold
+      if (this.ctx.config.benchmark?.compare) {
+        const compareFile = pathe.resolve(this.ctx.config.root, this.ctx.config.benchmark?.compare)
+        try {
+          this.rendererOptions.compare = JSON.parse(
+            await fs.promises.readFile(compareFile, 'utf-8'),
+          )
+        }
+        catch (e) {
+          this.ctx.logger.error(`Failed to read '${compareFile}'`, e)
+        }
+      }
       const files = this.ctx.state.getFiles(this.watchFilters)
       if (!this.renderer)
         this.renderer = createTableRenderer(files, this.rendererOptions).start()
@@ -34,6 +50,17 @@ export class TableReporter extends BaseReporter {
     await this.stopListRender()
     this.ctx.logger.log()
     await super.onFinished(files, errors)
+
+    // write output for future comparison
+    let outputFile = getOutputFile(this.ctx.config.benchmark, 'default')
+    if (outputFile) {
+      outputFile = pathe.resolve(this.ctx.config.root, outputFile)
+      const outputDirectory = pathe.dirname(outputFile)
+      if (!fs.existsSync(outputDirectory))
+        await fs.promises.mkdir(outputDirectory, { recursive: true })
+      const output = createBenchmarkOutput(files)
+      await fs.promises.writeFile(outputFile, JSON.stringify(output, null, 2))
+    }
   }
 
   async onWatcherStart() {
@@ -58,3 +85,19 @@ export class TableReporter extends BaseReporter {
     super.onUserConsoleLog(log)
   }
 }
+
+export interface TableBenchmarkOutput {
+  [id: string]: Omit<BenchmarkResult, 'samples'>
+}
+
+function createBenchmarkOutput(files: File[]) {
+  const result: TableBenchmarkOutput = {}
+  for (const test of getTasks(files)) {
+    if (test.meta?.benchmark && test.result?.benchmark) {
+      // strip gigantic "samples"
+      const { samples: _samples, ...rest } = test.result.benchmark
+      result[test.id] = rest
+    }
+  }
+  return result
+}

packages/vitest/src/node/reporters/benchmark/table/tableRender.ts

@@ -1,17 +1,19 @@
 import c from 'picocolors'
 import cliTruncate from 'cli-truncate'
 import stripAnsi from 'strip-ansi'
-import type { Benchmark, BenchmarkResult, Task } from '../../../../types'
+import type { BenchmarkResult, Task } from '../../../../types'
 import { getTests, notNullish } from '../../../../utils'
 import { F_RIGHT } from '../../../../utils/figures'
 import type { Logger } from '../../../logger'
 import { getCols, getStateSymbol } from '../../renderers/utils'
+import type { TableBenchmarkOutput } from '.'
 
 export interface TableRendererOptions {
   renderSucceed?: boolean
   logger: Logger
   showHeap: boolean
   slowTestThreshold: number
+  compare?: TableBenchmarkOutput
 }
 
 const outputMap = new WeakMap<Task, string>()
@@ -35,19 +37,6 @@ function formatNumber(number: number) {
 
 const tableHead = ['name', 'hz', 'min', 'max', 'mean', 'p75', 'p99', 'p995', 'p999', 'rme', 'samples']
 
-function renderTableHead(tasks: Task[]) {
-  const benches = tasks
-    .map(i => i.meta?.benchmark ? i.result?.benchmark : undefined)
-    .filter(notNullish)
-  const allItems = benches.map(renderBenchmarkItems).concat([tableHead])
-  return `${' '.repeat(3)}${tableHead.map((i, idx) => {
-    const width = Math.max(...allItems.map(i => i[idx].length))
-    return idx
-      ? i.padStart(width, ' ')
-      : i.padEnd(width, ' ') // name
-  }).map(c.bold).join('  ')}`
-}
-
 function renderBenchmarkItems(result: BenchmarkResult) {
   return [
     result.name,
@@ -60,26 +49,36 @@ function renderBenchmarkItems(result: BenchmarkResult) {
     formatNumber(result.p995 || 0),
     formatNumber(result.p999 || 0),
     `±${(result.rme || 0).toFixed(2)}%`,
-    result.samples.length.toString(),
+    // TODO: persist only sampleCount?
+    result.samples.length ? result.samples.length.toString() : '-',
+  ]
+}
+
+function computeColumnWidths(results: BenchmarkResult[]): number[] {
+  const rows = [
+    tableHead,
+    ...results.map(v => renderBenchmarkItems(v)),
   ]
+  return Array.from(
+    tableHead,
+    (_, i) => Math.max(...rows.map(row => stripAnsi(row[i]).length)),
+  )
 }
-function renderBenchmark(task: Benchmark, tasks: Task[]): string {
-  const result = task.result?.benchmark
-  if (!result)
-    return task.name
-
-  const benches = tasks
-    .map(i => i.meta?.benchmark ? i.result?.benchmark : undefined)
-    .filter(notNullish)
-  const allItems = benches.map(renderBenchmarkItems).concat([tableHead])
-  const items = renderBenchmarkItems(result)
-  const padded = items.map((i, idx) => {
-    const width = Math.max(...allItems.map(i => i[idx].length))
-    return idx
-      ? i.padStart(width, ' ')
-      : i.padEnd(width, ' ') // name
-  })
 
+function padRow(row: string[], widths: number[]) {
+  return row.map((v, i) =>
+    i
+      ? v.padStart(widths[i], ' ')
+      : v.padEnd(widths[i], ' '), // name
+  )
+}
+
+function renderTableHead(widths: number[]) {
+  return ' '.repeat(3) + padRow(tableHead, widths).map(c.bold).join('  ')
+}
+
+function renderBenchmark(result: BenchmarkResult, widths: number[]) {
+  const padded = padRow(renderBenchmarkItems(result), widths)
   return [
     padded[0], // name
     c.blue(padded[1]), // hz
@@ -92,23 +91,42 @@ function renderBenchmark(task: Benchmark, tasks: Task[]): string {
     c.cyan(padded[8]), // p999
     c.dim(padded[9]), // rem
     c.dim(padded[10]), // sample
-    result.rank === 1
-      ? c.bold(c.green(' fastest'))
-      : (result.rank === benches.length && benches.length > 2)
-          ? c.bold(c.gray(' slowest'))
-          : '',
   ].join('  ')
 }
 
 function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): string {
   const output: string[] = []
 
+  const benchMap: Record<string, { current: BenchmarkResult; baseline?: BenchmarkResult }> = {}
+  for (const t of tasks) {
+    if (t.meta.benchmark && t.result?.benchmark) {
+      benchMap[t.id] = {
+        current: t.result.benchmark,
+      }
+      if (options.compare && options.compare[t.id]) {
+        benchMap[t.id].baseline = {
+          ...options.compare[t.id],
+          samples: [],
+          name: '',
+        }
+      }
+    }
+  }
+  const benchCount = Object.entries(benchMap).length
+
+  // compute column widths
+  const columnWidths = computeColumnWidths(
+    Object.values(benchMap)
+      .flatMap(v => [v.current, v.baseline])
+      .filter(notNullish),
+  )
+
   let idx = 0
   for (const task of tasks) {
     const padding = '  '.repeat(level ? 1 : 0)
     let prefix = ''
     if (idx === 0 && task.meta?.benchmark)
-      prefix += `${renderTableHead(tasks)}\n${padding}`
+      prefix += `${renderTableHead(columnWidths)}\n${padding}`
 
     prefix += ` ${getStateSymbol(task)} `
 
@@ -131,11 +149,37 @@ function renderTree(tasks: Task[], options: TableRendererOptions, level = 0): st
     if (level === 0)
       name = formatFilepath(name)
 
-    const body = task.meta?.benchmark
-      ? renderBenchmark(task as Benchmark, tasks)
-      : name
+    const bench = benchMap[task.id]
+    if (bench) {
+      let body = renderBenchmark(bench.current, columnWidths)
+      if (options.compare && bench.baseline) {
+        if (bench.current.hz) {
+          const diff = bench.current.hz / bench.baseline.hz
+          const diffFixed = diff.toFixed(2)
+          if (diffFixed === '1.0.0')
+            body += `  ${c.gray(`[${diffFixed}x]`)}`
+          if (diff > 1)
+            body += `  ${c.blue(`[${diffFixed}x] ⇑`)}`
+          else
+            body += `  ${c.red(`[${diffFixed}x] ⇓`)}`
+        }
+        output.push(padding + prefix + body + suffix)
+        const bodyBaseline = renderBenchmark(bench.baseline, columnWidths)
+        output.push(`${padding}   ${bodyBaseline}  ${c.dim('(baseline)')}`)
+      }
+      else {
+        if (bench.current.rank === 1 && benchCount > 1)
+          body += `  ${c.bold(c.green(' fastest'))}`
+
+        if (bench.current.rank === benchCount && benchCount > 2)
+          body += `  ${c.bold(c.gray(' slowest'))}`
 
-    output.push(padding + prefix + body + suffix)
+        output.push(padding + prefix + body + suffix)
+      }
+    }
+    else {
+      output.push(padding + prefix + name + suffix)
+    }
 
     if ((task.result?.state !== 'pass') && outputMap.get(task) != null) {
       let data: string | undefined = outputMap.get(task)

packages/vitest/src/types/benchmark.ts

@@ -39,6 +39,11 @@ export interface BenchmarkUserOptions {
    * Also definable individually per reporter by using an object instead.
    */
   outputFile?: string | (Partial<Record<BenchmarkBuiltinReporters, string>> & Record<string, string>)
+
+  /**
+   * benchmark output file to compare against
+   */
+  compare?: string
 }
 
 export interface Benchmark extends Custom {

packages/vitest/src/types/config.ts

@@ -824,6 +824,11 @@ export interface UserConfig extends InlineConfig {
    * Override vite config's clearScreen from cli
    */
   clearScreen?: boolean
+
+  /**
+   * benchmark.compare option exposed at the top level for cli
+   */
+  compare?: string
 }
 
 export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'filters' | 'browser' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence' | 'typecheck' | 'runner' | 'poolOptions' | 'pool' | 'cliExclude'> {
@@ -850,7 +855,7 @@ export interface ResolvedConfig extends Omit<Required<UserConfig>, 'config' | 'f
   api?: ApiConfig
   cliExclude?: string[]
 
-  benchmark?: Required<Omit<BenchmarkUserOptions, 'outputFile'>> & Pick<BenchmarkUserOptions, 'outputFile'>
+  benchmark?: Required<Omit<BenchmarkUserOptions, 'outputFile' | 'compare'>> & Pick<BenchmarkUserOptions, 'outputFile' | 'compare'>
   shard?: {
     index: number
     count: number