fix(reporters): --merge-reports to show each total run times (#7877)

Author: AriPerkkio

packages/vitest/src/node/core.ts

@@ -499,7 +499,8 @@ export class Vitest {
       throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.')
     }
 
-    const { files, errors, coverages } = await readBlobs(this.version, directory || this.config.mergeReports, this.projects)
+    const { files, errors, coverages, executionTimes } = await readBlobs(this.version, directory || this.config.mergeReports, this.projects)
+    this.state.blobs = { files, errors, coverages, executionTimes }
 
     await this.report('onInit', this)
     await this.report('onPathsCollected', files.flatMap(f => f.filepath))

packages/vitest/src/node/reporters/base.ts

@@ -489,7 +489,11 @@ export abstract class BaseReporter implements Reporter {
       this.log(padSummaryTitle('Duration'), formatTime(collectTime + testsTime + setupTime))
     }
     else {
-      const executionTime = this.end - this.start
+      const blobs = this.ctx.state.blobs
+
+      // Execution time is either sum of all runs of `--merge-reports` or the current run's time
+      const executionTime = blobs?.executionTimes ? sum(blobs.executionTimes, time => time) : this.end - this.start
+
       const environmentTime = sum(files, file => file.environmentLoad)
       const prepareTime = sum(files, file => file.prepareDuration)
       const transformTime = sum(this.ctx.projects, project => project.vitenode.getTotalDuration())
@@ -506,6 +510,10 @@ export abstract class BaseReporter implements Reporter {
       ].filter(Boolean).join(', ')
 
       this.log(padSummaryTitle('Duration'), formatTime(executionTime) + c.dim(` (${timers})`))
+
+      if (blobs?.executionTimes) {
+        this.log(padSummaryTitle('Per blob') + blobs.executionTimes.map(time => ` ${formatTime(time)}`).join(''))
+      }
     }
 
     this.log()

packages/vitest/src/node/reporters/blob.ts

@@ -13,6 +13,7 @@ export interface BlobOptions {
 }
 
 export class BlobReporter implements Reporter {
+  start = 0
   ctx!: Vitest
   options: BlobOptions
 
@@ -26,13 +27,16 @@ export class BlobReporter implements Reporter {
     }
 
     this.ctx = ctx
+    this.start = performance.now()
   }
 
   async onFinished(
     files: File[] = [],
     errors: unknown[] = [],
     coverage: unknown,
   ): Promise<void> {
+    const executionTime = performance.now() - this.start
+
     let outputFile
       = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob')
     if (!outputFile) {
@@ -56,31 +60,38 @@ export class BlobReporter implements Reporter {
       },
     )
 
-    const report = stringify([
+    const report = [
       this.ctx.version,
       files,
       errors,
       modules,
       coverage,
-    ] satisfies MergeReport)
+      executionTime,
+    ] satisfies MergeReport
 
     const reportFile = resolve(this.ctx.config.root, outputFile)
+    await writeBlob(report, reportFile)
 
-    const dir = dirname(reportFile)
-    if (!existsSync(dir)) {
-      await mkdir(dir, { recursive: true })
-    }
-
-    await writeFile(reportFile, report, 'utf-8')
     this.ctx.logger.log('blob report written to', reportFile)
   }
 }
 
+export async function writeBlob(content: MergeReport, filename: string): Promise<void> {
+  const report = stringify(content)
+
+  const dir = dirname(filename)
+  if (!existsSync(dir)) {
+    await mkdir(dir, { recursive: true })
+  }
+
+  await writeFile(filename, report, 'utf-8')
+}
+
 export async function readBlobs(
   currentVersion: string,
   blobsDirectory: string,
   projectsArray: TestProject[],
-): Promise<{ files: File[]; errors: unknown[]; coverages: unknown[] }> {
+): Promise<MergedBlobs> {
   // using process.cwd() because --merge-reports can only be used in CLI
   const resolvedDir = resolve(process.cwd(), blobsDirectory)
   const blobsFiles = await readdir(resolvedDir)
@@ -93,15 +104,15 @@ export async function readBlobs(
       )
     }
     const content = await readFile(fullPath, 'utf-8')
-    const [version, files, errors, moduleKeys, coverage] = parse(
+    const [version, files, errors, moduleKeys, coverage, executionTime] = parse(
       content,
     ) as MergeReport
     if (!version) {
       throw new TypeError(
         `vitest.mergeReports() expects all paths in "${blobsDirectory}" to be files generated by the blob reporter, but "${filename}" is not a valid blob file`,
       )
     }
-    return { version, files, errors, moduleKeys, coverage, file: filename }
+    return { version, files, errors, moduleKeys, coverage, file: filename, executionTime }
   })
   const blobs = await Promise.all(promises)
 
@@ -153,20 +164,30 @@ export async function readBlobs(
     })
   const errors = blobs.flatMap(blob => blob.errors)
   const coverages = blobs.map(blob => blob.coverage)
+  const executionTimes = blobs.map(blob => blob.executionTime)
 
   return {
     files,
     errors,
     coverages,
+    executionTimes,
   }
 }
 
+export interface MergedBlobs {
+  files: File[]
+  errors: unknown[]
+  coverages: unknown[]
+  executionTimes: number[]
+}
+
 type MergeReport = [
   vitestVersion: string,
   files: File[],
   errors: unknown[],
   modules: MergeReportModuleKeys[],
   coverage: unknown,
+  executionTime: number,
 ]
 
 type SerializedModuleNode = [

packages/vitest/src/node/state.ts

@@ -1,6 +1,7 @@
 import type { File, Task, TaskResultPack } from '@vitest/runner'
 import type { UserConsoleLog } from '../types/general'
 import type { TestProject } from './project'
+import type { MergedBlobs } from './reporters/blob'
 import { createFileTask } from '@vitest/runner/utils'
 import { TestCase, TestModule, TestSuite } from './reporters/reported-tasks'
 
@@ -20,6 +21,7 @@ export class StateManager {
   errorsSet: Set<unknown> = new Set()
   processTimeoutCauses: Set<string> = new Set()
   reportedTasksMap: WeakMap<Task, TestModule | TestCase | TestSuite> = new WeakMap()
+  blobs?: MergedBlobs
 
   catchError(err: unknown, type: string): void {
     if (isAggregateError(err)) {

test/reporters/tests/merge-reports.test.ts

@@ -1,37 +1,42 @@
+import type { File, Test } from '@vitest/runner/types'
+import { rmSync } from 'node:fs'
 import { resolve } from 'node:path'
-import { expect, test } from 'vitest'
+import { createFileTask } from '@vitest/runner/utils'
+import { beforeEach, expect, test } from 'vitest'
+import { version } from 'vitest/package.json'
+import { writeBlob } from 'vitest/src/node/reporters/blob.js'
 import { runVitest } from '../../test-utils'
 
+// always relative to CWD because it's used only from the CLI,
+// so we need to correctly resolve it here
+const reportsDir = resolve('./fixtures/merge-reports/.vitest-reports')
+
+beforeEach(() => {
+  rmSync(reportsDir, { force: true, recursive: true })
+})
+
 test('merge reports', async () => {
   await runVitest({
     root: './fixtures/merge-reports',
     include: ['first.test.ts'],
     reporters: [['blob', { outputFile: './.vitest-reports/first-run.json' }]],
   })
+
   await runVitest({
     root: './fixtures/merge-reports',
     include: ['second.test.ts'],
     reporters: [['blob', { outputFile: './.vitest-reports/second-run.json' }]],
   })
 
-  // always relative to CWD because it's used only from the CLI,
-  // so we need to correctly resolve it here
-  const mergeReports = resolve('./fixtures/merge-reports/.vitest-reports')
-
   const { stdout: reporterDefault, stderr: stderrDefault, exitCode } = await runVitest({
     root: './fixtures/merge-reports',
-    mergeReports,
+    mergeReports: reportsDir,
     reporters: [['default', { isTTY: false }]],
   })
 
   expect(exitCode).toBe(1)
 
-  // remove "RUN v{} path" and "Duration" because it's not stable
-  const stdoutCheck = reporterDefault
-    .split('\n')
-    .slice(2, -3)
-    .join('\n')
-    .replace(/Start at [\w\s:]+/, 'Start at <time>')
+  const stdoutCheck = trimReporterOutput(reporterDefault)
   const stderrArr = stderrDefault.split('\n')
   const stderrCheck = [
     ...stderrArr.slice(4, 19),
@@ -76,9 +81,8 @@ test('merge reports', async () => {
     "
   `)
 
-  expect(stdoutCheck.replace(/\d+ms/g, '<time>')).toMatchInlineSnapshot(`
-    "
-    stdout | first.test.ts
+  expect(stdoutCheck).toMatchInlineSnapshot(`
+    "stdout | first.test.ts
     global scope
 
     stdout | first.test.ts > test 1-1
@@ -105,12 +109,13 @@ test('merge reports', async () => {
 
      Test Files  2 failed (2)
           Tests  2 failed | 3 passed (5)
-       Start at <time>"
+       Duration  <time> (transform <time>, setup <time>, collect <time>, tests <time>, environment <time>, prepare <time>)
+       Per blob  <time> <time>"
   `)
 
   const { stdout: reporterJson } = await runVitest({
     root: './fixtures/merge-reports',
-    mergeReports,
+    mergeReports: reportsDir,
     reporters: [['json', { outputFile: /** so it outputs into stdout */ null }]],
   })
 
@@ -234,3 +239,62 @@ test('merge reports', async () => {
     }
   `)
 })
+
+test('total and merged execution times are shown', async () => {
+  for (const [_index, name] of ['first.test.ts', 'second.test.ts'].entries()) {
+    const index = 1 + _index
+    const file = createFileTask(
+      resolve('./fixtures/merge-reports', name),
+      resolve('./fixtures/merge-reports'),
+      '',
+    )
+    file.tasks.push(createTest('some test', file))
+
+    await writeBlob(
+      [version, [file], [], [], undefined, 1500 * index],
+      resolve(`./fixtures/merge-reports/.vitest-reports/blob-${index}-2.json`),
+    )
+  }
+
+  const { stdout } = await runVitest({
+    root: resolve('./fixtures/merge-reports'),
+    mergeReports: resolve('./fixtures/merge-reports/.vitest-reports'),
+    reporters: [['default', { isTTY: false }]],
+  })
+
+  expect(stdout).toContain('✓ first.test.ts (1 test)')
+  expect(stdout).toContain('✓ second.test.ts (1 test)')
+
+  expect(stdout).toContain('Duration  4.50s')
+  expect(stdout).toContain('Per blob  1.50s 3.00s')
+})
+
+function trimReporterOutput(report: string) {
+  const rows = report
+    .replace(/\d+ms/g, '<time>')
+    .replace(/\d+\.\d+s/g, '<time>')
+    .split('\n')
+
+  // Trim start and end, capture just rendered tree
+  rows.splice(0, 1 + rows.findIndex(row => row.includes('RUN  v')))
+  rows.splice(rows.findIndex(row => row.includes('Start at')), 1)
+
+  return rows.join('\n').trim()
+}
+
+function createTest(name: string, file: File): Test {
+  file.result = { state: 'pass' }
+
+  return {
+    type: 'test',
+    name,
+    id: `${file.id}_0`,
+    mode: 'run',
+    file,
+    suite: file,
+    timeout: 0,
+    result: { state: 'pass' },
+    meta: {},
+    context: {} as any,
+  }
+}

test/reporters/vitest.config.ts

@@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config'
 
 export default defineConfig({
   test: {
+    watch: false,
     exclude: ['node_modules', 'fixtures', 'dist'],
     reporters: ['verbose'],
     testTimeout: 100000,