Skip to content

Commit

Permalink
fix(benchmark): run benchmark suites sequentially (#5444)
Browse files Browse the repository at this point in the history
  • Loading branch information
hi-ogawa authored Apr 2, 2024
1 parent 0930c29 commit 1f54834
Show file tree
Hide file tree
Showing 18 changed files with 151 additions and 71 deletions.
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ dist
ltex*
.DS_Store
bench/test/*/*/
**/benchmark/bench.json
**/bench.json
**/browser/browser.json
cypress/videos
cypress/downloads
Expand Down
3 changes: 2 additions & 1 deletion packages/vitest/src/node/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ export function resolveConfig(
if (resolved.minWorkers)
resolved.minWorkers = Number(resolved.minWorkers)

resolved.fileParallelism ??= true
// run benchmark sequentially by default
resolved.fileParallelism ??= mode !== 'benchmark'

if (!resolved.fileParallelism) {
// ignore user config, parallelism cannot be implemented without limiting workers
Expand Down
5 changes: 3 additions & 2 deletions packages/vitest/src/runtime/runners/benchmark.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ async function runBenchmarkSuite(suite: Suite, runner: NodeBenchmarkRunner) {
benchmarkSuiteGroup.push(task)
}

if (benchmarkSuiteGroup.length)
await Promise.all(benchmarkSuiteGroup.map(subSuite => runBenchmarkSuite(subSuite, runner)))
// run sub suites sequentially
for (const subSuite of benchmarkSuiteGroup)
await runBenchmarkSuite(subSuite, runner)

if (benchmarkGroup.length) {
const defer = createDefer()
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 3 additions & 0 deletions test/benchmark/fixtures/basic/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import { defineConfig } from 'vitest/config'

export default defineConfig({})
26 changes: 26 additions & 0 deletions test/benchmark/fixtures/sequential/f1.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import { bench, describe } from "vitest"
import { appendLog, benchOptions, sleepBench } from "./helper";

bench("B1", async () => {
await appendLog("F1 / B1")
await sleepBench();
}, benchOptions)

describe("S1", () => {
bench("B1", async () => {
await appendLog("F1 / S1 / B1")
await sleepBench();
}, benchOptions)

bench("B2", async () => {
await appendLog("F1 / S1 / B2")
await sleepBench();
}, benchOptions)
})

describe("S2", () => {
bench("B1", async () => {
await appendLog("F1 / S2 / B1")
await sleepBench();
}, benchOptions)
})
9 changes: 9 additions & 0 deletions test/benchmark/fixtures/sequential/f2.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { bench, describe } from "vitest"
import { appendLog, benchOptions, sleepBench } from "./helper";

describe("S1", () => {
bench("B1", async () => {
await appendLog("F2 / S1 / B1")
await sleepBench();
}, benchOptions)
})
19 changes: 19 additions & 0 deletions test/benchmark/fixtures/sequential/helper.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import fs from "node:fs";

const SLEEP_BENCH_MS = Number(process.env["SLEEP_BENCH_MS"] || 10);
const BENCH_ITERATIONS = Number(process.env["BENCH_ITERATIONS"] || 3);

export const sleepBench = () => new Promise(resolve => setTimeout(resolve, SLEEP_BENCH_MS))

export const testLogFile = new URL("./test.log", import.meta.url);

export async function appendLog(data: string) {
await fs.promises.appendFile(testLogFile, data + "\n");
}

export const benchOptions = {
time: 0,
iterations: BENCH_ITERATIONS,
warmupIterations: 0,
warmupTime: 0,
}
6 changes: 6 additions & 0 deletions test/benchmark/fixtures/sequential/setup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import fs from "node:fs";
import { testLogFile } from "./helper";

export default async function setup() {
await fs.promises.rm(testLogFile, { force: true });
}
10 changes: 10 additions & 0 deletions test/benchmark/fixtures/sequential/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
import { defineConfig } from "vitest/config"

// to see the difference better, increase sleep time and iterations e.g. by
// SLEEP_BENCH_MS=100 pnpm -C test/benchmark test bench -- --root fixtures/sequential --fileParallelism

export default defineConfig({
test: {
globalSetup: ["./setup.ts"]
}
});
6 changes: 2 additions & 4 deletions test/benchmark/package.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
{
"name": "@vitest/benchmark",
"name": "@vitest/benchmark-sequential",
"type": "module",
"private": true,
"scripts": {
"test": "node --test specs/* && echo '1'",
"bench:json": "vitest bench --reporter=json",
"bench": "vitest bench"
"test": "vitest"
},
"devDependencies": {
"vitest": "workspace:*"
Expand Down
40 changes: 0 additions & 40 deletions test/benchmark/specs/runner.test.mjs

This file was deleted.

20 changes: 20 additions & 0 deletions test/benchmark/test/__snapshots__/sequential.test.ts.snap
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html

exports[`sequential 1`] = `
"F1 / S1 / B1
F1 / S1 / B1
F1 / S1 / B1
F1 / S1 / B2
F1 / S1 / B2
F1 / S1 / B2
F1 / S2 / B1
F1 / S2 / B1
F1 / S2 / B1
F1 / B1
F1 / B1
F1 / B1
F2 / S1 / B1
F2 / S1 / B1
F2 / S1 / B1
"
`;
38 changes: 38 additions & 0 deletions test/benchmark/test/basic.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import fs from 'node:fs'
import { expect, it } from 'vitest'
import * as pathe from 'pathe'
import { runVitest } from '../../test-utils'

it('basic', { timeout: 60_000 }, async () => {
const root = pathe.join(import.meta.dirname, '../fixtures/basic')
const benchFile = pathe.join(root, 'bench.json')
fs.rmSync(benchFile, { force: true })

await runVitest({
root,
allowOnly: true,
benchmark: {
reporters: 'json',
outputFile: 'bench.json',
},
}, [], 'benchmark')

const benchResult = await fs.promises.readFile(benchFile, 'utf-8')
const resultJson = JSON.parse(benchResult)

expect(Object.keys(resultJson.testResults)).toEqual(
expect.arrayContaining([
'sort',
'timeout',
'a0',
'c1',
'a2',
'b3',
'b4',
]),
)

const skipped = ['skip', 's0', 's1', 's2', 's3', 'sb4', 's4', 'unimplemented suite', 'unimplemented test']
for (const b of skipped)
expect(benchResult).not.toContain(b)
})
11 changes: 11 additions & 0 deletions test/benchmark/test/sequential.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import fs from 'node:fs'
import { expect, it } from 'vitest'
import * as pathe from 'pathe'
import { runVitest } from '../../test-utils'

it('sequential', async () => {
const root = pathe.join(import.meta.dirname, '../fixtures/sequential')
await runVitest({ root }, [], 'benchmark')
const testLog = await fs.promises.readFile(pathe.join(root, 'test.log'), 'utf-8')
expect(testLog).toMatchSnapshot()
})
24 changes: 1 addition & 23 deletions test/benchmark/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
import { defineConfig } from 'vitest/config'

function noop() {}

export default defineConfig({
test: {
update: false,
allowOnly: true,
benchmark: {
outputFile: './bench.json',
reporters: ['json', {
onInit: noop,
onPathsCollected: noop,
onCollected: noop,
onFinished: noop,
onTaskUpdate: noop,
onTestRemoved: noop,
onWatcherStart: noop,
onWatcherRerun: noop,
onServerRestart: noop,
onUserConsoleLog: noop,
}],
},
},
})
export default defineConfig({})

0 comments on commit 1f54834

Please sign in to comment.