diff --git a/.gitignore b/.gitignore index 700fe350bb06..02fe571b8450 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ docs/public/sponsors .eslintcache docs/.vitepress/cache/ !test/cli/fixtures/dotted-files/**/.cache +.vitest-reports \ No newline at end of file diff --git a/README.md b/README.md index 1babe9962dfa..a33f98c65a92 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Next generation testing framework powered by Vite. - [Vite](https://vitejs.dev/)'s config, transformers, resolvers, and plugins. Use the same setup from your app! - [Jest Snapshot](https://jestjs.io/docs/snapshot-testing) -- [Chai](https://www.chaijs.com/) built-in for assertions, with [Jest expect](https://jestjs.io/docs/expect) compatible APIs. +- [Chai](https://www.chaijs.com/) built-in for assertions, with [Jest expect](https://jestjs.io/docs/expect) compatible APIs - [Smart & instant watch mode](https://vitest.dev/guide/features.html#watch-mode), like HMR for tests! - [Native code coverage](https://vitest.dev/guide/features.html#coverage) via [`v8`](https://v8.dev/blog/javascript-code-coverage) or [`istanbul`](https://istanbul.js.org/). - [Tinyspy](https://github.com/tinylibs/tinyspy) built-in for mocking, stubbing, and spies. @@ -45,6 +45,7 @@ Next generation testing framework powered by Vite. - ESM first, top level await - Out-of-box TypeScript / JSX support - Filtering, timeouts, concurrent for suite and tests +- Sharding support > Vitest 1.0 requires Vite >=v5.0.0 and Node >=v18.0.0 diff --git a/docs/.vitepress/components.d.ts b/docs/.vitepress/components.d.ts index 8557a7924e77..5805e44bf38f 100644 --- a/docs/.vitepress/components.d.ts +++ b/docs/.vitepress/components.d.ts @@ -1,10 +1,10 @@ /* eslint-disable */ -/* prettier-ignore */ // @ts-nocheck // Generated by unplugin-vue-components // Read more: https://github.com/vuejs/core/pull/3399 export {} +/* prettier-ignore */ declare module 'vue' { export interface GlobalComponents { Contributors: typeof import('./components/Contributors.vue')['default'] diff --git a/docs/.vitepress/components/FeaturesList.vue b/docs/.vitepress/components/FeaturesList.vue index e72b53576d6f..9327df9cd91e 100644 --- a/docs/.vitepress/components/FeaturesList.vue +++ b/docs/.vitepress/components/FeaturesList.vue @@ -4,7 +4,7 @@ dir="auto" flex="~ col gap2 md:gap-3" > - Vite's config, transformers, resolvers, and plugins. + Vite's config, transformers, resolvers, and plugins Use the same setup from your app to run the tests! Smart & instant watch mode, like HMR for tests! Component testing for Vue, React, Svelte, Lit, Marko and more @@ -25,6 +25,7 @@ Code coverage via v8 or istanbul Rust-like in-source testing Type Testing via expect-type + Sharding support diff --git a/docs/guide/cli-table.md b/docs/guide/cli-table.md index ddcd0045d8f5..19084089c5c1 100644 --- a/docs/guide/cli-table.md +++ b/docs/guide/cli-table.md @@ -120,3 +120,4 @@ | `--no-color` | Removes colors from the console output | | `--clearScreen` | Clear terminal screen when re-running tests during watch mode (default: `true`) | | `--standalone` | Start Vitest without running tests. File filters will be ignored, tests will be running only on change (default: `false`) | +| `--mergeReports [path]` | Paths to blob reports directory. If this options is used, Vitest won't run any tests, it will only report previously recorded tests | diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 9043dd2d2a49..0b4807ef5a6a 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -113,4 +113,18 @@ vitest --api=false You cannot use this option with `--watch` enabled (enabled in dev by default). ::: +::: tip +If `--reporter=blob` is used without an output file, the default path will include the current shard config to avoid collisions with other Vitest processes. +::: + +### merge-reports + +- **Type:** `boolean | string` + +Merges every blob report located in the specified folder (`.vitest-reports` by default). You can use any reporters with this command (except [`blob`](/guide/reporters#blob-reporter)): + +```sh +vitest --merge-reports --reporter=junit +``` + [cac's dot notation]: https://github.com/cacjs/cac#dot-nested-options diff --git a/docs/guide/features.md b/docs/guide/features.md index db0be1c40cbe..fdc6679c3e83 100644 --- a/docs/guide/features.md +++ b/docs/guide/features.md @@ -229,3 +229,14 @@ test('my types work properly', () => { assertType(mount({ name: 42 })) }) ``` + +## Sharding + +Run tests on different machines using [`--shard`](/guide/cli#shard) and [`--reporter=blob`](/guide/reporters#blob-reporter) flags. +All test results can be merged at the end of your CI pipeline using `--merge-reports` command: + +```bash +vitest --shard=1/2 --reporter=blob +vitest --shard=2/2 --reporter=blob +vitest --merge-reports --reporter=junit +``` diff --git a/docs/guide/reporters.md b/docs/guide/reporters.md index 617bd791fadc..7bced6f09170 100644 --- a/docs/guide/reporters.md +++ b/docs/guide/reporters.md @@ -462,6 +462,26 @@ export default defineConfig({ Github Actions Github Actions +### Blob Reporter + +Stores test results on the machine so they can be later merged using [`--merge-reports`](/guide/cli#merge-reports) command. +By default, stores all results in `.vitest-reports` folder, but can be overriden with `--outputFile` or `--outputFile.blob` flags. + +```bash +npx vitest --reporter=blob --outputFile=reports/blob-1.json +``` + +We recommend using this reporter if you are running Vitest on different machines with the [`--shard`](/guide/cli#shard) flag. +All blob reports can be merged into any report by using `--merge-reports` command at the end of your CI pipeline: + +```bash +npx vitest --merge-reports=reports --reporter=json --reporter=default +``` + +::: tip +Both `--reporter=blob` and `--merge-reports` do not work in watch mode. +::: + ## Custom Reporters You can use third-party custom reporters installed from NPM by specifying their package name in the reporters' option: diff --git a/packages/runner/src/collect.ts b/packages/runner/src/collect.ts index 5cf3f3ba4e00..e567ba5201cb 100644 --- a/packages/runner/src/collect.ts +++ b/packages/runner/src/collect.ts @@ -1,8 +1,7 @@ -import { relative } from 'pathe' import { processError } from '@vitest/utils/error' import type { File, SuiteHooks } from './types' import type { VitestRunner } from './types/runner' -import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from './utils/collect' +import { calculateSuiteHash, createFileTask, interpretTaskModes, someTasksAreOnly } from './utils/collect' import { clearCollectorContext, createSuiteHooks, getDefaultSuite } from './suite' import { getHooks, setHooks } from './map' import { collectorContext } from './context' @@ -16,19 +15,9 @@ export async function collectTests(paths: string[], runner: VitestRunner): Promi const config = runner.config for (const filepath of paths) { - const path = relative(config.root, filepath) - const file: File = { - id: generateHash(`${path}${config.name || ''}`), - name: path, - type: 'suite', - mode: 'run', - filepath, - tasks: [], - meta: Object.create(null), - projectName: config.name, - file: undefined!, - } - file.file = file + const file = createFileTask(filepath, config.root, config.name) + + runner.onCollectStart?.(file) clearCollectorContext(filepath, runner) diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index ce4d18f7a942..633277afbc99 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -53,6 +53,10 @@ export interface VitestRunner { * First thing that's getting called before actually collecting and running tests. */ onBeforeCollect?: (paths: string[]) => unknown + /** + * Called after the file task was created but not collected yet. + */ + onCollectStart?: (file: File) => unknown /** * Called after collecting tests and before "onBeforeRun". */ diff --git a/packages/runner/src/utils/collect.ts b/packages/runner/src/utils/collect.ts index 7318fdf028ef..eef88cc295aa 100644 --- a/packages/runner/src/utils/collect.ts +++ b/packages/runner/src/utils/collect.ts @@ -1,5 +1,6 @@ import { processError } from '@vitest/utils/error' -import type { Suite, TaskBase } from '../types' +import { relative } from 'pathe' +import type { File, Suite, TaskBase } from '../types' /** * If any tasks been marked as `only`, mark all other tasks as `skip`. @@ -92,3 +93,20 @@ export function calculateSuiteHash(parent: Suite) { calculateSuiteHash(t) }) } + +export function createFileTask(filepath: string, root: string, projectName: string) { + const path = relative(root, filepath) + const file: File = { + id: generateHash(`${path}${projectName || ''}`), + name: path, + type: 'suite', + mode: 'run', + filepath, + tasks: [], + meta: Object.create(null), + projectName, + file: undefined!, + } + file.file = file + return file +} diff --git a/packages/ui/client/components/views/ViewConsoleOutput.vue b/packages/ui/client/components/views/ViewConsoleOutput.vue index afde92011c9e..f98016aea766 100644 --- a/packages/ui/client/components/views/ViewConsoleOutput.vue +++ b/packages/ui/client/components/views/ViewConsoleOutput.vue @@ -15,6 +15,8 @@ const formattedLogs = computed(() => { function getTaskName(id?: string) { const task = id && client.state.idMap.get(id) + if (task && 'filepath' in task) + return task.name return (task ? getNames(task).slice(1).join(' > ') : '-') || '-' } diff --git a/packages/ui/client/composables/client/index.ts b/packages/ui/client/composables/client/index.ts index 257872c00061..f6a7366fb7af 100644 --- a/packages/ui/client/composables/client/index.ts +++ b/packages/ui/client/composables/client/index.ts @@ -3,8 +3,7 @@ import type { WebSocketStatus } from '@vueuse/core' import type { ErrorWithDiff, File, ResolvedConfig } from 'vitest' import type { Ref } from 'vue' import { reactive } from 'vue' -import { relative } from 'pathe' -import { generateHash } from '@vitest/runner/utils' +import { createFileTask } from '@vitest/runner/utils' import type { RunState } from '../../../types' import { ENTRY_URL, isReport } from '../../constants' import { parseError } from '../error' @@ -92,21 +91,8 @@ watch( ]) if (_config.standalone) { const filenames = await client.rpc.getTestFiles() - const files = filenames.map(([name, filepath]) => { - const path = relative(_config.root, filepath) - return { - filepath, - name: path, - id: /* #__PURE__ */ generateHash(`${path}${name || ''}`), - mode: 'skip', - type: 'suite', - result: { - state: 'skip', - }, - meta: {}, - tasks: [], - projectName: name, - } + const files = filenames.map(([{ name, root }, filepath]) => { + return /* #__PURE__ */ createFileTask(filepath, root, name) }) client.state.collectFiles(files) } diff --git a/packages/vite-node/src/utils.ts b/packages/vite-node/src/utils.ts index cffe441da604..db4eae90584e 100644 --- a/packages/vite-node/src/utils.ts +++ b/packages/vite-node/src/utils.ts @@ -45,11 +45,9 @@ export function normalizeRequestId(id: string, base?: string): string { .replace(/\?+$/, '') // remove end query mark } -export const queryRE = /\?.*$/s -export const hashRE = /#.*$/s - +const postfixRE = /[?#].*$/ export function cleanUrl(url: string): string { - return url.replace(hashRE, '').replace(queryRE, '') + return url.replace(postfixRE, '') } const internalRequests = [ diff --git a/packages/vitest/src/api/setup.ts b/packages/vitest/src/api/setup.ts index 37c3bb92dbb2..40d533409b90 100644 --- a/packages/vitest/src/api/setup.ts +++ b/packages/vitest/src/api/setup.ts @@ -11,7 +11,7 @@ import type { ViteDevServer } from 'vite' import type { StackTraceParserOptions } from '@vitest/utils/source-map' import { API_PATH } from '../constants' import type { Vitest } from '../node' -import type { File, ModuleGraphData, Reporter, TaskResultPack, UserConsoleLog } from '../types' +import type { Awaitable, File, ModuleGraphData, Reporter, SerializableSpec, TaskResultPack, UserConsoleLog } from '../types' import { getModuleGraph, isPrimitive, noop, stringifyReplace } from '../utils' import type { WorkspaceProject } from '../node/workspace' import { parseErrorStacktrace } from '../utils/source-map' @@ -166,7 +166,10 @@ export function setup(vitestOrWorkspace: Vitest | WorkspaceProject, _server?: Vi }, async getTestFiles() { const spec = await ctx.globTestFiles() - return spec.map(([project, file]) => [project.getName(), file]) as [string, string][] + return spec.map(([project, file]) => [{ + name: project.getName(), + root: project.config.root, + }, file]) }, }, { @@ -208,6 +211,14 @@ export class WebSocketReporter implements Reporter { }) } + onSpecsCollected(specs?: SerializableSpec[] | undefined): Awaitable { + if (this.clients.size === 0) + return + this.clients.forEach((client) => { + client.onSpecsCollected?.(specs)?.catch?.(noop) + }) + } + async onTaskUpdate(packs: TaskResultPack[]) { if (this.clients.size === 0) return diff --git a/packages/vitest/src/api/types.ts b/packages/vitest/src/api/types.ts index 45afb11fdf6e..5a72b2a4c105 100644 --- a/packages/vitest/src/api/types.ts +++ b/packages/vitest/src/api/types.ts @@ -15,7 +15,7 @@ export interface WebSocketHandlers { getCountOfFailedTests: () => number sendLog: (log: UserConsoleLog) => void getFiles: () => File[] - getTestFiles: () => Promise<[name: string, file: string][]> + getTestFiles: () => Promise<[{ name: string; root: string }, file: string][]> getPaths: () => string[] getConfig: () => ResolvedConfig resolveSnapshotPath: (testPath: string) => string @@ -39,7 +39,7 @@ export interface WebSocketHandlers { debug: (...args: string[]) => void } -export interface WebSocketEvents extends Pick { +export interface WebSocketEvents extends Pick { onCancel: (reason: CancelReason) => void onFinishedReportCoverage: () => void } diff --git a/packages/vitest/src/node/cli/cli-api.ts b/packages/vitest/src/node/cli/cli-api.ts index b6f783c8b0bd..a9ed2e9b9d3b 100644 --- a/packages/vitest/src/node/cli/cli-api.ts +++ b/packages/vitest/src/node/cli/cli-api.ts @@ -95,7 +95,9 @@ export async function startVitest( }) try { - if (ctx.config.standalone) + if (ctx.config.mergeReports) + await ctx.mergeReports() + else if (ctx.config.standalone) await ctx.init() else await ctx.start(cliFilters) diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index c9c935373e00..6acc96b4a804 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -606,6 +606,15 @@ export const cliOptionsConfig: VitestCLIOptions = { standalone: { description: 'Start Vitest without running tests. File filters will be ignored, tests will be running only on change (default: `false`)', }, + mergeReports: { + description: 'Paths to blob reports directory. If this options is used, Vitest won\'t run any tests, it will only report previously recorded tests', + argument: '[path]', + transform(value) { + if (!value || typeof value === 'boolean') + return '.vitest-reports' + return value + }, + }, // disable CLI options cliExclude: null, diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 6b438170d8cf..d769c55af031 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -139,6 +139,9 @@ export function resolveConfig( if (resolved.standalone && !resolved.watch) throw new Error(`Vitest standalone mode requires --watch`) + if (resolved.mergeReports && resolved.watch) + throw new Error(`Cannot merge reports with --watch enabled`) + if (resolved.maxWorkers) resolved.maxWorkers = Number(resolved.maxWorkers) diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index d4da6fb277ab..6ef32188c4e0 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -9,11 +9,11 @@ import mm from 'micromatch' import c from 'picocolors' import { ViteNodeRunner } from 'vite-node/client' import { SnapshotManager } from '@vitest/snapshot/manager' -import type { CancelReason, File } from '@vitest/runner' +import type { CancelReason, File, TaskResultPack } from '@vitest/runner' import { ViteNodeServer } from 'vite-node/server' import type { defineWorkspace } from 'vitest/config' import { version } from '../../package.json' with { type: 'json' } -import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, UserWorkspaceConfig, VitestRunMode } from '../types' +import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, SerializableSpec, UserConfig, UserConsoleLog, UserWorkspaceConfig, VitestRunMode } from '../types' import { getTasks, hasFailed, noop, slash, toArray, wildcardPatternToRegExp } from '../utils' import { getCoverageProvider } from '../integrations/coverage' import { CONFIG_NAMES, configFiles, workspacesFiles as workspaceFiles } from '../constants' @@ -28,6 +28,7 @@ import { Logger } from './logger' import { VitestCache } from './cache' import { WorkspaceProject, initializeProject } from './workspace' import { VitestPackageInstaller } from './packageInstaller' +import { BlobReporter, readBlobs } from './reporters/blob' const WATCHER_DEBOUNCE = 100 @@ -196,6 +197,12 @@ export class Vitest { || this.projects[0] } + public getProjectByName(name: string) { + return this.projects.find(p => p.getName() === name) + || this.getCoreWorkspaceProject() + || this.projects[0] + } + private async getWorkspaceConfigPath() { if (this.config.workspace) return this.config.workspace @@ -382,6 +389,58 @@ export class Vitest { return Promise.all(this.projects.map(w => w.initBrowserProvider())) } + async mergeReports() { + if (this.reporters.some(r => r instanceof BlobReporter)) + throw new Error('Cannot merge reports when `--reporter=blob` is used. Remove blob reporter from the config first.') + + const { files, errors } = await readBlobs(this.config.mergeReports, this.projects) + + await this.report('onInit', this) + await this.report('onPathsCollected', files.flatMap(f => f.filepath)) + + const workspaceSpecs = new Map() + for (const file of files) { + const project = this.getProjectByName(file.projectName) + const specs = workspaceSpecs.get(project) || [] + specs.push(file) + workspaceSpecs.set(project, specs) + } + + for (const [project, files] of workspaceSpecs) { + const filepaths = files.map(f => f.filepath) + this.state.clearFiles(project, filepaths) + files.forEach((file) => { + file.logs?.forEach(log => this.state.updateUserLog(log)) + }) + this.state.collectFiles(files) + } + + await this.report('onCollected', files).catch(noop) + + for (const file of files) { + const logs: UserConsoleLog[] = [] + const taskPacks: TaskResultPack[] = [] + + const tasks = getTasks(file) + for (const task of tasks) { + if (task.logs) + logs.push(...task.logs) + taskPacks.push([task.id, task.result, task.meta]) + } + logs.sort((log1, log2) => log1.time - log2.time) + + for (const log of logs) + await this.report('onUserConsoleLog', log).catch(noop) + + await this.report('onTaskUpdate', taskPacks).catch(noop) + } + + if (hasFailed(files)) + process.exitCode = 1 + + await this.report('onFinished', files, errors) + } + async start(filters?: string[]) { this._onClose = [] @@ -536,13 +595,17 @@ export class Vitest { this.distPath = join(vitestDir, 'dist') } - async runFiles(paths: WorkspaceSpec[], allTestsRun: boolean) { + async runFiles(specs: WorkspaceSpec[], allTestsRun: boolean) { await this.initializeDistPath() - const filepaths = paths.map(([, file]) => file) + const filepaths = specs.map(([, file]) => file) this.state.collectPaths(filepaths) await this.report('onPathsCollected', filepaths) + await this.report('onSpecsCollected', specs.map( + ([project, file]) => + [{ name: project.getName(), root: project.config.root }, file] as SerializableSpec, + )) // previous run await this.runningPromise @@ -562,10 +625,10 @@ export class Vitest { if (!this.isFirstRun && this.config.coverage.cleanOnRerun) await this.coverageProvider?.clean() - await this.initializeGlobalSetup(paths) + await this.initializeGlobalSetup(specs) try { - await this.pool.runTests(paths, invalidates) + await this.pool.runTests(specs, invalidates) } catch (err) { this.state.catchError(err, 'Unhandled Error') @@ -581,8 +644,8 @@ export class Vitest { })() .finally(async () => { // can be duplicate files if different projects are using the same file - const specs = Array.from(new Set(paths.map(([, p]) => p))) - await this.report('onFinished', this.state.getFiles(specs), this.state.getUnhandledErrors()) + const files = Array.from(new Set(specs.map(([, p]) => p))) + await this.report('onFinished', this.state.getFiles(files), this.state.getUnhandledErrors()) await this.reportCoverage(allTestsRun) this.runningPromise = undefined diff --git a/packages/vitest/src/node/reporters/base.ts b/packages/vitest/src/node/reporters/base.ts index 36a6454f3a1b..3014c8ce6a83 100644 --- a/packages/vitest/src/node/reporters/base.ts +++ b/packages/vitest/src/node/reporters/base.ts @@ -18,11 +18,15 @@ const WAIT_FOR_CHANGE_CANCELLED = `\n${c.bold(c.inverse(c.red(' CANCELLED ')))}$ const LAST_RUN_LOG_TIMEOUT = 1_500 +export interface BaseOptions { + isTTY?: boolean +} + export abstract class BaseReporter implements Reporter { start = 0 end = 0 watchFilters?: string[] - isTTY = isNode && process.stdout?.isTTY && !isCI + isTTY: boolean ctx: Vitest = undefined! private _filesInWatchMode = new Map() @@ -32,7 +36,8 @@ export abstract class BaseReporter implements Reporter { private _timeStart = new Date() private _offUnhandledRejection?: () => void - constructor() { + constructor(options: BaseOptions = {}) { + this.isTTY = options.isTTY ?? (isNode && process.stdout?.isTTY && !isCI) this.registerUnhandledRejection() } diff --git a/packages/vitest/src/node/reporters/blob.ts b/packages/vitest/src/node/reporters/blob.ts new file mode 100644 index 000000000000..bedf64b9bf79 --- /dev/null +++ b/packages/vitest/src/node/reporters/blob.ts @@ -0,0 +1,125 @@ +import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises' +import { existsSync } from 'node:fs' +import { parse, stringify } from 'flatted' +import { dirname, resolve } from 'pathe' +import { cleanUrl } from 'vite-node/utils' +import type { File, Reporter, Vitest } from '../../types' +import { getOutputFile } from '../../utils/config-helpers' +import type { WorkspaceProject } from '../workspace' + +export interface BlobOptions { + outputFile?: string +} + +export class BlobReporter implements Reporter { + ctx!: Vitest + options: BlobOptions + + constructor(options: BlobOptions) { + this.options = options + } + + onInit(ctx: Vitest): void { + if (ctx.config.watch) + throw new Error('Blob reporter is not supported in watch mode') + + this.ctx = ctx + } + + async onFinished(files: File[] = [], errors: unknown[] = []) { + let outputFile = this.options.outputFile ?? getOutputFile(this.ctx.config, 'blob') + if (!outputFile) { + const shard = this.ctx.config.shard + outputFile = shard + ? `.vitest-reports/blob-${shard.index}-${shard.count}.json` + : '.vitest-reports/blob.json' + } + + const moduleKeys = this.ctx.projects.map((project) => { + return [project.getName(), [...project.server.moduleGraph.idToModuleMap.keys()]] + }) + + const report = stringify([this.ctx.version, files, errors, moduleKeys] satisfies MergeReport) + + const reportFile = resolve(this.ctx.config.root, outputFile) + + const dir = dirname(reportFile) + if (!existsSync(dir)) + await mkdir(dir, { recursive: true }) + + await writeFile( + reportFile, + report, + 'utf-8', + ) + this.ctx.logger.log('blob report written to', reportFile) + } +} + +export async function readBlobs(blobsDirectory: string, projectsArray: WorkspaceProject[]) { + // using process.cwd() because --merge-reports can only be used in CLI + const resolvedDir = resolve(process.cwd(), blobsDirectory) + const blobsFiles = await readdir(resolvedDir) + const promises = blobsFiles.map(async (file) => { + const content = await readFile(resolve(resolvedDir, file), 'utf-8') + const [version, files, errors, moduleKeys] = parse(content) as MergeReport + return { version, files, errors, moduleKeys } + }) + const blobs = await Promise.all(promises) + + if (!blobs.length) + throw new Error(`vitest.mergeReports() requires at least one blob file paths in the config`) + + // fake module graph - it is used to check if module is imported, but we don't use values inside + const projects = Object.fromEntries(projectsArray.map(p => [p.getName(), p])) + + blobs.forEach((blob) => { + blob.moduleKeys.forEach(([projectName, moduleIds]) => { + const project = projects[projectName] + if (!project) + return + moduleIds.forEach((moduleId) => { + project.server.moduleGraph.idToModuleMap.set(moduleId, { + id: moduleId, + url: moduleId, + file: cleanUrl(moduleId), + ssrTransformResult: null, + transformResult: null, + importedBindings: null, + importedModules: new Set(), + importers: new Set(), + type: 'js', + clientImportedModules: new Set(), + ssrError: null, + ssrImportedModules: new Set(), + ssrModule: null, + acceptedHmrDeps: new Set(), + acceptedHmrExports: null, + lastHMRTimestamp: 0, + lastInvalidationTimestamp: 0, + }) + }) + }) + }) + + const files = blobs.flatMap(blob => blob.files).sort((f1, f2) => { + const time1 = f1.result?.startTime || 0 + const time2 = f2.result?.startTime || 0 + return time1 - time2 + }) + const errors = blobs.flatMap(blob => blob.errors) + + return { + files, + errors, + } +} + +type MergeReport = [ + vitestVersion: string, + files: File[], + errors: unknown[], + moduleKeys: MergeReportModuleKeys[], +] + +type MergeReportModuleKeys = [projectName: string, moduleIds: string[]] diff --git a/packages/vitest/src/node/reporters/index.ts b/packages/vitest/src/node/reporters/index.ts index 06a62055dc27..c3f83ec8ad79 100644 --- a/packages/vitest/src/node/reporters/index.ts +++ b/packages/vitest/src/node/reporters/index.ts @@ -9,8 +9,10 @@ import { type JUnitOptions, JUnitReporter } from './junit' import { TapFlatReporter } from './tap-flat' import { HangingProcessReporter } from './hanging-process' import { GithubActionsReporter } from './github-actions' -import type { BaseReporter } from './base' +import type { BaseOptions, BaseReporter } from './base' import type { HTMLOptions } from './html' +import type { BlobOptions } from './blob' +import { BlobReporter } from './blob' export { DefaultReporter, @@ -31,6 +33,7 @@ export type { JsonAssertionResult, JsonTestResult, JsonTestResults } from './jso export const ReportersMap = { 'default': DefaultReporter, 'basic': BasicReporter, + 'blob': BlobReporter, 'verbose': VerboseReporter, 'dot': DotReporter, 'json': JsonReporter, @@ -44,11 +47,12 @@ export const ReportersMap = { export type BuiltinReporters = keyof typeof ReportersMap export interface BuiltinReporterOptions { - 'default': never - 'basic': never + 'default': BaseOptions + 'basic': BaseOptions 'verbose': never - 'dot': never + 'dot': BaseOptions 'json': JsonOptions + 'blob': BlobOptions 'tap': never 'tap-flat': never 'junit': JUnitOptions diff --git a/packages/vitest/src/node/state.ts b/packages/vitest/src/node/state.ts index 7e14cc0bb5e4..edf7063adcfa 100644 --- a/packages/vitest/src/node/state.ts +++ b/packages/vitest/src/node/state.ts @@ -1,7 +1,7 @@ -import { relative } from 'pathe' import type { File, Task, TaskResultPack } from '@vitest/runner' // can't import actual functions from utils, because it's incompatible with @vitest/browsers +import { createFileTask } from '@vitest/runner/utils' import type { AggregateError as AggregateErrorPonyfill } from '../utils/base' import type { UserConsoleLog } from '../types/general' import type { WorkspaceProject } from './workspace' @@ -91,6 +91,11 @@ export class StateManager { files.forEach((file) => { const existing = (this.filesMap.get(file.filepath) || []) const otherProject = existing.filter(i => i.projectName !== file.projectName) + const currentFile = existing.find(i => i.projectName === file.projectName) + // keep logs for the previous file because it should alway be initiated before the collections phase + // which means that all logs are collected during the collection and not inside tests + if (currentFile) + file.logs = currentFile.logs otherProject.push(file) this.filesMap.set(file.filepath, otherProject) this.updateId(file) @@ -98,17 +103,27 @@ export class StateManager { } // this file is reused by ws-client, and shoult not rely on heavy dependencies like workspace - clearFiles(_project: { config: { name: string } }, paths: string[] = []) { + clearFiles(_project: { config: { name: string; root: string } }, paths: string[] = []) { const project = _project as WorkspaceProject paths.forEach((path) => { const files = this.filesMap.get(path) - if (!files) + const fileTask = createFileTask(path, project.config.root, project.config.name) + this.idMap.set(fileTask.id, fileTask) + if (!files) { + this.filesMap.set(path, [fileTask]) return + } const filtered = files.filter(file => file.projectName !== project.config.name) - if (!filtered.length) - this.filesMap.delete(path) - else - this.filesMap.set(path, filtered) + // always keep a File task, so we can associate logs with it + if (!filtered.length) { + this.filesMap.set(path, [fileTask]) + } + else { + this.filesMap.set(path, [ + ...filtered, + fileTask, + ]) + } }) } @@ -150,24 +165,6 @@ export class StateManager { } cancelFiles(files: string[], root: string, projectName: string) { - this.collectFiles(files.map((filepath) => { - const file: File = { - filepath, - name: relative(root, filepath), - id: filepath, - mode: 'skip', - type: 'suite', - result: { - state: 'skip', - }, - meta: {}, - // Cancelled files have not yet collected tests - tasks: [], - projectName, - file: null!, - } - file.file = file - return file - })) + this.collectFiles(files.map(filepath => createFileTask(filepath, root, projectName))) } } diff --git a/packages/vitest/src/runtime/runners/test.ts b/packages/vitest/src/runtime/runners/test.ts index 3f4f5a34bc0b..73c9f0c7d8e7 100644 --- a/packages/vitest/src/runtime/runners/test.ts +++ b/packages/vitest/src/runtime/runners/test.ts @@ -23,6 +23,10 @@ export class VitestTestRunner implements VitestRunner { return this.__vitest_executor.executeId(filepath) } + onCollectStart(file: File) { + this.workerState.current = file + } + onBeforeRunFiles() { this.snapshotClient.clear() } diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index a45326944a1d..5124a3d47876 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -864,6 +864,12 @@ export interface UserConfig extends InlineConfig { * benchmark.outputJson option exposed at the top level for cli */ outputJson?: string + + /** + * Directory of blob reports to merge + * @default '.vitest-reports' + */ + mergeReports?: string } export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'browser' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence' | 'typecheck' | 'runner' | 'poolOptions' | 'pool' | 'cliExclude'> { diff --git a/packages/vitest/src/types/reporter.ts b/packages/vitest/src/types/reporter.ts index 5c696b92be0c..be0aa2fe8062 100644 --- a/packages/vitest/src/types/reporter.ts +++ b/packages/vitest/src/types/reporter.ts @@ -5,6 +5,7 @@ import type { File, TaskResultPack } from './tasks' export interface Reporter { onInit?: (ctx: Vitest) => void onPathsCollected?: (paths?: string[]) => Awaitable + onSpecsCollected?: (specs?: SerializableSpec[]) => Awaitable onCollected?: (files?: File[]) => Awaitable onFinished?: (files?: File[], errors?: unknown[]) => Awaitable onTaskUpdate?: (packs: TaskResultPack[]) => Awaitable @@ -21,3 +22,4 @@ export interface Reporter { } export type { Vitest } +export type SerializableSpec = [project: { name: string; root: string }, file: string] diff --git a/packages/ws-client/src/index.ts b/packages/ws-client/src/index.ts index 3069899bb392..3c137fc3a051 100644 --- a/packages/ws-client/src/index.ts +++ b/packages/ws-client/src/index.ts @@ -52,6 +52,12 @@ export function createClient(url: string, options: VitestClientOptions = {}) { let onMessage: Function const functions: WebSocketEvents = { + onSpecsCollected(specs) { + specs?.forEach(([config, file]) => { + ctx.state.clearFiles({ config }, [file]) + }) + handlers.onSpecsCollected?.(specs) + }, onPathsCollected(paths) { ctx.state.collectPaths(paths) handlers.onPathsCollected?.(paths) @@ -66,6 +72,7 @@ export function createClient(url: string, options: VitestClientOptions = {}) { }, onUserConsoleLog(log) { ctx.state.updateUserLog(log) + handlers.onUserConsoleLog?.(log) }, onFinished(files, errors) { handlers.onFinished?.(files, errors) diff --git a/test/cli/test/setup-files.test.ts b/test/cli/test/setup-files.test.ts index 93342aa62a9b..bd320fb7e7da 100644 --- a/test/cli/test/setup-files.test.ts +++ b/test/cli/test/setup-files.test.ts @@ -10,7 +10,7 @@ test.each(['threads', 'vmThreads'])('%s: print stdout and stderr correctly when pool, }) - const filepath = 'console-setup.ts' + const filepath = 'empty.test.ts' expect(stdout).toContain(`stdout | ${filepath}`) expect(stderr).toContain(`stderr | ${filepath}`) }) diff --git a/test/config/test/failures.test.ts b/test/config/test/failures.test.ts index 79d8af2ca1fa..c4f49d16e82b 100644 --- a/test/config/test/failures.test.ts +++ b/test/config/test/failures.test.ts @@ -1,11 +1,11 @@ import { expect, test } from 'vitest' -import type { UserConfig } from 'vitest/config' +import type { UserConfig } from 'vitest' import { version } from 'vitest/package.json' import { normalize, resolve } from 'pathe' import * as testUtils from '../../test-utils' -function runVitest(config: NonNullable & { shard?: any }) { +function runVitest(config: NonNullable & { shard?: any }) { return testUtils.runVitest({ root: './fixtures/test', ...config }, []) } @@ -153,3 +153,9 @@ test('nextTick can be mocked inside worker_threads', async () => { expect(stderr).not.toMatch('Error') }) + +test('mergeReports doesn\'t work with watch mode enabled', async () => { + const { stderr } = await runVitest({ watch: true, mergeReports: '.vitest-reports' }) + + expect(stderr).toMatch('Cannot merge reports with --watch enabled') +}) diff --git a/test/core/test/cli-test.test.ts b/test/core/test/cli-test.test.ts index 384b96166ee4..841657df52b8 100644 --- a/test/core/test/cli-test.test.ts +++ b/test/core/test/cli-test.test.ts @@ -310,6 +310,12 @@ test('clearScreen', async () => { `) }) +test('merge-reports', () => { + expect(getCLIOptions('--merge-reports')).toEqual({ mergeReports: '.vitest-reports' }) + expect(getCLIOptions('--merge-reports=different-folder')).toEqual({ mergeReports: 'different-folder' }) + expect(getCLIOptions('--merge-reports different-folder')).toEqual({ mergeReports: 'different-folder' }) +}) + test('public parseCLI works correctly', () => { expect(parseCLI('vitest dev')).toEqual({ filter: [], diff --git a/test/reporters/fixtures/merge-reports/first.test.ts b/test/reporters/fixtures/merge-reports/first.test.ts new file mode 100644 index 000000000000..123f422c4882 --- /dev/null +++ b/test/reporters/fixtures/merge-reports/first.test.ts @@ -0,0 +1,16 @@ +import { test, expect, beforeEach } from 'vitest' + +console.log('global scope') + +beforeEach(() => { + console.log('beforeEach') +}) + +test('test 1-1', () => { + expect(1).toBe(1) +}) + +test('test 1-2', () => { + console.log('test 1-2') + expect(1).toBe(2) +}) diff --git a/test/reporters/fixtures/merge-reports/second.test.ts b/test/reporters/fixtures/merge-reports/second.test.ts new file mode 100644 index 000000000000..98a248f6657c --- /dev/null +++ b/test/reporters/fixtures/merge-reports/second.test.ts @@ -0,0 +1,16 @@ +import { describe, test, expect } from 'vitest' + +test('test 2-1', () => { + console.log('test 2-1') + expect(1).toBe(2) +}) + +describe('group', () => { + test('test 2-2', () => { + expect(1).toBe(1) + }) + + test('test 2-3', () => { + expect(1).toBe(1) + }) +}) diff --git a/test/reporters/fixtures/merge-reports/vitest.config.js b/test/reporters/fixtures/merge-reports/vitest.config.js new file mode 100644 index 000000000000..b1c6ea436a54 --- /dev/null +++ b/test/reporters/fixtures/merge-reports/vitest.config.js @@ -0,0 +1 @@ +export default {} diff --git a/test/reporters/tests/merge-reports.test.ts b/test/reporters/tests/merge-reports.test.ts new file mode 100644 index 000000000000..3f1161f038e3 --- /dev/null +++ b/test/reporters/tests/merge-reports.test.ts @@ -0,0 +1,226 @@ +import { resolve } from 'node:path' +import { expect, test } from 'vitest' +import { runVitest } from '../../test-utils' + +test('merge reports', async () => { + await runVitest({ + root: './fixtures/merge-reports', + include: ['first.test.ts'], + reporters: [['blob', { outputFile: './.vitest-reports/first-run.json' }]], + }) + await runVitest({ + root: './fixtures/merge-reports', + include: ['second.test.ts'], + reporters: [['blob', { outputFile: './.vitest-reports/second-run.json' }]], + }) + + // always relative to CWD because it's used only from the CLI, + // so we need to correctly resolve it here + const mergeReports = resolve('./fixtures/merge-reports/.vitest-reports') + + const { stdout: reporterDefault, stderr: stderrDefault, exitCode } = await runVitest({ + root: './fixtures/merge-reports', + mergeReports, + reporters: [['default', { isTTY: false }]], + }) + + expect(exitCode).toBe(1) + + // remove "RUN v{} path" and "Duration" because it's not stable + const stdoutCheck = reporterDefault + .split('\n') + .slice(2, -3) + .join('\n') + .replace(/Start at [\w\s\d:]+/, 'Start at