diff --git a/docs/config/update.md b/docs/config/update.md index a154000bd696..d38ad1ba863c 100644 --- a/docs/config/update.md +++ b/docs/config/update.md @@ -5,11 +5,17 @@ outline: deep # update {#update} -- **Type:** `boolean | 'new' | 'all'` +- **Type:** `boolean | 'new' | 'all' | 'none'` - **Default:** `false` -- **CLI:** `-u`, `--update`, `--update=false`, `--update=new` +- **CLI:** `-u`, `--update`, `--update=false`, `--update=new`, `--update=none` -Update snapshot files. The behaviour depends on the value: +Define snapshot update behavior. -- `true` or `'all'`: updates all changed snapshots and delete obsolete ones +- `true` or `'all'`: updates all changed snapshots and deletes obsolete ones - `new`: generates new snapshots without changing or deleting obsolete ones +- `none`: does not write snapshots and fails on snapshot mismatches, missing snapshots, and obsolete snapshots + +When `update` is `false` (the default), Vitest resolves snapshot update mode by environment: + +- Local runs (non-CI): works same as `new` +- CI runs (`process.env.CI` is truthy): works same as `none` diff --git a/docs/guide/cli-generated.md b/docs/guide/cli-generated.md index 52f67fc9a247..d86486f98517 100644 --- a/docs/guide/cli-generated.md +++ b/docs/guide/cli-generated.md @@ -16,7 +16,7 @@ Path to config file - **CLI:** `-u, --update [type]` - **Config:** [update](/config/update) -Update snapshot (accepts boolean, "new" or "all") +Update snapshot (accepts boolean, "new", "all" or "none") ### watch diff --git a/docs/guide/snapshot.md b/docs/guide/snapshot.md index b9a3ba6405d7..18b0b58a636a 100644 --- a/docs/guide/snapshot.md +++ b/docs/guide/snapshot.md @@ -79,6 +79,12 @@ Or you can use the `--update` or `-u` flag in the CLI to make Vitest update snap vitest -u ``` +### CI behavior + +By default, Vitest does not write snapshots in CI (`process.env.CI` is truthy) and any snapshot mismatches, missing snapshots, and obsolete snapshots fail the run. See [`update`](/config/update) for the details. + +An **obsolete snapshot** is a snapshot entry (or snapshot file) that no longer matches any collected test. This usually happens after removing or renaming tests. + ## File Snapshots When calling `toMatchSnapshot()`, we store all snapshots in a formatted snap file. That means we need to escape some characters (namely the double-quote `"` and backtick `` ` ``) in the snapshot string. Meanwhile, you might lose the syntax highlighting for the snapshot content (if they are in some language). diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index b10a61856685..ddf7641096b2 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -84,7 +84,7 @@ export const cliOptionsConfig: VitestCLIOptions = { }, update: { shorthand: 'u', - description: 'Update snapshot (accepts boolean, "new" or "all")', + description: 'Update snapshot (accepts boolean, "new", "all" or "none")', argument: '[type]', }, watch: { diff --git a/packages/vitest/src/node/config/resolveConfig.ts b/packages/vitest/src/node/config/resolveConfig.ts index eb886f865d8e..3c849ea68201 100644 --- a/packages/vitest/src/node/config/resolveConfig.ts +++ b/packages/vitest/src/node/config/resolveConfig.ts @@ -566,7 +566,7 @@ export function resolveConfig( expand: resolved.expandSnapshotDiff ?? false, snapshotFormat: resolved.snapshotFormat || {}, updateSnapshot: - UPDATE_SNAPSHOT === 'all' || UPDATE_SNAPSHOT === 'new' + UPDATE_SNAPSHOT === 'all' || UPDATE_SNAPSHOT === 'new' || UPDATE_SNAPSHOT === 'none' ? UPDATE_SNAPSHOT : isCI && !UPDATE_SNAPSHOT ? 'none' : UPDATE_SNAPSHOT ? 'all' : 'new', resolveSnapshotPath: options.resolveSnapshotPath, diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index e603e1d4b8d9..29c83bc1108a 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -380,7 +380,7 @@ export interface InlineConfig { * * @default false */ - update?: boolean | 'all' | 'new' + update?: boolean | 'all' | 'new' | 'none' /** * Watch mode diff --git a/test/cli/test/around-each.test.ts b/test/cli/test/around-each.test.ts index 247b64066fce..fff738ef7e12 100644 --- a/test/cli/test/around-each.test.ts +++ b/test/cli/test/around-each.test.ts @@ -1463,6 +1463,9 @@ test('aroundAll throws error when runSuite is not called', async () => { expect(errorTree()).toMatchInlineSnapshot(` { "no-run.test.ts": { + "__module_errors__": [ + "The \`runSuite()\` callback was not called in the \`aroundAll\` hook. Make sure to call \`runSuite()\` to run the suite.", + ], "test": "skipped", }, } @@ -1584,6 +1587,9 @@ test('aroundAll setup phase timeout', async () => { expect(errorTree()).toMatchInlineSnapshot(` { "timeout.test.ts": { + "__module_errors__": [ + "The setup phase of "aroundAll" hook timed out after 10ms.", + ], "test": "skipped", }, } @@ -1635,6 +1641,9 @@ test('aroundAll teardown phase timeout', async () => { expect(errorTree()).toMatchInlineSnapshot(` { "teardown-timeout.test.ts": { + "__module_errors__": [ + "The teardown phase of "aroundAll" hook timed out after 10ms.", + ], "test": "passed", }, } @@ -1974,6 +1983,9 @@ test('tests are skipped when aroundAll setup fails', async () => { expect(errorTree()).toMatchInlineSnapshot(` { "aroundAll-setup-error.test.ts": { + "__module_errors__": [ + "aroundAll setup error", + ], "test should be skipped": "skipped", }, } @@ -2461,6 +2473,9 @@ test('nested aroundAll setup error is not propagated to outer runSuite catch', a expect(errorTree()).toMatchInlineSnapshot(` { "nested-around-all-setup-error.test.ts": { + "__module_errors__": [ + "inner aroundAll setup error", + ], "repro": "skipped", }, } @@ -2524,6 +2539,9 @@ test('nested aroundAll teardown error is not propagated to outer runSuite catch' expect(errorTree()).toMatchInlineSnapshot(` { "nested-around-all-teardown-error.test.ts": { + "__module_errors__": [ + "inner aroundAll teardown error", + ], "repro": "passed", }, } @@ -2712,6 +2730,11 @@ test('three nested aroundAll teardown errors are all reported', async () => { expect(errorTree()).toMatchInlineSnapshot(` { "triple-around-all-teardown-errors.test.ts": { + "__module_errors__": [ + "inner aroundAll teardown error", + "middle aroundAll teardown error", + "outer aroundAll teardown error", + ], "repro": "passed", }, } diff --git a/test/cli/test/mocking.test.ts b/test/cli/test/mocking.test.ts index 16c3dd6935d6..8c0afda75423 100644 --- a/test/cli/test/mocking.test.ts +++ b/test/cli/test/mocking.test.ts @@ -43,26 +43,14 @@ test('spy is not called here', () => { }) test('invalid packages', async () => { - const { results, errorTree } = await runVitest({ + const { stderr, errorTree } = await runVitest({ root: path.join(import.meta.dirname, '../fixtures/invalid-package'), }) - const testModuleErrors = Object.fromEntries( - results.map(testModule => [ - testModule.relativeModuleId, - testModule.errors().map(e => e.message), - ]), - ) // requires Vite 8 for relaxed import analysis validataion // https://github.com/vitejs/vite/pull/21601 if (rolldownVersion) { - expect(testModuleErrors).toMatchInlineSnapshot(` - { - "mock-bad-dep.test.ts": [], - "mock-wrapper-and-bad-dep.test.ts": [], - "mock-wrapper.test.ts": [], - } - `) + expect(stderr).toMatchInlineSnapshot(`""`) expect(errorTree()).toMatchInlineSnapshot(` { "mock-bad-dep.test.ts": { @@ -78,32 +66,31 @@ test('invalid packages', async () => { `) } else { - expect(testModuleErrors).toMatchInlineSnapshot(` + expect(errorTree()).toMatchInlineSnapshot(` { - "mock-bad-dep.test.ts": [ - "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", - ], - "mock-wrapper-and-bad-dep.test.ts": [ - "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", - ], - "mock-wrapper.test.ts": [ - "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", - ], + "mock-bad-dep.test.ts": { + "__module_errors__": [ + "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", + ], + }, + "mock-wrapper-and-bad-dep.test.ts": { + "__module_errors__": [ + "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", + ], + }, + "mock-wrapper.test.ts": { + "__module_errors__": [ + "Failed to resolve entry for package "test-dep-invalid". The package may have incorrect main/module/exports specified in its package.json.", + ], + }, } `) - expect(errorTree()).toMatchInlineSnapshot(` - { - "mock-bad-dep.test.ts": {}, - "mock-wrapper-and-bad-dep.test.ts": {}, - "mock-wrapper.test.ts": {}, - } - `) } }) test('mocking modules with syntax error', async () => { // TODO: manual mocked module still gets transformed so this is not supported yet. - const { errorTree, results } = await runInlineTests({ + const { errorTree } = await runInlineTests({ './syntax-error.js': `syntax error`, './basic.test.js': /* ts */ ` import * as dep from './syntax-error.js' @@ -118,38 +105,31 @@ test('can mock invalid module', () => { `, }) - const testModuleErrors = Object.fromEntries( - results.map(testModule => [ - testModule.relativeModuleId, - testModule.errors().map(e => e.message), - ]), - ) if (rolldownVersion) { - expect(testModuleErrors).toMatchInlineSnapshot(` + expect(errorTree()).toMatchInlineSnapshot(` { - "basic.test.js": [ - "Parse failure: Parse failed with 1 error: + "basic.test.js": { + "__module_errors__": [ + "Parse failure: Parse failed with 1 error: Expected a semicolon or an implicit semicolon after a statement, but found none 1: syntax error ^ At file: /syntax-error.js:1:6", - ], + ], + }, } `) } else { - expect(testModuleErrors).toMatchInlineSnapshot(` + expect(errorTree()).toMatchInlineSnapshot(` { - "basic.test.js": [ - "Parse failure: Expected ';', '}' or + "basic.test.js": { + "__module_errors__": [ + "Parse failure: Expected ';', '}' or At file: /syntax-error.js:1:7", - ], + ], + }, } `) } - expect(errorTree()).toMatchInlineSnapshot(` - { - "basic.test.js": {}, - } - `) }) diff --git a/test/snapshots/test/ci.test.ts b/test/snapshots/test/ci.test.ts new file mode 100644 index 000000000000..37eec75015cc --- /dev/null +++ b/test/snapshots/test/ci.test.ts @@ -0,0 +1,54 @@ +import fs from 'node:fs' +import path from 'node:path' +import { expect, test } from 'vitest' +import { runVitestCli } from '../../test-utils' + +test('CI behavior', async () => { + // cleanup snapshot + const root = path.join(import.meta.dirname, 'fixtures/ci') + fs.rmSync(path.join(root, '__snapshots__'), { recursive: true, force: true }) + + // snapshot fails with CI + let result = await runVitestCli({ + nodeOptions: { + env: { + CI: 'true', + GITHUB_ACTIONS: 'true', + }, + }, + }, '--root', root) + expect(result.stderr).toMatchInlineSnapshot(` + " + ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL basic.test.ts > basic + Error: Snapshot \`basic 1\` mismatched + ❯ basic.test.ts:4:16 + 2| + 3| test("basic", () => { + 4| expect("ok").toMatchSnapshot() + | ^ + 5| }) + 6| + + ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + " + `) + + // snapshot created without CI + result = await runVitestCli( + { + nodeOptions: { + env: { + CI: '', + GITHUB_ACTIONS: '', + }, + }, + }, + '--root', + root, + ) + expect(result.stderr).toMatchInlineSnapshot(`""`) + expect(result.stdout).toContain('Snapshots 1 written') +}) diff --git a/test/snapshots/test/fixtures/ci/.gitignore b/test/snapshots/test/fixtures/ci/.gitignore new file mode 100644 index 000000000000..b05c2dfa7007 --- /dev/null +++ b/test/snapshots/test/fixtures/ci/.gitignore @@ -0,0 +1 @@ +__snapshots__ diff --git a/test/snapshots/test/fixtures/ci/basic.test.ts b/test/snapshots/test/fixtures/ci/basic.test.ts new file mode 100644 index 000000000000..9727e71ae6f2 --- /dev/null +++ b/test/snapshots/test/fixtures/ci/basic.test.ts @@ -0,0 +1,5 @@ +import { test, expect } from "vitest" + +test("basic", () => { + expect("ok").toMatchSnapshot() +}) diff --git a/test/snapshots/test/obsolete.test.ts b/test/snapshots/test/obsolete.test.ts index 8e66f3583174..a6992ccc4adc 100644 --- a/test/snapshots/test/obsolete.test.ts +++ b/test/snapshots/test/obsolete.test.ts @@ -1,39 +1,68 @@ import fs from 'node:fs' import path from 'node:path' import { expect, test } from 'vitest' -import { runVitestCli } from '../../test-utils' +import { runVitest } from '../../test-utils' -test('obsolete snapshot fails CI', async () => { +test('obsolete snapshot fails with update:none', async () => { // cleanup snapshot const root = path.join(import.meta.dirname, 'fixtures/obsolete') fs.rmSync(path.join(root, 'src/__snapshots__'), { recursive: true, force: true }) // initial run to write snapshot - let vitest = await runVitestCli('--root', root, '--update') - expect(vitest.stdout).toContain('Snapshots 5 written') - expect(vitest.stderr).toBe('') + let result = await runVitest({ root, update: true }) + expect(result.stderr).toBe('') + expect(result.errorTree()).toMatchInlineSnapshot(` + Object { + "src/test1.test.ts": Object { + "bar": "passed", + "foo": "passed", + "fuu": "passed", + }, + "src/test2.test.ts": Object { + "bar": "passed", + "foo": "passed", + }, + } + `) // test fails with obsolete snapshots - // (use cli to test `updateSnapshot: 'none'`) - vitest = await runVitestCli( - { - nodeOptions: { - env: { - CI: 'true', - TEST_OBSOLETE: 'true', - }, - }, - }, - '--root', + result = await runVitest({ root, - ) - expect(vitest.stdout).toContain('2 obsolete') - expect(vitest.stdout).toContain('Test Files 1 failed | 1 passed') - expect(vitest.stdout).toContain('Tests 5 passed') - expect(vitest.stderr).toContain(` -Error: Obsolete snapshots found when no snapshot update is expected. -· foo 1 -· fuu 1 -`) - expect(vitest.exitCode).toBe(1) + update: 'none', + env: { + TEST_OBSOLETE: 'true', + }, + }) + expect(result.stderr).toMatchInlineSnapshot(` + " + ⎯⎯⎯⎯⎯⎯ Failed Suites 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL src/test1.test.ts [ src/test1.test.ts ] + Error: Obsolete snapshots found when no snapshot update is expected. + · foo 1 + · fuu 1 + + ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + " + `) + expect(result.errorTree()).toMatchInlineSnapshot(` + Object { + "src/test1.test.ts": Object { + "__module_errors__": Array [ + "Obsolete snapshots found when no snapshot update is expected. + · foo 1 + · fuu 1 + ", + ], + "bar": "passed", + "foo": "passed", + "fuu": "passed", + }, + "src/test2.test.ts": Object { + "bar": "passed", + "foo": "passed", + }, + } + `) }) diff --git a/test/snapshots/test/test-update.test.ts b/test/snapshots/test/test-update.test.ts index 5df1f63fc2e3..272b502d1f20 100644 --- a/test/snapshots/test/test-update.test.ts +++ b/test/snapshots/test/test-update.test.ts @@ -67,7 +67,7 @@ test('test update', async () => { `) // re-run without update and files are unchanged - const result2 = await runVitest({ root: dstDir, update: false }) + const result2 = await runVitest({ root: dstDir, update: 'none' }) expect(result2.stderr).toMatchInlineSnapshot(`""`) expect(result2.errorTree()).toEqual(result.errorTree()) expect(readFiles(dstDir)).toEqual(resultFiles) diff --git a/test/test-utils/index.ts b/test/test-utils/index.ts index 5b2d3a1828b2..738239854376 100644 --- a/test/test-utils/index.ts +++ b/test/test-utils/index.ts @@ -575,6 +575,16 @@ export function buildErrorTree(testModules: TestModule[]) { } return suiteChildren }, + (testModule, moduleChildren) => { + const errors = testModule.errors().map(error => error.message) + if (errors.length > 0) { + return { + ...moduleChildren, + __module_errors__: errors, + } + } + return moduleChildren + }, ) } @@ -582,6 +592,7 @@ export function buildTestTree( testModules: TestModule[], onTestCase?: (result: TestCase) => unknown, onTestSuite?: (testSuite: TestSuite, suiteChildren: Record) => unknown, + onTestModule?: (testModule: TestModule, moduleChildren: Record) => unknown, ) { type TestTree = Record @@ -613,7 +624,8 @@ export function buildTestTree( for (const module of testModules) { // Use relative module ID for cleaner output const key = module.relativeModuleId - tree[key] = walkCollection(module.children) + const moduleChildren = walkCollection(module.children) + tree[key] = onTestModule ? onTestModule(module, moduleChildren) : moduleChildren } return tree