diff --git a/docs/api/advanced/vitest.md b/docs/api/advanced/vitest.md index fd1fd33e571d..98609be1054b 100644 --- a/docs/api/advanced/vitest.md +++ b/docs/api/advanced/vitest.md @@ -349,7 +349,7 @@ This makes this method very slow, unless you disable isolation before collecting function cancelCurrentRun(reason: CancelReason): Promise ``` -This method will gracefully cancel all ongoing tests. It will wait for started tests to finish running and will not run tests that were scheduled to run but haven't started yet. +This method will gracefully cancel all ongoing tests. It will stop the on-going tests and will not run tests that were scheduled to run but haven't started yet. ## setGlobalTestNamePattern diff --git a/packages/runner/src/context.ts b/packages/runner/src/context.ts index 1559d77eac36..f793b03c6901 100644 --- a/packages/runner/src/context.ts +++ b/packages/runner/src/context.ts @@ -109,6 +109,31 @@ export function withTimeout any>( }) as T } +export function withCancel any>( + fn: T, + signal: AbortSignal, +): T { + return (function runWithCancel(...args: T extends (...args: infer A) => any ? A : never) { + return new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason)) + + try { + const result = fn(...args) as PromiseLike + + if (typeof result === 'object' && result != null && typeof result.then === 'function') { + result.then(resolve, reject) + } + else { + resolve(result) + } + } + catch (error) { + reject(error) + } + }) + }) as T +} + const abortControllers = new WeakMap() export function abortIfTimeout([context]: [TestContext?], error: Error): void { diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 21fcad7aacc3..99c19f6bde70 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -792,6 +792,12 @@ function failTask(result: TaskResult, err: unknown, diffOptions: DiffOptions | u return } + if (err instanceof TestRunAbortError) { + result.state = 'skip' + result.note = err.message + return + } + result.state = 'fail' const errors = Array.isArray(err) ? err : [err] for (const e of errors) { @@ -814,6 +820,20 @@ function markTasksAsSkipped(suite: Suite, runner: VitestRunner) { }) } +function markPendingTasksAsSkipped(suite: Suite, runner: VitestRunner, note?: string) { + suite.tasks.forEach((t) => { + if (!t.result || t.result.state === 'run') { + t.mode = 'skip' + t.result = { ...t.result, state: 'skip', note } + updateTask('test-cancel', t, runner) + } + + if (t.type === 'suite') { + markPendingTasksAsSkipped(t, runner, note) + } + }) +} + export async function runSuite(suite: Suite, runner: VitestRunner): Promise { await runner.onBeforeRunSuite?.(suite) @@ -1028,8 +1048,10 @@ export async function startTests(specs: string[] | FileSpecification[], runner: runner.cancel = (reason) => { // We intentionally create only one error since there is only one test run that can be cancelled const error = new TestRunAbortError('The test run was aborted by the user.', reason) - getRunningTests().forEach(test => - abortContextSignal(test.context, error), + getRunningTests().forEach((test) => { + abortContextSignal(test.context, error) + markPendingTasksAsSkipped(test.file, runner, error.message) + }, ) return cancel?.(reason) } diff --git a/packages/runner/src/suite.ts b/packages/runner/src/suite.ts index c920fe1f0d56..35ff9899f167 100644 --- a/packages/runner/src/suite.ts +++ b/packages/runner/src/suite.ts @@ -32,6 +32,7 @@ import { collectTask, createTestContext, runWithSuite, + withCancel, withTimeout, } from './context' import { configureProps, TestFixtures, withFixtures } from './fixture' @@ -412,7 +413,7 @@ function createSuiteCollector( setFn( task, withTimeout( - withAwaitAsyncAssertions(withFixtures(handler, { context }), task), + withCancel(withAwaitAsyncAssertions(withFixtures(handler, { context }), task), task.context.signal), timeout, false, stackTraceError, diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index c1f9f4c2b85c..db5eb0b61e74 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -259,6 +259,7 @@ export type TaskUpdateEvent | 'test-prepare' | 'test-finished' | 'test-retried' + | 'test-cancel' | 'suite-prepare' | 'suite-finished' | 'before-hook-start' diff --git a/packages/vitest/src/node/test-run.ts b/packages/vitest/src/node/test-run.ts index cef2910b122d..e2570bd6ef53 100644 --- a/packages/vitest/src/node/test-run.ts +++ b/packages/vitest/src/node/test-run.ts @@ -232,6 +232,11 @@ export class TestRun { return } + if (event === 'test-cancel' && entity.type === 'test') { + // This is used to just update state of the task + return + } + if (event === 'test-prepare' && entity.type === 'test') { return await this.vitest.report('onTestCaseReady', entity) } diff --git a/test/cli/fixtures/cancel-run/blocked-test-cases.test.ts b/test/cli/fixtures/cancel-run/blocked-test-cases.test.ts new file mode 100644 index 000000000000..24f9aad45f9e --- /dev/null +++ b/test/cli/fixtures/cancel-run/blocked-test-cases.test.ts @@ -0,0 +1,26 @@ +import { afterEach, describe, test } from 'vitest' + +afterEach(async (context) => { + (context.task.meta as any).afterEachDone = true +}) + +describe('these should pass', () => { + test('one', async () => {}) + test('two', async () => {}) +}) + +test('this test starts and gets cancelled, its after each should be called', async ({ annotate }) => { + await annotate('Running long test, do the cancelling now!') + + await new Promise(resolve => setTimeout(resolve, 100_000)) +}) + +describe('these should not start but should be skipped', () => { + test('third, no after each expected', async () => {}) + + describe("nested", () => { + test('fourth, no after each expected', async () => {}) + }); +}) + +test('fifth, no after each expected', async () => {}) \ No newline at end of file diff --git a/test/cli/fixtures/cancel-run/blocked-thread.test.ts b/test/cli/fixtures/cancel-run/blocked-thread.test.ts new file mode 100644 index 000000000000..ca923b2ced26 --- /dev/null +++ b/test/cli/fixtures/cancel-run/blocked-thread.test.ts @@ -0,0 +1,7 @@ +import { execSync } from 'node:child_process' +import { test } from 'vitest' + +test('block whole test runner thread/process', { timeout: 30_000 }, async () => { + // Note that this can also block the RPC before onTestCaseReady is emitted to main thread + execSync("sleep 40") +}) diff --git a/test/cli/fixtures/cancel-run/slow-timeouting.test.ts b/test/cli/fixtures/cancel-run/slow-timeouting.test.ts deleted file mode 100644 index b6d5a924ef87..000000000000 --- a/test/cli/fixtures/cancel-run/slow-timeouting.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { test } from 'vitest' - -test('slow timeouting test', { timeout: 30_000 }, async () => { - console.log("Running slow timeouting test") - await new Promise(resolve => setTimeout(resolve, 40_000)) -}) diff --git a/test/cli/test/cancel-run.test.ts b/test/cli/test/cancel-run.test.ts index f8911c173c69..c8fa223fd7f0 100644 --- a/test/cli/test/cancel-run.test.ts +++ b/test/cli/test/cancel-run.test.ts @@ -1,10 +1,13 @@ +import type { TestModule } from 'vitest/node' import { Readable, Writable } from 'node:stream' import { stripVTControlCharacters } from 'node:util' import { createDefer } from '@vitest/utils/helpers' import { expect, onTestFinished, test, vi } from 'vitest' import { createVitest, registerConsoleShortcuts } from 'vitest/node' -test('can force cancel a run', async () => { +const CTRL_C = '\x03' + +test('can force cancel a run via CLI', async () => { const onExit = vi.fn() const exit = process.exit onTestFinished(() => { @@ -12,10 +15,11 @@ test('can force cancel a run', async () => { }) process.exit = onExit - const onTestCaseReady = createDefer() + const onTestModuleStart = createDefer() const vitest = await createVitest('test', { root: 'fixtures/cancel-run', - reporters: [{ onTestCaseReady: () => onTestCaseReady.resolve() }], + include: ['blocked-thread.test.ts'], + reporters: [{ onTestModuleStart: () => onTestModuleStart.resolve() }], }) onTestFinished(() => vitest.close()) @@ -27,17 +31,130 @@ test('can force cancel a run', async () => { const onLog = vi.spyOn(vitest.logger, 'log').mockImplementation(() => {}) const promise = vitest.start() - await onTestCaseReady + await onTestModuleStart // First CTRL+c should log warning about graceful exit - stdin.emit('data', '\x03') + stdin.emit('data', CTRL_C) + + // Let the test case start running + await new Promise(resolve => setTimeout(resolve, 100)) const logs = onLog.mock.calls.map(log => stripVTControlCharacters(log[0] || '').trim()) expect(logs).toContain('Cancelling test run. Press CTRL+c again to exit forcefully.') // Second CTRL+c should stop run - stdin.emit('data', '\x03') + stdin.emit('data', CTRL_C) await promise expect(onExit).toHaveBeenCalled() }) + +test('cancelling test run stops test execution immediately', async () => { + const onTestRunEnd = createDefer() + const onSlowTestRunning = createDefer() + const onTestCaseHooks: string[] = [] + + const vitest = await createVitest('test', { + root: 'fixtures/cancel-run', + include: ['blocked-test-cases.test.ts'], + reporters: [{ + onTestCaseReady(testCase) { + onTestCaseHooks.push(`onTestCaseReady ${testCase.name}`) + }, + onTestCaseResult(testCase) { + onTestCaseHooks.push(`onTestCaseResult ${testCase.name}`) + onTestCaseHooks.push('') // padding + }, + onTestCaseAnnotate: (_, annotation) => { + if (annotation.message === 'Running long test, do the cancelling now!') { + onSlowTestRunning.resolve() + } + }, + onTestRunEnd(testModules) { + onTestRunEnd.resolve(testModules) + }, + }], + }) + onTestFinished(() => vitest.close()) + + const promise = vitest.start() + + await onSlowTestRunning + await vitest.cancelCurrentRun('keyboard-input') + + const testModules = await onTestRunEnd + await Promise.all([vitest.close(), promise]) + + expect(testModules).toHaveLength(1) + + const tests = Array.from(testModules[0].children.allTests()).map(test => ({ + name: test.name, + status: test.result().state, + note: (test.result() as any).note, + afterEachRun: (test.meta() as any).afterEachDone === true, + })) + + expect(tests).toMatchInlineSnapshot(` + [ + { + "afterEachRun": true, + "name": "one", + "note": undefined, + "status": "passed", + }, + { + "afterEachRun": true, + "name": "two", + "note": undefined, + "status": "passed", + }, + { + "afterEachRun": true, + "name": "this test starts and gets cancelled, its after each should be called", + "note": "The test run was aborted by the user.", + "status": "skipped", + }, + { + "afterEachRun": false, + "name": "third, no after each expected", + "note": "The test run was aborted by the user.", + "status": "skipped", + }, + { + "afterEachRun": false, + "name": "fourth, no after each expected", + "note": "The test run was aborted by the user.", + "status": "skipped", + }, + { + "afterEachRun": false, + "name": "fifth, no after each expected", + "note": "The test run was aborted by the user.", + "status": "skipped", + }, + ] + `) + + expect(onTestCaseHooks).toMatchInlineSnapshot(` + [ + "onTestCaseReady one", + "onTestCaseResult one", + "", + "onTestCaseReady two", + "onTestCaseResult two", + "", + "onTestCaseReady this test starts and gets cancelled, its after each should be called", + "onTestCaseResult this test starts and gets cancelled, its after each should be called", + "", + "onTestCaseReady third, no after each expected", + "onTestCaseResult third, no after each expected", + "", + "onTestCaseReady fourth, no after each expected", + "onTestCaseResult fourth, no after each expected", + "", + "onTestCaseReady fifth, no after each expected", + "onTestCaseResult fifth, no after each expected", + "", + ] + `) +})