diff --git a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts index 46980d7e4155..16f454daa71f 100644 --- a/packages/browser/src/client/tester/expect/toMatchScreenshot.ts +++ b/packages/browser/src/client/tester/expect/toMatchScreenshot.ts @@ -120,5 +120,8 @@ export default async function toMatchScreenshot( ] .filter(element => element !== null) .join('\n'), + meta: { + outcome: result.outcome, + }, } } diff --git a/packages/browser/src/client/tester/runner.ts b/packages/browser/src/client/tester/runner.ts index 839723c9a52c..faddbcb9f090 100644 --- a/packages/browser/src/client/tester/runner.ts +++ b/packages/browser/src/client/tester/runner.ts @@ -156,14 +156,16 @@ export function createBrowserRunner( } onTaskFinished = async (task: Task) => { + const lastErrorContext = task.result?.errors?.at(-1)?.context if ( this.config.browser.screenshotFailures && document.body.clientHeight > 0 && task.result?.state === 'fail' && task.type === 'test' - && task.artifacts.every( - artifact => artifact.type !== 'internal:toMatchScreenshot', - ) + && !( + lastErrorContext + && Reflect.get(lastErrorContext, 'assertionName') === 'toMatchScreenshot' + && Reflect.get(lastErrorContext, 'meta')?.outcome !== 'unstable-screenshot') ) { const screenshot = await page.screenshot({ timeout: this.config.browser.providerOptions?.actionTimeout ?? 5_000, diff --git a/packages/browser/src/node/commands/screenshotMatcher/index.ts b/packages/browser/src/node/commands/screenshotMatcher/index.ts index 306571c604e7..df466112ccd0 100644 --- a/packages/browser/src/node/commands/screenshotMatcher/index.ts +++ b/packages/browser/src/node/commands/screenshotMatcher/index.ts @@ -273,6 +273,7 @@ function buildOutput( case 'unstable-screenshot': return { pass: false, + outcome: outcome.type, reference: outcome.reference && { path: outcome.reference.path, width: outcome.reference.image.metadata.width, @@ -286,6 +287,7 @@ function buildOutput( case 'missing-reference': { return { pass: false, + outcome: outcome.type, reference: { path: outcome.reference.path, width: outcome.reference.image.metadata.width, @@ -302,11 +304,12 @@ function buildOutput( case 'update-reference': case 'matched-immediately': case 'matched-after-comparison': - return { pass: true } + return { pass: true, outcome: outcome.type } case 'mismatch': return { pass: false, + outcome: outcome.type, reference: { path: outcome.reference.path, width: outcome.reference.image.metadata.width, @@ -333,6 +336,7 @@ function buildOutput( return { pass: false, + outcome: null as never, actual: null, reference: null, diff: null, diff --git a/packages/browser/src/shared/screenshotMatcher/types.ts b/packages/browser/src/shared/screenshotMatcher/types.ts index 2eb25c21d3ab..e95d4e99cab9 100644 --- a/packages/browser/src/shared/screenshotMatcher/types.ts +++ b/packages/browser/src/shared/screenshotMatcher/types.ts @@ -17,6 +17,10 @@ interface ScreenshotData { path: string; width: number; height: number } export type ScreenshotMatcherOutput = Promise< { pass: false + outcome: + | 'unstable-screenshot' + | 'missing-reference' + | 'mismatch' reference: ScreenshotData | null actual: ScreenshotData | null diff: ScreenshotData | null @@ -24,5 +28,9 @@ export type ScreenshotMatcherOutput = Promise< } | { pass: true + outcome: + | 'update-reference' + | 'matched-immediately' + | 'matched-after-comparison' } > diff --git a/packages/expect/src/jest-extend.ts b/packages/expect/src/jest-extend.ts index c1dfbabe7a55..cc5d313177e6 100644 --- a/packages/expect/src/jest-extend.ts +++ b/packages/expect/src/jest-extend.ts @@ -65,7 +65,12 @@ function getMatcherState( } class JestExtendError extends Error { - constructor(message: string, public actual?: any, public expected?: any) { + constructor( + message: string, + public actual?: any, + public expected?: any, + public context?: { assertionName: string; meta?: object }, + ) { super(message) } } @@ -92,23 +97,33 @@ function JestExtendPlugin( && typeof (result as any).then === 'function' ) { const thenable = result as PromiseLike - return thenable.then(({ pass, message, actual, expected }) => { + return thenable.then(({ pass, message, actual, expected, meta }) => { if ((pass && isNot) || (!pass && !isNot)) { const errorMessage = customMessage != null ? customMessage : message() - throw new JestExtendError(errorMessage, actual, expected) + throw new JestExtendError( + errorMessage, + actual, + expected, + { assertionName: expectAssertionName, meta }, + ) } }) } - const { pass, message, actual, expected } = result as SyncExpectationResult + const { pass, message, actual, expected, meta } = result as SyncExpectationResult if ((pass && isNot) || (!pass && !isNot)) { const errorMessage = customMessage != null ? customMessage : message() - throw new JestExtendError(errorMessage, actual, expected) + throw new JestExtendError( + errorMessage, + actual, + expected, + { assertionName: expectAssertionName, meta }, + ) } } diff --git a/packages/expect/src/types.ts b/packages/expect/src/types.ts index 7f84cbe690a2..d49dd6e4d276 100644 --- a/packages/expect/src/types.ts +++ b/packages/expect/src/types.ts @@ -91,6 +91,7 @@ export interface SyncExpectationResult { message: () => string actual?: any expected?: any + meta?: object } export type AsyncExpectationResult = Promise diff --git a/test/browser/specs/failure-screenshot.test.ts b/test/browser/specs/failure-screenshot.test.ts new file mode 100644 index 000000000000..1024828fe714 --- /dev/null +++ b/test/browser/specs/failure-screenshot.test.ts @@ -0,0 +1,100 @@ +import type { TestFsStructure } from '../../test-utils' +import { describe, expect, test } from 'vitest' +import { runInlineTests } from '../../test-utils' +import utilsContent from '../fixtures/expect-dom/utils?raw' +import { instances, provider } from '../settings' + +const testFilename = 'basic.test.ts' + +async function runBrowserTests( + structure: TestFsStructure, +) { + return runInlineTests({ + ...structure, + 'vitest.config.js': ` + import { ${provider.name} } from '@vitest/browser-${provider.name}' + export default { + test: { + browser: { + enabled: true, + screenshotFailures: true, + provider: ${provider.name}(), + ui: false, + headless: true, + instances: ${JSON.stringify(instances.slice(0, 1) /* logic not bound to browser instance */)}, + }, + reporters: ['verbose'], + update: 'new', + }, + }`, + }) +} + +describe('failure screenshots', () => { + describe('`toMatchScreenshot`', () => { + test('usually does NOT produce a failure screenshot', async () => { + const { stderr } = await runBrowserTests( + { + [testFilename]: /* ts */` + import { page } from 'vitest/browser' + import { test } from 'vitest' + import { render } from './utils' + + test('screenshot-initial', async ({ expect }) => { + render('
Test
') + await expect(page.getByTestId('el')).toMatchScreenshot() + }) + `, + 'utils.ts': utilsContent, + }, + ) + + expect(stderr).toContain('No existing reference screenshot found; a new one was created.') + expect(stderr).not.toContain('Failure screenshot:') + }) + + test('unstable screenshot fails produces a failure screenshot', async () => { + const { stderr } = await runBrowserTests( + { + [testFilename]: /* ts */` + import { page } from 'vitest/browser' + import { test } from 'vitest' + import { render } from './utils' + + test('screenshot-unstable', async ({ expect }) => { + render('
Test
') + await expect(page.getByTestId('el')).toMatchScreenshot({ timeout: 1 }) + }) + `, + 'utils.ts': utilsContent, + }, + ) + + expect(stderr).toContain('Could not capture a stable screenshot within 1ms.') + expect(stderr).toContain('Failure screenshot:') + }) + + test('`expect.soft` produces a failure screenshot', async () => { + const { stderr } = await runBrowserTests( + { + [testFilename]: /* ts */` + import { page } from 'vitest/browser' + import { test } from 'vitest' + import { render } from './utils' + + test('screenshot-soft-then-fail', async ({ expect }) => { + render('
Test
') + await expect.soft(page.getByTestId('el')).toMatchScreenshot() + expect(1).toBe(2) + }) + `, + 'utils.ts': utilsContent, + }, + ) + + expect(stderr).toContain('No existing reference screenshot found; a new one was created.') + expect(stderr).toContain('expected 1 to be 2') + expect(stderr).toContain('Failure screenshot:') + }) + }) +})