diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 541a547bc157c..9f577cad9caf8 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -19,6 +19,7 @@ test('basic test', async ({ page }, testInfo) => { - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - `description` ?<[string]> Optional description. + - `location` ?<[Location]> Optional location in the source where the annotation is added. The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index a3c9c2db995c6..22a8588fb0934 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -9,6 +9,7 @@ - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - `description` ?<[string]> Optional description. + - `location` ?<[Location]> Optional location in the source where the annotation is added. [`property: TestResult.annotations`] of the last test run. diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 263b299c49ee1..ef41200b40c27 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -19,6 +19,7 @@ The list of files or buffers attached during the test execution through [`proper - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. - `description` ?<[string]> Optional description. + - `location` ?<[Location]> Optional location in the source where the annotation is added. The list of annotations applicable to the current test. Includes: * annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`]; diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index a8d1c12110bcc..84ba3abbda7f1 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -58,6 +58,7 @@ List of steps inside this step. - type: <[Array]<[Object]>> - `type` <[string]> Annotation type, for example `'skip'`. - `description` ?<[string]> Optional description. + - `location` ?<[Location]> Optional location in the source where the annotation is added. The list of annotations applicable to the current test step. diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 19ffb9fb6189b..b7ab5648f2516 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -102,7 +102,7 @@ export class TestTypeImpl { details = fnOrDetails; } - const validatedDetails = validateTestDetails(details); + const validatedDetails = validateTestDetails(details, location); const test = new TestCase(title, body, this, location); test._requireFile = suite._requireFile; test.annotations.push(...validatedDetails.annotations); @@ -112,9 +112,9 @@ export class TestTypeImpl { if (type === 'only' || type === 'fail.only') test._only = true; if (type === 'skip' || type === 'fixme' || type === 'fail') - test.annotations.push({ type }); + test.annotations.push({ type, location }); else if (type === 'fail.only') - test.annotations.push({ type: 'fail' }); + test.annotations.push({ type: 'fail', location }); } private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) { @@ -141,7 +141,7 @@ export class TestTypeImpl { body = fn!; } - const validatedDetails = validateTestDetails(details); + const validatedDetails = validateTestDetails(details, location); const child = new Suite(title, 'describe'); child._requireFile = suite._requireFile; child.location = location; @@ -156,7 +156,7 @@ export class TestTypeImpl { if (type === 'parallel' || type === 'parallel.only') child._parallelMode = 'parallel'; if (type === 'skip' || type === 'fixme') - child._staticAnnotations.push({ type }); + child._staticAnnotations.push({ type, location }); for (let parent: Suite | undefined = suite; parent; parent = parent.parent) { if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel') @@ -227,7 +227,7 @@ export class TestTypeImpl { if (modifierArgs.length >= 1 && !modifierArgs[0]) return; const description = modifierArgs[1]; - suite._staticAnnotations.push({ type, description }); + suite._staticAnnotations.push({ type, description, location }); } return; } @@ -237,7 +237,7 @@ export class TestTypeImpl { throw new Error(`test.${type}() can only be called inside test, describe block or fixture`); if (typeof modifierArgs[0] === 'function') throw new Error(`test.${type}() with a function can only be called inside describe block`); - testInfo[type](...modifierArgs as [any, any]); + testInfo._modifier(type, location, modifierArgs as [any, any]); } private _setTimeout(location: Location, timeout: number) { @@ -270,7 +270,7 @@ export class TestTypeImpl { let result: Awaited>> | undefined = undefined; result = await raceAgainstDeadline(async () => { try { - return await step.info._runStepBody(expectation === 'skip', body); + return await step.info._runStepBody(expectation === 'skip', body, step.location); } catch (e) { // If the step timed out, the test fixtures will tear down, which in turn // will abort unfinished actions in the step body. Record such errors here. @@ -309,8 +309,9 @@ function throwIfRunningInsideJest() { } } -function validateTestDetails(details: TestDetails) { - const annotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []); +function validateTestDetails(details: TestDetails, location: Location) { + const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []); + const annotations = originalAnnotations.map(annotation => ({ ...annotation, location })); const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []); for (const tag of tags) { if (tag[0] !== '@') diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 5f0ac0f693904..df3e1c84a4ded 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -355,8 +355,9 @@ export class TeleReporterReceiver { if (!!payload.attachments) result.attachments = this._parseAttachments(payload.attachments); if (payload.annotations) { + this._absoluteAnnotationLocationsInplace(payload.annotations); result.annotations = payload.annotations; - test.annotations = result.annotations; + test.annotations = payload.annotations; } this._reporter.onTestEnd?.(test, result); // Free up the memory as won't see these step ids. @@ -499,9 +500,17 @@ export class TeleReporterReceiver { test.retries = payload.retries; test.tags = payload.tags ?? []; test.annotations = payload.annotations ?? []; + this._absoluteAnnotationLocationsInplace(test.annotations); return test; } + private _absoluteAnnotationLocationsInplace(annotations: TestAnnotation[]) { + for (const annotation of annotations) { + if (annotation.location) + annotation.location = this._absoluteLocation(annotation.location); + } + } + private _absoluteLocation(location: reporterTypes.Location): reporterTypes.Location; private _absoluteLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined; private _absoluteLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined { diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 1b6ff2f2ad98f..9c642f5de46a1 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -499,7 +499,15 @@ class HtmlBuilder { private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] { // Annotations can be pushed directly, with a wrong type. - return annotations.map(a => ({ type: a.type, description: a.description === undefined ? undefined : String(a.description) })); + return annotations.map(a => ({ + type: a.type, + description: a.description === undefined ? undefined : String(a.description), + location: a.location ? { + file: a.location.file, + line: a.location.line, + column: a.location.column, + } : undefined, + })); } private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult { diff --git a/packages/playwright/src/reporters/merge.ts b/packages/playwright/src/reporters/merge.ts index 10f12a5a54ca0..719cf3c567a19 100644 --- a/packages/playwright/src/reporters/merge.ts +++ b/packages/playwright/src/reporters/merge.ts @@ -26,7 +26,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { createReporters } from '../runner/reporters'; import { relativeFilePath } from '../util'; -import type { ReporterDescription } from '../../types/test'; +import type { ReporterDescription, TestAnnotation } from '../../types/test'; import type { TestError } from '../../types/testReporter'; import type { FullConfigInternal } from '../common/config'; import type { BlobReportMetadata, JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonOnConfigureEvent, JsonOnEndEvent, JsonOnProjectEvent, JsonProject, JsonSuite, JsonTestCase } from '../isomorphic/teleReceiver'; @@ -484,7 +484,10 @@ class PathSeparatorPatcher { return; } if (jsonEvent.method === 'onTestEnd') { + const test = jsonEvent.params.test; + test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); const testResult = jsonEvent.params.result; + testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); testResult.errors.forEach(error => this._updateErrorLocations(error)); (testResult.attachments ?? []).forEach(attachment => { if (attachment.path) @@ -500,6 +503,7 @@ class PathSeparatorPatcher { if (jsonEvent.method === 'onStepEnd') { const step = jsonEvent.params.step; this._updateErrorLocations(step.error); + step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); return; } if (jsonEvent.method === 'onAttach') { @@ -524,10 +528,12 @@ class PathSeparatorPatcher { if (isFileSuite) suite.title = this._updatePath(suite.title); for (const entry of suite.entries) { - if ('testId' in entry) + if ('testId' in entry) { this._updateLocation(entry.location); - else + entry.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation)); + } else { this._updateSuite(entry); + } } } @@ -538,6 +544,10 @@ class PathSeparatorPatcher { } } + private _updateAnnotationLocation(annotation: TestAnnotation) { + this._updateLocation(annotation.location); + } + private _updateLocation(location?: JsonLocation) { if (location) location.file = this._updatePath(location.file); diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index b403cab303960..44fbcec7230d7 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -22,6 +22,7 @@ import { serializeRegexPatterns } from '../isomorphic/teleReceiver'; import type { ReporterV2 } from './reporterV2'; import type * as reporterTypes from '../../types/testReporter'; +import type { TestAnnotation } from '../../types/test'; import type * as teleReceiver from '../isomorphic/teleReceiver'; export type TeleReporterEmitterOptions = { @@ -224,7 +225,7 @@ export class TeleReporterEmitter implements ReporterV2 { retries: test.retries, tags: test.tags, repeatEachIndex: test.repeatEachIndex, - annotations: test.annotations, + annotations: this._relativeAnnotationLocations(test.annotations), }; } @@ -244,7 +245,7 @@ export class TeleReporterEmitter implements ReporterV2 { duration: result.duration, status: result.status, errors: result.errors, - annotations: result.annotations?.length ? result.annotations : undefined, + annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined, }; } @@ -294,10 +295,17 @@ export class TeleReporterEmitter implements ReporterV2 { duration: step.duration, error: step.error, attachments: step.attachments.length ? step.attachments.map(a => result.attachments.indexOf(a)) : undefined, - annotations: step.annotations.length ? step.annotations : undefined, + annotations: step.annotations.length ? this._relativeAnnotationLocations(step.annotations) : undefined, }; } + private _relativeAnnotationLocations(annotations: TestAnnotation[]): TestAnnotation[] { + return annotations.map(annotation => ({ + ...annotation, + location: annotation.location ? this._relativeLocation(annotation.location) : undefined, + })); + } + private _relativeLocation(location: reporterTypes.Location): reporterTypes.Location; private _relativeLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined; private _relativeLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined { diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 0a7736d3a649f..19a2fa5071281 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -23,6 +23,7 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util'; import { TestTracing } from './testTracing'; import { testInfoError } from './util'; +import { wrapFunctionWithLocation } from '../transform/transform'; import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; @@ -80,6 +81,12 @@ export class TestInfoImpl implements TestInfo { _hasUnhandledError = false; _allowSkips = false; + // ------------ Main methods ------------ + skip: (arg?: any, description?: string) => void; + fixme: (arg?: any, description?: string) => void; + fail: (arg?: any, description?: string) => void; + slow: (arg?: any, description?: string) => void; + // ------------ TestInfo fields ------------ readonly testId: string; readonly repeatEachIndex: number; @@ -205,9 +212,14 @@ export class TestInfoImpl implements TestInfo { }; this._tracing = new TestTracing(this, workerParams.artifactsDir); + + this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args)); + this.fixme = wrapFunctionWithLocation((location, ...args) => this._modifier('fixme', location, args)); + this.fail = wrapFunctionWithLocation((location, ...args) => this._modifier('fail', location, args)); + this.slow = wrapFunctionWithLocation((location, ...args) => this._modifier('slow', location, args)); } - private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) { + _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, modifierArgs: [arg?: any, description?: string]) { if (typeof modifierArgs[1] === 'function') { throw new Error([ 'It looks like you are calling test.skip() inside the test and pass a callback.', @@ -222,7 +234,7 @@ export class TestInfoImpl implements TestInfo { return; const description = modifierArgs[1]; - this.annotations.push({ type, description }); + this.annotations.push({ type, description, location }); if (type === 'slow') { this._timeoutManager.slow(); } else if (type === 'skip' || type === 'fixme') { @@ -567,22 +579,6 @@ export class TestInfoImpl implements TestInfo { return this._resolveSnapshotPaths(kind, name.length <= 1 ? name[0] : name, 'dontUpdateSnapshotIndex').absoluteSnapshotPath; } - skip(...args: [arg?: any, description?: string]) { - this._modifier('skip', args); - } - - fixme(...args: [arg?: any, description?: string]) { - this._modifier('fixme', args); - } - - fail(...args: [arg?: any, description?: string]) { - this._modifier('fail', args); - } - - slow(...args: [arg?: any, description?: string]) { - this._modifier('slow', args); - } - setTimeout(timeout: number) { this._timeoutManager.setTimeout(timeout); } @@ -594,14 +590,25 @@ export class TestStepInfoImpl implements TestStepInfo { private _testInfo: TestInfoImpl; private _stepId: string; + skip: (arg?: any, description?: string) => void; + constructor(testInfo: TestInfoImpl, stepId: string) { this._testInfo = testInfo; this._stepId = stepId; + this.skip = wrapFunctionWithLocation((location: Location, ...args: unknown[]) => { + // skip(); + // skip(condition: boolean, description: string); + if (args.length > 0 && !args[0]) + return; + const description = args[1] as (string|undefined); + this.annotations.push({ type: 'skip', description, location }); + throw new StepSkipError(description); + }); } - async _runStepBody(skip: boolean, body: (step: TestStepInfo) => T | Promise) { + async _runStepBody(skip: boolean, body: (step: TestStepInfo) => T | Promise, location?: Location) { if (skip) { - this.annotations.push({ type: 'skip' }); + this.annotations.push({ type: 'skip', location }); return undefined as T; } try { @@ -620,16 +627,6 @@ export class TestStepInfoImpl implements TestStepInfo { async attach(name: string, options?: { body?: string | Buffer; contentType?: string; path?: string; }): Promise { this._attachToStep(await normalizeAndSaveAttachment(this._testInfo.outputPath(), name, options)); } - - skip(...args: unknown[]) { - // skip(); - // skip(condition: boolean, description: string); - if (args.length > 0 && !args[0]) - return; - const description = args[1] as (string|undefined); - this.annotations.push({ type: 'skip', description }); - throw new StepSkipError(description); - } } export class TestSkipError extends Error { diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index 59dbae0f4fdc8..e9f515a76083d 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -494,7 +494,7 @@ export class WorkerMain extends ProcessRunner { continue; const fn = async (fixtures: any) => { const result = await modifier.fn(fixtures); - testInfo[modifier.type](!!result, modifier.description); + testInfo._modifier(modifier.type, modifier.location, [!!result, modifier.description]); }; inheritFixtureNames(modifier.fn, fn); runnables.push({ diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 0f63922e2eeba..1eec00ea64bff 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -2325,6 +2325,11 @@ export interface TestInfo { * Optional description. */ description?: string; + + /** + * Optional location in the source where the annotation is added. + */ + location?: Location; }>; /** @@ -2575,7 +2580,9 @@ export type TestDetailsAnnotation = { description?: string; }; -export type TestAnnotation = TestDetailsAnnotation; +export type TestAnnotation = TestDetailsAnnotation & { + location?: Location; +}; export type TestDetails = { tag?: string | string[]; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index bbf22d3bc64a9..d03244a1ec7b4 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -451,6 +451,11 @@ export interface TestCase { * Optional description. */ description?: string; + + /** + * Optional location in the source where the annotation is added. + */ + location?: Location; }>; /** @@ -607,6 +612,11 @@ export interface TestResult { * Optional description. */ description?: string; + + /** + * Optional location in the source where the annotation is added. + */ + location?: Location; }>; /** @@ -722,6 +732,11 @@ export interface TestStep { * Optional description. */ description?: string; + + /** + * Optional location in the source where the annotation is added. + */ + location?: Location; }>; /** diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts index ab5d3e45840e8..4da80d8de056d 100644 --- a/tests/playwright-test/access-data.spec.ts +++ b/tests/playwright-test/access-data.spec.ts @@ -61,7 +61,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => { expect(exitCode).toBe(0); const test = report.suites[0].specs[0].tests[0]; expect(test.annotations).toEqual([ - { type: 'slow', description: 'just slow' }, + { type: 'slow', description: 'just slow', location: { file: expect.any(String), line: 10, column: 14 } }, { type: 'myname', description: 'hello' } ]); expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 26cd4082db2ae..de7c1a10e44f0 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1758,6 +1758,9 @@ test('merge reports with different rootDirs and path separators', async ({ runIn console.log('test:', test.location.file); console.log('test title:', test.titlePath()[2]); } + onTestEnd(test) { + console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(',')); + } }; `, 'merge.config.ts': `module.exports = { @@ -1769,7 +1772,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn };`, 'dir1/tests1/a.test.js': ` import { test, expect } from '@playwright/test'; - test('math 1', async ({}) => { }); + test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { }); `, }; await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] }); @@ -1780,7 +1783,7 @@ test('merge reports with different rootDirs and path separators', async ({ runIn };`, 'dir2/tests2/b.test.js': ` import { test, expect } from '@playwright/test'; - test('math 2', async ({}) => { }); + test('math 2', { annotation: { type: 'issue' } }, async ({}) => { }); `, }; await runInlineTest(files2, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir2/playwright.config.ts')] }); @@ -1802,12 +1805,16 @@ test('merge reports with different rootDirs and path separators', async ({ runIn { const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--config', 'merge.config.ts'] }); + const testPath1 = test.info().outputPath('mergeRoot', 'tests1', 'a.test.js'); + const testPath2 = test.info().outputPath('mergeRoot', 'tests2', 'b.test.js'); expect(exitCode).toBe(0); expect(output).toContain(`rootDir: ${test.info().outputPath('mergeRoot')}`); - expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests1', 'a.test.js')}`); + expect(output).toContain(`test: ${testPath1}`); expect(output).toContain(`test title: ${'tests1' + path.sep + 'a.test.js'}`); - expect(output).toContain(`test: ${test.info().outputPath('mergeRoot', 'tests2', 'b.test.js')}`); + expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`); + expect(output).toContain(`test: ${testPath2}`); expect(output).toContain(`test title: ${'tests2' + path.sep + 'b.test.js'}`); + expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`); } }); @@ -1823,6 +1830,9 @@ test('merge reports without --config preserves path separators', async ({ runInl console.log('test:', test.location.file); console.log('test title:', test.titlePath()[2]); } + onTestEnd(test) { + console.log('annotations:', test.annotations.map(a => 'type: ' + a.type + ', description: ' + a.description + ', file: ' + a.location.file).join(',')); + } }; `, 'dir1/playwright.config.ts': `module.exports = { @@ -1830,11 +1840,11 @@ test('merge reports without --config preserves path separators', async ({ runInl };`, 'dir1/tests1/a.test.js': ` import { test, expect } from '@playwright/test'; - test('math 1', async ({}) => { }); + test('math 1', { annotation: { type: 'warning', description: 'Some warning' } }, async ({}) => { }); `, 'dir1/tests2/b.test.js': ` import { test, expect } from '@playwright/test'; - test('math 2', async ({}) => { }); + test('math 2', { annotation: { type: 'issue' } }, async ({}) => { }); `, }; await runInlineTest(files1, { workers: 1 }, undefined, { additionalArgs: ['--config', test.info().outputPath('dir1/playwright.config.ts')] }); @@ -1854,11 +1864,15 @@ test('merge reports without --config preserves path separators', async ({ runInl const { exitCode, output } = await mergeReports(allReportsDir, undefined, { additionalArgs: ['--reporter', './echo-reporter.js'] }); expect(exitCode).toBe(0); const otherSeparator = path.sep === '/' ? '\\' : '/'; + const testPath1 = test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator); + const testPath2 = test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator); expect(output).toContain(`rootDir: ${test.info().outputPath('dir1').replaceAll(path.sep, otherSeparator)}`); - expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests1', 'a.test.js').replaceAll(path.sep, otherSeparator)}`); + expect(output).toContain(`test: ${testPath1}`); expect(output).toContain(`test title: ${'tests1' + otherSeparator + 'a.test.js'}`); - expect(output).toContain(`test: ${test.info().outputPath('dir1', 'tests2', 'b.test.js').replaceAll(path.sep, otherSeparator)}`); + expect(output).toContain(`annotations: type: warning, description: Some warning, file: ${testPath1}`); + expect(output).toContain(`test: ${testPath2}`); expect(output).toContain(`test title: ${'tests2' + otherSeparator + 'b.test.js'}`); + expect(output).toContain(`annotations: type: issue, description: undefined, file: ${testPath2}`); }); test('merge reports should preserve attachments', async ({ runInlineTest, mergeReports, showReport, page }) => { diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 1e28e8b288416..afe152ff7e2aa 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -584,7 +584,9 @@ test('should report annotations from test declaration', async ({ runInlineTest } const visit = suite => { for (const test of suite.tests || []) { const annotations = test.annotations.map(a => { - return a.description ? a.type + '=' + a.description : a.type; + const description = a.description ? a.type + '=' + a.description : a.type; + const location = a.location ? '(' + a.location.line + ':' + a.location.column + ')' : ''; + return description + location; }); console.log('\\n%%title=' + test.title + ', annotations=' + annotations.join(',')); } @@ -609,7 +611,7 @@ test('should report annotations from test declaration', async ({ runInlineTest } expect(test.info().annotations).toEqual([]); }); test('foo', { annotation: { type: 'foo' } }, () => { - expect(test.info().annotations).toEqual([{ type: 'foo' }]); + expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 6, column: 11 } }]); }); test('foo-bar', { annotation: [ @@ -618,8 +620,8 @@ test('should report annotations from test declaration', async ({ runInlineTest } ], }, () => { expect(test.info().annotations).toEqual([ - { type: 'foo', description: 'desc' }, - { type: 'bar' }, + { type: 'foo', description: 'desc', location: { file: expect.any(String), line: 9, column: 11 } }, + { type: 'bar', location: { file: expect.any(String), line: 9, column: 11 } }, ]); }); test.skip('skip-foo', { annotation: { type: 'foo' } }, () => { @@ -636,11 +638,14 @@ test('should report annotations from test declaration', async ({ runInlineTest } }); test.describe('suite', { annotation: { type: 'foo' } }, () => { test('foo-suite', () => { - expect(test.info().annotations).toEqual([{ type: 'foo' }]); + expect(test.info().annotations).toEqual([{ type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } }]); }); test.describe('inner', { annotation: { type: 'bar' } }, () => { test('foo-bar-suite', () => { - expect(test.info().annotations).toEqual([{ type: 'foo' }, { type: 'bar' }]); + expect(test.info().annotations).toEqual([ + { type: 'foo', location: { file: expect.any(String), line: 32, column: 12 } }, + { type: 'bar', location: { file: expect.any(String), line: 36, column: 14 } } + ]); }); }); }); @@ -657,15 +662,15 @@ test('should report annotations from test declaration', async ({ runInlineTest } expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ `title=none, annotations=`, - `title=foo, annotations=foo`, - `title=foo-bar, annotations=foo=desc,bar`, - `title=skip-foo, annotations=foo,skip`, - `title=fixme-bar, annotations=bar,fixme`, - `title=fail-foo-bar, annotations=foo,bar=desc,fail`, - `title=foo-suite, annotations=foo`, - `title=foo-bar-suite, annotations=foo,bar`, - `title=skip-foo-suite, annotations=foo,skip`, - `title=fixme-bar-suite, annotations=bar,fixme`, + `title=foo, annotations=foo(6:11)`, + `title=foo-bar, annotations=foo=desc(9:11),bar(9:11)`, + `title=skip-foo, annotations=foo(20:12),skip(20:12)`, + `title=fixme-bar, annotations=bar(22:12),fixme(22:12)`, + `title=fail-foo-bar, annotations=foo(24:12),bar=desc(24:12),fail(24:12)`, + `title=foo-suite, annotations=foo(32:12)`, + `title=foo-bar-suite, annotations=foo(32:12),bar(36:14)`, + `title=skip-foo-suite, annotations=foo(45:21),skip(45:21)`, + `title=fixme-bar-suite, annotations=bar(49:21),fixme(49:21)`, ]); }); diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index b5dfc7a347f79..c82a61b20fb8d 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline expect(result.failed).toBe(0); expect(result.flaky).toBe(1); expect(result.output).toContain('Failed on first run'); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry', location: expect.anything() }]); }); diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts index e992fae2e78ae..e73e483b278c4 100644 --- a/tests/playwright-test/test-modifiers.spec.ts +++ b/tests/playwright-test/test-modifiers.spec.ts @@ -113,19 +113,19 @@ test('test modifiers should work', async ({ runInlineTest }) => { expectTest('passed3', 'passed', 'passed', []); expectTest('passed4', 'passed', 'passed', []); expectTest('passed5', 'passed', 'passed', []); - expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip' }]); - expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip' }]); - expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip' }]); - expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]); - expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme' }]); - expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason' }]); - expectTest('failed1', 'failed', 'failed', [{ type: 'fail' }]); - expectTest('failed2', 'failed', 'failed', [{ type: 'fail' }]); - expectTest('failed3', 'failed', 'failed', [{ type: 'fail' }]); - expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason' }]); - expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip' }]); - expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip' }]); - expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason' }]); + expectTest('skipped1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 20, column: 14 } }]); + expectTest('skipped2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 23, column: 14 } }]); + expectTest('skipped3', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 26, column: 14 } }]); + expectTest('skipped4', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 29, column: 14 } }]); + expectTest('skipped5', 'skipped', 'skipped', [{ type: 'fixme', location: { file: expect.any(String), line: 32, column: 14 } }]); + expectTest('skipped6', 'skipped', 'skipped', [{ type: 'fixme', description: 'reason', location: { file: expect.any(String), line: 35, column: 14 } }]); + expectTest('failed1', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 39, column: 14 } }]); + expectTest('failed2', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 43, column: 14 } }]); + expectTest('failed3', 'failed', 'failed', [{ type: 'fail', location: { file: expect.any(String), line: 47, column: 14 } }]); + expectTest('failed4', 'failed', 'failed', [{ type: 'fail', description: 'reason', location: { file: expect.any(String), line: 51, column: 14 } }]); + expectTest('suite1', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 56, column: 14 } }]); + expectTest('suite2', 'skipped', 'skipped', [{ type: 'skip', location: { file: expect.any(String), line: 61, column: 14 } }]); + expectTest('suite3', 'skipped', 'skipped', [{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 66, column: 14 } }]); expectTest('suite4', 'passed', 'passed', []); expect(result.passed).toBe(10); expect(result.skipped).toBe(9); @@ -407,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => { }); expect(result.exitCode).toBe(0); expect(result.skipped).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 20 } }]); }); test('modifier with a function should throw in the test', async ({ runInlineTest }) => { @@ -460,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', a expect(result.passed).toBe(1); expect(result.skipped).toBe(2); expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]); - expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); - expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]); + expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 14, column: 14 } }]); expect(result.outputLines).toEqual([ 'beforeEach', 'passed', @@ -493,8 +493,8 @@ test('test.skip without a callback in describe block should skip hooks', async ( }); expect(result.exitCode).toBe(0); expect(result.skipped).toBe(2); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); - expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]); + expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 10, column: 12 } }]); expect(result.output).not.toContain('%%'); }); @@ -598,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => { 'beforeAll', 'afterAll', ]); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); - expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]); + expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason', location: { file: expect.any(String), line: 5, column: 14 } }]); }); test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => { @@ -695,9 +695,9 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest } expect(result.passed).toBe(0); expect(result.skipped).toBe(2); expect(result.didNotRun).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]); - expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]); - expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow', location: { file: expect.any(String), line: 6, column: 14 } }]); + expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme', location: { file: expect.any(String), line: 9, column: 12 } }]); + expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip', location: { file: expect.any(String), line: 11, column: 12 } }]); expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]); }); @@ -721,9 +721,18 @@ test('should contain only one slow modifier', async ({ runInlineTest }) => { }); expect(result.exitCode).toBe(0); expect(result.passed).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'fixme' }, { type: 'issue', description: 'my-value' }]); - expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([{ type: 'skip' }, { type: 'issue', description: 'my-value' }]); - expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }, { type: 'issue', description: 'my-value' }]); + expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([ + { type: 'fixme', location: { file: expect.any(String), line: 3, column: 12 } }, + { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } } + ]); + expect(result.report.suites[1].specs[0].tests[0].annotations).toEqual([ + { type: 'skip', location: { file: expect.any(String), line: 3, column: 12 } }, + { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } } + ]); + expect(result.report.suites[2].specs[0].tests[0].annotations).toEqual([ + { type: 'slow', location: { file: expect.any(String), line: 3, column: 12 } }, + { type: 'issue', description: 'my-value', location: { file: expect.any(String), line: 4, column: 11 } } + ]); }); test('should skip beforeEach hooks upon modifiers', async ({ runInlineTest }) => { diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 99225d36bbf6e..a152928d87544 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -81,7 +81,9 @@ export type TestDetailsAnnotation = { description?: string; }; -export type TestAnnotation = TestDetailsAnnotation; +export type TestAnnotation = TestDetailsAnnotation & { + location?: Location; +}; export type TestDetails = { tag?: string | string[];