diff --git a/integration-tests/ci-visibility/playwright-did-not-run/did-not-run.js b/integration-tests/ci-visibility/playwright-did-not-run/did-not-run.js new file mode 100644 index 00000000000..efafbe946ff --- /dev/null +++ b/integration-tests/ci-visibility/playwright-did-not-run/did-not-run.js @@ -0,0 +1,9 @@ +'use strict' + +const { test, expect } = require('@playwright/test') + +test.describe('did not run', () => { + test('because of early bail', async () => { + expect(true).toBe(false) + }) +}) diff --git a/integration-tests/ci-visibility/playwright-did-not-run/fail-test.js b/integration-tests/ci-visibility/playwright-did-not-run/fail-test.js new file mode 100644 index 00000000000..12f2b6e39b1 --- /dev/null +++ b/integration-tests/ci-visibility/playwright-did-not-run/fail-test.js @@ -0,0 +1,9 @@ +'use strict' + +const { test, expect } = require('@playwright/test') + +test.describe('failing test', () => { + test('fails and causes early bail', async () => { + expect(true).toBe(false) + }) +}) diff --git a/integration-tests/playwright.config.js b/integration-tests/playwright.config.js index 9181f653655..ecd1ed748a1 100644 --- a/integration-tests/playwright.config.js +++ b/integration-tests/playwright.config.js @@ -3,6 +3,26 @@ // Playwright config file for integration tests const { devices } = require('@playwright/test') +const projects = [ + { + name: 'chromium', + use: { + ...devices['Desktop Chrome'] + } + } +] + +if (process.env.ADD_EXTRA_PLAYWRIGHT_PROJECT) { + projects.push({ + name: 'extra-project', + use: { + ...devices['Desktop Chrome'], + }, + dependencies: ['chromium'], + testMatch: 'did-not-run.js' + }) +} + const config = { baseURL: process.env.PW_BASE_URL, testDir: process.env.TEST_DIR || './ci-visibility/playwright-tests', @@ -11,14 +31,7 @@ const config = { workers: process.env.PLAYWRIGHT_WORKERS ? Number(process.env.PLAYWRIGHT_WORKERS) : undefined, reporter: 'line', /* Configure projects for major browsers */ - projects: [ - { - name: 'chromium', - use: { - ...devices['Desktop Chrome'] - } - } - ], + projects, testMatch: '**/*-test.js' } diff --git a/integration-tests/playwright/playwright.spec.js b/integration-tests/playwright/playwright.spec.js index 6de688de55f..2c139dc3abd 100644 --- a/integration-tests/playwright/playwright.spec.js +++ b/integration-tests/playwright/playwright.spec.js @@ -2070,5 +2070,39 @@ versions.forEach((version) => { }) }) }) + + contextNewVersions('playwright early bail', () => { + it('reports tests that did not run', async () => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + assert.equal(tests.length, 2) + const failedTest = tests.find(test => test.meta[TEST_STATUS] === 'fail') + assert.propertyVal(failedTest.meta, TEST_NAME, 'failing test fails and causes early bail') + const didNotRunTest = tests.find(test => test.meta[TEST_STATUS] === 'skip') + assert.propertyVal(didNotRunTest.meta, TEST_NAME, 'did not run because of early bail') + }) + + childProcess = exec( + './node_modules/.bin/playwright test -c playwright.config.js', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + PW_BASE_URL: `http://localhost:${webAppPort}`, + TEST_DIR: './ci-visibility/playwright-did-not-run', + ADD_EXTRA_PLAYWRIGHT_PROJECT: 'true' + }, + stdio: 'pipe' + } + ) + + await Promise.all([ + once(childProcess, 'exit'), + receiverPromise + ]) + }) + }) }) }) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index f9cc944beda..1faf3d36ec1 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -68,6 +68,8 @@ let modifiedFiles = {} const quarantinedOrDisabledTestsAttemptToFix = [] let quarantinedButNotAttemptToFixFqns = new Set() let rootDir = '' +let sessionProjects = [] + const MINIMUM_SUPPORTED_VERSION_RANGE_EFD = '>=1.38.0' // TODO: remove this once we drop support for v5 function isValidKnownTests (receivedKnownTests) { @@ -495,6 +497,7 @@ function dispatcherHook (dispatcherExport) { const dispatcher = this const worker = createWorker.apply(this, arguments) const projects = getProjectsFromDispatcher(dispatcher) + sessionProjects = projects // for older versions of playwright, `shouldCreateTestSpan` should always be true, // since the `_runTest` function wrapper is not available for older versions @@ -535,6 +538,7 @@ function dispatcherHookNew (dispatcherExport, runWrapper) { const dispatcher = this const worker = createWorker.apply(this, arguments) const projects = getProjectsFromDispatcher(dispatcher) + sessionProjects = projects worker.on('testBegin', ({ testId }) => { const test = getTestByTestId(dispatcher, testId) @@ -1255,3 +1259,46 @@ addHook({ return workerPackage }) + +function generateSummaryWrapper (generateSummary) { + return function () { + for (const test of this.suite.allTests()) { + // https://github.com/microsoft/playwright/blob/bf92ffecff6f30a292b53430dbaee0207e0c61ad/packages/playwright/src/reporters/base.ts#L279 + const didNotRun = test.outcome() === 'skipped' && + (!test.results.length || test.expectedStatus !== 'skipped') + if (didNotRun) { + const { + _requireFile: testSuiteAbsolutePath, + location: { line: testSourceLine }, + } = test + const browserName = getBrowserNameFromProjects(sessionProjects, test) + + testSkipCh.publish({ + testName: getTestFullname(test), + testSuiteAbsolutePath, + testSourceLine, + browserName, + }) + } + } + return generateSummary.apply(this, arguments) + } +} + +// If a playwright project B has a dependency on project A, +// and project A fails, the tests in project B will not run. +// This hook is used to report tests that did not run as skipped. +// Note: this is different from tests skipped via test.skip() or test.fixme() +addHook({ + name: 'playwright', + file: 'lib/reporters/base.js', + versions: ['>=1.38.0'] +}, (reportersPackage) => { + // v1.50.0 changed the name of the base reporter from BaseReporter to TerminalReporter + if (reportersPackage.TerminalReporter) { + shimmer.wrap(reportersPackage.TerminalReporter.prototype, 'generateSummary', generateSummaryWrapper) + } else if (reportersPackage.BaseReporter) { + shimmer.wrap(reportersPackage.BaseReporter.prototype, 'generateSummary', generateSummaryWrapper) + } + return reportersPackage +})