add integration tests

Author: juan-fernandez

integration-tests/ci-visibility/playwright-did-not-run/did-not-run.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.describe('did not run', () => {
+  test('because of early bail', async () => {
+    expect(true).toBe(false)
+  })
+})

integration-tests/ci-visibility/playwright-did-not-run/fail-test.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const { test, expect } = require('@playwright/test')
+
+test.describe('failing test', () => {
+  test('fails and causes early bail', async () => {
+    expect(true).toBe(false)
+  })
+})

integration-tests/playwright.config.js

@@ -3,6 +3,26 @@
 // Playwright config file for integration tests
 const { devices } = require('@playwright/test')
 
+const projects = [
+  {
+    name: 'chromium',
+    use: {
+      ...devices['Desktop Chrome']
+    }
+  }
+]
+
+if (process.env.ADD_EXTRA_PLAYWRIGHT_PROJECT) {
+  projects.push({
+    name: 'extra-project',
+    use: {
+      ...devices['Desktop Chrome'],
+    },
+    dependencies: ['chromium'],
+    testMatch: 'did-not-run.js'
+  })
+}
+
 const config = {
   baseURL: process.env.PW_BASE_URL,
   testDir: process.env.TEST_DIR || './ci-visibility/playwright-tests',
@@ -11,14 +31,7 @@ const config = {
   workers: process.env.PLAYWRIGHT_WORKERS ? Number(process.env.PLAYWRIGHT_WORKERS) : undefined,
   reporter: 'line',
   /* Configure projects for major browsers */
-  projects: [
-    {
-      name: 'chromium',
-      use: {
-        ...devices['Desktop Chrome']
-      }
-    }
-  ],
+  projects,
   testMatch: '**/*-test.js'
 }
 

integration-tests/playwright/playwright.spec.js

@@ -2070,5 +2070,39 @@ versions.forEach((version) => {
         })
       })
     })
+
+    contextNewVersions('playwright early bail', () => {
+      it('reports tests that did not run', async () => {
+        const receiverPromise = receiver
+          .gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
+            const events = payloads.flatMap(({ payload }) => payload.events)
+            const tests = events.filter(event => event.type === 'test').map(event => event.content)
+            assert.equal(tests.length, 2)
+            const failedTest = tests.find(test => test.meta[TEST_STATUS] === 'fail')
+            assert.propertyVal(failedTest.meta, TEST_NAME, 'failing test fails and causes early bail')
+            const didNotRunTest = tests.find(test => test.meta[TEST_STATUS] === 'skip')
+            assert.propertyVal(didNotRunTest.meta, TEST_NAME, 'did not run because of early bail')
+          })
+
+        childProcess = exec(
+          './node_modules/.bin/playwright test -c playwright.config.js',
+          {
+            cwd,
+            env: {
+              ...getCiVisAgentlessConfig(receiver.port),
+              PW_BASE_URL: `http://localhost:${webAppPort}`,
+              TEST_DIR: './ci-visibility/playwright-did-not-run',
+              ADD_EXTRA_PLAYWRIGHT_PROJECT: 'true'
+            },
+            stdio: 'pipe'
+          }
+        )
+
+        await Promise.all([
+          once(childProcess, 'exit'),
+          receiverPromise
+        ])
+      })
+    })
   })
 })

packages/datadog-instrumentations/src/playwright.js

@@ -1260,13 +1260,9 @@ addHook({
   return workerPackage
 })
 
-addHook({
-  name: 'playwright',
-  file: 'lib/reporters/base.js',
-  versions: ['>=1.38.0']
-}, (reportersPackage) => {
-  // from https://github.com/microsoft/playwright/blob/bf92ffecff6f30a292b53430dbaee0207e0c61ad/packages/playwright/src/reporters/base.ts#L279
-  shimmer.wrap(reportersPackage.TerminalReporter.prototype, 'generateSummary', generateSummary => function () {
+function generateSummaryWrapper (generateSummary) {
+  return function () {
+    // https://github.com/microsoft/playwright/blob/bf92ffecff6f30a292b53430dbaee0207e0c61ad/packages/playwright/src/reporters/base.ts#L279
     const didNotRunTests = this.suite.allTests().filter(test =>
       test.outcome() === 'skipped' && (!test.results.length || test.expectedStatus !== 'skipped')
     )
@@ -1285,6 +1281,24 @@ addHook({
       })
     }
     return generateSummary.apply(this, arguments)
-  })
+  }
+}
+
+// If a playwright project B has a dependency on project A,
+// and project A fails, the tests in project B will not run.
+// This hook is used to report tests that did not run as skipped.
+// Note: this is different from tests skipped via test.skip() or test.fixme()
+addHook({
+  name: 'playwright',
+  file: 'lib/reporters/base.js',
+  versions: ['>=1.38.0']
+}, (reportersPackage) => {
+  // v1.50.0 changed the name of the base reporter from BaseReporter to TerminalReporter
+  if (reportersPackage.TerminalReporter) {
+    shimmer.wrap(reportersPackage.TerminalReporter.prototype, 'generateSummary', generateSummaryWrapper)
+  }
+  if (reportersPackage.BaseReporter) {
+    shimmer.wrap(reportersPackage.BaseReporter.prototype, 'generateSummary', generateSummaryWrapper)
+  }
   return reportersPackage
 })