diff --git a/ci/init.js b/ci/init.js index 32f3c58f537..3599b2e05f4 100644 --- a/ci/init.js +++ b/ci/init.js @@ -3,6 +3,7 @@ const tracer = require('../packages/dd-trace') const { isTrue } = require('../packages/dd-trace/src/util') const isJestWorker = !!process.env.JEST_WORKER_ID +const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID const options = { startupLogs: false, @@ -37,6 +38,12 @@ if (isJestWorker) { } } +if (isCucumberWorker) { + options.experimental = { + exporter: 'cucumber_worker' + } +} + if (shouldInit) { tracer.init(options) tracer.use('fs', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index 2fed24f39ea..d2ebaefe267 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -3,7 +3,8 @@ declare const exporters: { AGENT: 'agent', DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', - JEST_WORKER: 'jest_worker' + JEST_WORKER: 'jest_worker', + CUCUMBER_WORKER: 'cucumber_worker' } export = exporters diff --git a/ext/exporters.js b/ext/exporters.js index 82425b2a550..b615d28f459 100644 --- a/ext/exporters.js +++ b/ext/exporters.js @@ -4,5 +4,6 @@ module.exports = { AGENT: 'agent', DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', - JEST_WORKER: 'jest_worker' + JEST_WORKER: 'jest_worker', + CUCUMBER_WORKER: 'cucumber_worker' } diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index 4cd71e7423b..5fa6372e8c8 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -30,7 +30,8 @@ const { TEST_EARLY_FLAKE_ENABLED, TEST_IS_NEW, TEST_IS_RETRY, - TEST_NAME + TEST_NAME, + CUCUMBER_IS_PARALLEL } = require('../../packages/dd-trace/src/plugins/util/test') const isOldNode = semver.satisfies(process.version, '<=16') @@ -44,7 +45,7 @@ const moduleType = [ './node_modules/nyc/bin/nyc.js -r=text-summary ' + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', parallelModeCommand: './node_modules/.bin/cucumber-js ' + - 'ci-visibility/features/farewell.feature --parallel 2 --publish-quiet', + 'ci-visibility/features/*.feature --parallel 2', featuresPath: 'ci-visibility/features/', fileExtension: 'js' } @@ -86,35 +87,8 @@ versions.forEach(version => { childProcess.kill() await receiver.stop() }) - const reportMethods = ['agentless', 'evp proxy'] - it('does not crash with parallel mode', (done) => { - let testOutput - childProcess = exec( - parallelModeCommand, - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - DD_TRACE_DEBUG: 1, - DD_TRACE_LOG_LEVEL: 'warn' - }, - stdio: 'inherit' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', (code) => { - assert.notInclude(testOutput, 'TypeError') - assert.include(testOutput, 'Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - assert.equal(code, 0) - done() - }) - }).timeout(50000) + const reportMethods = ['agentless', 'evp proxy'] reportMethods.forEach((reportMethod) => { context(`reporting via ${reportMethod}`, () => { @@ -123,112 +97,134 @@ versions.forEach(version => { isAgentless = reportMethod === 'agentless' envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) }) - it('can run and report tests', (done) => { - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) - - const testSessionEvent = events.find(event => event.type === 'test_session_end') - const testModuleEvent = events.find(event => event.type === 'test_module_end') - const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') - const testEvents = events.filter(event => event.type === 'test') - - const stepEvents = events.filter(event => event.type === 'span') - - const { content: testSessionEventContent } = testSessionEvent - const { content: testModuleEventContent } = testModuleEvent - - assert.exists(testSessionEventContent.test_session_id) - assert.exists(testSessionEventContent.meta[TEST_COMMAND]) - assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) - assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) - assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') - - assert.exists(testModuleEventContent.test_session_id) - assert.exists(testModuleEventContent.test_module_id) - assert.exists(testModuleEventContent.meta[TEST_COMMAND]) - assert.exists(testModuleEventContent.meta[TEST_MODULE]) - assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) - assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') - assert.equal( - testModuleEventContent.test_session_id.toString(10), - testSessionEventContent.test_session_id.toString(10) - ) + const runModes = ['serial'] - assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ - `test_suite.${featuresPath}farewell.feature`, - `test_suite.${featuresPath}greetings.feature` - ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ - 'pass', - 'fail' - ]) + if (version !== '7.0.0') { // only on latest or 9 if node is old + runModes.push('parallel') + } - testSuiteEvents.forEach(({ - content: { - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - }) + runModes.forEach((runMode) => { + it(`(${runMode}) can run and report tests`, (done) => { + const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand - assert.includeMembers(testEvents.map(test => test.content.resource), [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ]) - assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ - 'pass', - 'pass', - 'pass', - 'fail', - 'skip' - ]) + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') + + const stepEvents = events.filter(event => event.type === 'span') + + const { content: testSessionEventContent } = testSessionEvent + const { content: testModuleEventContent } = testModuleEvent + + if (runMode === 'parallel') { + assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') + } - testEvents.forEach(({ - content: { - meta, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId + assert.exists(testSessionEventContent.test_session_id) + assert.exists(testSessionEventContent.meta[TEST_COMMAND]) + assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) + assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) + assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') + + assert.exists(testModuleEventContent.test_session_id) + assert.exists(testModuleEventContent.test_module_id) + assert.exists(testModuleEventContent.meta[TEST_COMMAND]) + assert.exists(testModuleEventContent.meta[TEST_MODULE]) + assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) + assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') + assert.equal( + testModuleEventContent.test_session_id.toString(10), + testSessionEventContent.test_session_id.toString(10) + ) + + assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ + `test_suite.${featuresPath}farewell.feature`, + `test_suite.${featuresPath}greetings.feature` + ]) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ + 'pass', + 'fail' + ]) + + testSuiteEvents.forEach(({ + content: { + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + }) + + assert.includeMembers(testEvents.map(test => test.content.resource), [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ]) + assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ + 'pass', + 'pass', + 'pass', + 'fail', + 'skip' + ]) + + testEvents.forEach(({ + content: { + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) + // Can read DD_TAGS + assert.propertyVal(meta, 'test.customtag', 'customvalue') + assert.propertyVal(meta, 'test.customtag2', 'customvalue2') + if (runMode === 'parallel') { + assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') + } + }) + + stepEvents.forEach(stepEvent => { + assert.equal(stepEvent.content.name, 'cucumber.step') + assert.property(stepEvent.content.meta, 'cucumber.step') + }) + }, 5000) + + childProcess = exec( + runCommand, + { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' + }, + stdio: 'pipe' } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) - // Can read DD_TAGS - assert.propertyVal(meta, 'test.customtag', 'customvalue') - assert.propertyVal(meta, 'test.customtag2', 'customvalue2') - }) + ) - stepEvents.forEach(stepEvent => { - assert.equal(stepEvent.content.name, 'cucumber.step') - assert.property(stepEvent.content.meta, 'cucumber.step') + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) }) - }, 5000).then(() => done()).catch(done) - - childProcess = exec( - runTestsCommand, - { - cwd, - env: { - ...envVars, - DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2' - }, - stdio: 'pipe' - } - ) + }) }) context('intelligent test runner', () => { it('can report git metadata', (done) => { diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index 71d7cb392c3..b1a07d8781c 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -22,6 +22,8 @@ const skippableSuitesCh = channel('ci:cucumber:test-suite:skippable') const sessionStartCh = channel('ci:cucumber:session:start') const sessionFinishCh = channel('ci:cucumber:session:finish') +const workerReportTraceCh = channel('ci:cucumber:worker-report:trace') + const itrSkippedSuitesCh = channel('ci:cucumber:itr:skipped-suites') const { @@ -29,7 +31,8 @@ const { resetCoverage, mergeCoverage, fromCoverageMapToCoverage, - getTestSuitePath + getTestSuitePath, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE } = require('../../dd-trace/src/plugins/util/test') const isMarkedAsUnskippable = (pickle) => { @@ -47,13 +50,19 @@ const numRetriesByPickleId = new Map() let pickleByFile = {} const pickleResultByFile = {} + +const sessionAsyncResource = new AsyncResource('bound-anonymous-fn') + let skippableSuites = [] let itrCorrelationId = '' let isForcedToRun = false let isUnskippable = false +let isSuitesSkippingEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 let knownTests = [] +let skippedSuites = [] +let isSuitesSkipped = false function getSuiteStatusFromTestStatuses (testStatuses) { if (testStatuses.some(status => status === 'fail')) { @@ -106,6 +115,43 @@ function getTestStatusFromRetries (testStatuses) { return 'pass' } +function getChannelPromise (channelToPublishTo) { + return new Promise(resolve => { + sessionAsyncResource.runInAsyncScope(() => { + channelToPublishTo.publish({ onDone: resolve }) + }) + }) +} + +function getFilteredPickles (runtime, suitesToSkip) { + return runtime.pickleIds.reduce((acc, pickleId) => { + const test = runtime.eventDataCollector.getPickle(pickleId) + const testSuitePath = getTestSuitePath(test.uri, process.cwd()) + + const isUnskippable = isMarkedAsUnskippable(test) + const isSkipped = suitesToSkip.includes(testSuitePath) + + if (isSkipped && !isUnskippable) { + acc.skippedSuites.add(testSuitePath) + } else { + acc.picklesToRun.push(pickleId) + } + return acc + }, { skippedSuites: new Set(), picklesToRun: [] }) +} + +function getPickleByFile (runtime) { + return runtime.pickleIds.reduce((acc, pickleId) => { + const test = runtime.eventDataCollector.getPickle(pickleId) + if (acc[test.uri]) { + acc[test.uri].push(test) + } else { + acc[test.uri] = [test] + } + return acc + }, {}) +} + function wrapRun (pl, isLatestVersion) { if (patched.has(pl)) return @@ -125,7 +171,8 @@ function wrapRun (pl, isLatestVersion) { testStartCh.publish({ testName: this.pickle.name, testFileAbsolutePath, - testSourceLine + testSourceLine, + isParallel: !!process.env.CUCUMBER_WORKER_ID }) try { const promise = run.apply(this, arguments) @@ -193,12 +240,6 @@ function wrapRun (pl, isLatestVersion) { } function pickleHook (PickleRunner) { - if (process.env.CUCUMBER_WORKER_ID) { - // Parallel mode is not supported - log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - return PickleRunner - } - const pl = PickleRunner.default wrapRun(pl, false) @@ -207,12 +248,6 @@ function pickleHook (PickleRunner) { } function testCaseHook (TestCaseRunner) { - if (process.env.CUCUMBER_WORKER_ID) { - // Parallel mode is not supported - log.warn('Unable to initialize CI Visibility because Cucumber is running in parallel mode.') - return TestCaseRunner - } - const pl = TestCaseRunner.default wrapRun(pl, true) @@ -220,76 +255,21 @@ function testCaseHook (TestCaseRunner) { return TestCaseRunner } -addHook({ - name: '@cucumber/cucumber', - versions: ['7.0.0 - 7.2.1'], - file: 'lib/runtime/pickle_runner.js' -}, pickleHook) - -addHook({ - name: '@cucumber/cucumber', - versions: ['>=7.3.0'], - file: 'lib/runtime/test_case_runner.js' -}, testCaseHook) - -function getFilteredPickles (runtime, suitesToSkip) { - return runtime.pickleIds.reduce((acc, pickleId) => { - const test = runtime.eventDataCollector.getPickle(pickleId) - const testSuitePath = getTestSuitePath(test.uri, process.cwd()) - - const isUnskippable = isMarkedAsUnskippable(test) - const isSkipped = suitesToSkip.includes(testSuitePath) - - if (isSkipped && !isUnskippable) { - acc.skippedSuites.add(testSuitePath) - } else { - acc.picklesToRun.push(pickleId) - } - return acc - }, { skippedSuites: new Set(), picklesToRun: [] }) -} - -function getPickleByFile (runtime) { - return runtime.pickleIds.reduce((acc, pickleId) => { - const test = runtime.eventDataCollector.getPickle(pickleId) - if (acc[test.uri]) { - acc[test.uri].push(test) - } else { - acc[test.uri] = [test] - } - return acc - }, {}) -} - -function getWrappedStart (start, frameworkVersion) { +function getWrappedStart (start, frameworkVersion, isParallel = false) { return async function () { if (!libraryConfigurationCh.hasSubscribers) { return start.apply(this, arguments) } - const asyncResource = new AsyncResource('bound-anonymous-fn') - let onDone - - const configPromise = new Promise(resolve => { - onDone = resolve - }) - - asyncResource.runInAsyncScope(() => { - libraryConfigurationCh.publish({ onDone }) - }) + let errorSkippableRequest - const configurationResponse = await configPromise + const configurationResponse = await getChannelPromise(libraryConfigurationCh) isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries + isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled if (isEarlyFlakeDetectionEnabled) { - const knownTestsPromise = new Promise(resolve => { - onDone = resolve - }) - asyncResource.runInAsyncScope(() => { - knownTestsCh.publish({ onDone }) - }) - const knownTestsResponse = await knownTestsPromise + const knownTestsResponse = await getChannelPromise(knownTestsCh) if (!knownTestsResponse.err) { knownTests = knownTestsResponse.knownTests } else { @@ -297,35 +277,26 @@ function getWrappedStart (start, frameworkVersion) { } } - const skippableSuitesPromise = new Promise(resolve => { - onDone = resolve - }) + if (isSuitesSkippingEnabled) { + const skippableResponse = await getChannelPromise(skippableSuitesCh) - asyncResource.runInAsyncScope(() => { - skippableSuitesCh.publish({ onDone }) - }) + errorSkippableRequest = skippableResponse.err + skippableSuites = skippableResponse.skippableSuites - const skippableResponse = await skippableSuitesPromise + if (!errorSkippableRequest) { + const filteredPickles = getFilteredPickles(this, skippableSuites) + const { picklesToRun } = filteredPickles + isSuitesSkipped = picklesToRun.length !== this.pickleIds.length - const err = skippableResponse.err - skippableSuites = skippableResponse.skippableSuites + log.debug( + () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.` + ) - let skippedSuites = [] - let isSuitesSkipped = false + this.pickleIds = picklesToRun - if (!err) { - const filteredPickles = getFilteredPickles(this, skippableSuites) - const { picklesToRun } = filteredPickles - isSuitesSkipped = picklesToRun.length !== this.pickleIds.length - - log.debug( - () => `${picklesToRun.length} out of ${this.pickleIds.length} suites are going to run.` - ) - - this.pickleIds = picklesToRun - - skippedSuites = Array.from(filteredPickles.skippedSuites) - itrCorrelationId = skippableResponse.itrCorrelationId + skippedSuites = Array.from(filteredPickles.skippedSuites) + itrCorrelationId = skippableResponse.itrCorrelationId + } } pickleByFile = getPickleByFile(this) @@ -333,11 +304,11 @@ function getWrappedStart (start, frameworkVersion) { const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` - asyncResource.runInAsyncScope(() => { + sessionAsyncResource.runInAsyncScope(() => { sessionStartCh.publish({ command, frameworkVersion }) }) - if (!err && skippedSuites.length) { + if (!errorSkippableRequest && skippedSuites.length) { itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) } @@ -355,7 +326,7 @@ function getWrappedStart (start, frameworkVersion) { global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) } - asyncResource.runInAsyncScope(() => { + sessionAsyncResource.runInAsyncScope(() => { sessionFinishCh.publish({ status: success ? 'pass' : 'fail', isSuitesSkipped, @@ -363,7 +334,8 @@ function getWrappedStart (start, frameworkVersion) { numSkippedSuites: skippedSuites.length, hasUnskippableSuites: isUnskippable, hasForcedToRunSuites: isForcedToRun, - isEarlyFlakeDetectionEnabled + isEarlyFlakeDetectionEnabled, + isParallel }) }) return success @@ -432,7 +404,8 @@ function getWrappedRunTest (runTestFunction) { testSuiteCodeCoverageCh.publish({ coverageFiles, - suiteFile: testFileAbsolutePath + suiteFile: testFileAbsolutePath, + testSuitePath }) // We need to reset coverage to get a code coverage per suite // Before that, we preserve the original coverage @@ -440,14 +413,97 @@ function getWrappedRunTest (runTestFunction) { resetCoverage(global.__coverage__) } - testSuiteFinishCh.publish(testSuiteStatus) + testSuiteFinishCh.publish({ status: testSuiteStatus, testSuitePath }) } return runTestCaseResult } } -// From 7.3.0 onwards, runPickle becomes runTestCase +function getWrappedParseWorkerMessage (parseWorkerMessageFunction) { + return function (worker, message) { + // If the message is an array, it's a dd-trace message, so we need to stop cucumber processing, + // or cucumber will throw an error + // TODO: identify the message better + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === CUCUMBER_WORKER_TRACE_PAYLOAD_CODE) { + sessionAsyncResource.runInAsyncScope(() => { + workerReportTraceCh.publish(payload) + }) + return + } + } + + const { jsonEnvelope } = message + if (!jsonEnvelope) { + return parseWorkerMessageFunction.apply(this, arguments) + } + let parsed = jsonEnvelope + + if (typeof parsed === 'string') { + try { + parsed = JSON.parse(jsonEnvelope) + } catch (e) { + // ignore errors and continue + return parseWorkerMessageFunction.apply(this, arguments) + } + } + if (parsed.testCaseStarted) { + const { pickleId } = this.eventDataCollector.testCaseMap[parsed.testCaseStarted.testCaseId] + const pickle = this.eventDataCollector.getPickle(pickleId) + const testFileAbsolutePath = pickle.uri + // First test in suite + if (!pickleResultByFile[testFileAbsolutePath]) { + pickleResultByFile[testFileAbsolutePath] = [] + testSuiteStartCh.publish({ + testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd()) + }) + } + } + + const parseWorkerResponse = parseWorkerMessageFunction.apply(this, arguments) + + // after calling `parseWorkerMessageFunction`, the test status can already be read + if (parsed.testCaseFinished) { + const { pickle, worstTestStepResult } = + this.eventDataCollector.getTestCaseAttempt(parsed.testCaseFinished.testCaseStartedId) + + const { status } = getStatusFromResultLatest(worstTestStepResult) + + const testFileAbsolutePath = pickle.uri + const finished = pickleResultByFile[testFileAbsolutePath] + finished.push(status) + + if (finished.length === pickleByFile[testFileAbsolutePath].length) { + testSuiteFinishCh.publish({ + status: getSuiteStatusFromTestStatuses(finished), + testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd()) + }) + } + } + + return parseWorkerResponse + } +} + +// Test start / finish for older versions. The only hook executed in workers when in parallel mode +addHook({ + name: '@cucumber/cucumber', + versions: ['7.0.0 - 7.2.1'], + file: 'lib/runtime/pickle_runner.js' +}, pickleHook) + +// Test start / finish for newer versions. The only hook executed in workers when in parallel mode +addHook({ + name: '@cucumber/cucumber', + versions: ['>=7.3.0'], + file: 'lib/runtime/test_case_runner.js' +}, testCaseHook) + +// From 7.3.0 onwards, runPickle becomes runTestCase. Not executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedRunTest` generates suite start and finish events addHook({ name: '@cucumber/cucumber', versions: ['>=7.3.0'], @@ -459,6 +515,9 @@ addHook({ return runtimePackage }) +// Not executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedRunTest` generates suite start and finish events addHook({ name: '@cucumber/cucumber', versions: ['>=7.0.0 <7.3.0'], @@ -469,3 +528,21 @@ addHook({ return runtimePackage }) + +// Only executed in parallel mode. +// `getWrappedStart` generates session start and finish events +// `getWrappedGiveWork` generates suite start events and sets pickleResultByFile (used by suite finish events) +// `getWrappedParseWorkerMessage` generates suite finish events +addHook({ + name: '@cucumber/cucumber', + versions: ['>=8.0.0'], + file: 'lib/runtime/parallel/coordinator.js' +}, (coordinatorPackage, frameworkVersion) => { + shimmer.wrap(coordinatorPackage.default.prototype, 'start', start => getWrappedStart(start, frameworkVersion, true)) + shimmer.wrap( + coordinatorPackage.default.prototype, + 'parseWorkerMessage', + parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage) + ) + return coordinatorPackage +}) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index ac84285335e..5c9f378c802 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -18,7 +18,14 @@ const { TEST_SOURCE_FILE, TEST_EARLY_FLAKE_ENABLED, TEST_IS_NEW, - TEST_IS_RETRY + TEST_IS_RETRY, + TEST_SUITE_ID, + TEST_SESSION_ID, + TEST_COMMAND, + TEST_MODULE, + TEST_MODULE_ID, + TEST_SUITE, + CUCUMBER_IS_PARALLEL } = require('../../dd-trace/src/plugins/util/test') const { RESOURCE_NAME } = require('../../../ext/tags') const { COMPONENT, ERROR_MESSAGE } = require('../../dd-trace/src/constants') @@ -32,6 +39,22 @@ const { TELEMETRY_ITR_UNSKIPPABLE, TELEMETRY_CODE_COVERAGE_NUM_FILES } = require('../../dd-trace/src/ci-visibility/telemetry') +const id = require('../../dd-trace/src/id') + +const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID + +function getTestSuiteTags (testSuiteSpan) { + const suiteTags = { + [TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(), + [TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(), + [TEST_COMMAND]: testSuiteSpan.context()._tags[TEST_COMMAND], + [TEST_MODULE]: 'cucumber' + } + if (testSuiteSpan.context()._parentId) { + suiteTags[TEST_MODULE_ID] = testSuiteSpan.context()._parentId.toString(10) + } + return suiteTags +} class CucumberPlugin extends CiPlugin { static get id () { @@ -43,6 +66,8 @@ class CucumberPlugin extends CiPlugin { this.sourceRoot = process.cwd() + this.testSuiteSpanByPath = {} + this.addSub('ci:cucumber:session:finish', ({ status, isSuitesSkipped, @@ -50,7 +75,8 @@ class CucumberPlugin extends CiPlugin { testCodeCoverageLinesTotal, hasUnskippableSuites, hasForcedToRunSuites, - isEarlyFlakeDetectionEnabled + isEarlyFlakeDetectionEnabled, + isParallel }) => { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} addIntelligentTestRunnerSpanTags( @@ -70,6 +96,9 @@ class CucumberPlugin extends CiPlugin { if (isEarlyFlakeDetectionEnabled) { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') } + if (isParallel) { + this.testSessionSpan.setTag(CUCUMBER_IS_PARALLEL, 'true') + } this.testSessionSpan.setTag(TEST_STATUS, status) this.testModuleSpan.setTag(TEST_STATUS, status) @@ -101,7 +130,7 @@ class CucumberPlugin extends CiPlugin { if (itrCorrelationId) { testSuiteMetadata[ITR_CORRELATION_ID] = itrCorrelationId } - this.testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', { + const testSuiteSpan = this.tracer.startSpan('cucumber.test_suite', { childOf: this.testModuleSpan, tags: { [COMPONENT]: this.constructor.id, @@ -109,25 +138,29 @@ class CucumberPlugin extends CiPlugin { ...testSuiteMetadata } }) + this.testSuiteSpanByPath[testSuitePath] = testSuiteSpan + this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite') if (this.libraryConfig?.isCodeCoverageEnabled) { this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_STARTED, 'suite', { library: 'istanbul' }) } }) - this.addSub('ci:cucumber:test-suite:finish', status => { - this.testSuiteSpan.setTag(TEST_STATUS, status) - this.testSuiteSpan.finish() + this.addSub('ci:cucumber:test-suite:finish', ({ status, testSuitePath }) => { + const testSuiteSpan = this.testSuiteSpanByPath[testSuitePath] + testSuiteSpan.setTag(TEST_STATUS, status) + testSuiteSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'suite') }) - this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => { + this.addSub('ci:cucumber:test-suite:code-coverage', ({ coverageFiles, suiteFile, testSuitePath }) => { if (!this.libraryConfig?.isCodeCoverageEnabled) { return } if (!coverageFiles.length) { this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) } + const testSuiteSpan = this.testSuiteSpanByPath[testSuitePath] const relativeCoverageFiles = [...coverageFiles, suiteFile] .map(filename => getTestSuitePath(filename, this.repositoryRoot)) @@ -135,8 +168,8 @@ class CucumberPlugin extends CiPlugin { this.telemetry.distribution(TELEMETRY_CODE_COVERAGE_NUM_FILES, {}, relativeCoverageFiles.length) const formattedCoverage = { - sessionId: this.testSuiteSpan.context()._traceId, - suiteId: this.testSuiteSpan.context()._spanId, + sessionId: testSuiteSpan.context()._traceId, + suiteId: testSuiteSpan.context()._spanId, files: relativeCoverageFiles } @@ -144,7 +177,7 @@ class CucumberPlugin extends CiPlugin { this.telemetry.ciVisEvent(TELEMETRY_CODE_COVERAGE_FINISHED, 'suite', { library: 'istanbul' }) }) - this.addSub('ci:cucumber:test:start', ({ testName, testFileAbsolutePath, testSourceLine }) => { + this.addSub('ci:cucumber:test:start', ({ testName, testFileAbsolutePath, testSourceLine, isParallel }) => { const store = storage.getStore() const testSuite = getTestSuitePath(testFileAbsolutePath, this.sourceRoot) const testSourceFile = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot) @@ -153,6 +186,10 @@ class CucumberPlugin extends CiPlugin { [TEST_SOURCE_START]: testSourceLine, [TEST_SOURCE_FILE]: testSourceFile } + if (isParallel) { + extraTags[CUCUMBER_IS_PARALLEL] = 'true' + } + const testSpan = this.startTestSpan(testName, testSuite, extraTags) this.enter(testSpan, store) @@ -172,6 +209,36 @@ class CucumberPlugin extends CiPlugin { this.enter(span, store) }) + this.addSub('ci:cucumber:worker-report:trace', (traces) => { + const formattedTraces = JSON.parse(traces).map(trace => + trace.map(span => ({ + ...span, + span_id: id(span.span_id), + trace_id: id(span.trace_id), + parent_id: id(span.parent_id) + })) + ) + + // We have to update the test session, test module and test suite ids + // before we export them in the main process + formattedTraces.forEach(trace => { + trace.forEach(span => { + if (span.name === 'cucumber.test') { + const testSuite = span.meta[TEST_SUITE] + const testSuiteSpan = this.testSuiteSpanByPath[testSuite] + + const testSuiteTags = getTestSuiteTags(testSuiteSpan) + span.meta = { + ...span.meta, + ...testSuiteTags + } + } + }) + + this.tracer._exporter.export(trace) + }) + }) + this.addSub('ci:cucumber:test:finish', ({ isStep, status, skipReason, errorMessage, isNew, isEfdRetry }) => { const span = storage.getStore().span const statusTag = isStep ? 'step.status' : TEST_STATUS @@ -201,6 +268,10 @@ class CucumberPlugin extends CiPlugin { { hasCodeOwners: !!span.context()._tags[TEST_CODE_OWNERS] } ) finishAllTraceSpans(span) + // If it's a worker, flushing is cheap, as it's just sending data to the main process + if (isCucumberWorker) { + this.tracer._exporter.flush() + } } }) @@ -213,10 +284,11 @@ class CucumberPlugin extends CiPlugin { } startTestSpan (testName, testSuite, extraTags) { + const testSuiteSpan = this.testSuiteSpanByPath[testSuite] return super.startTestSpan( testName, testSuite, - this.testSuiteSpan, + testSuiteSpan, extraTags ) } diff --git a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js deleted file mode 100644 index 8f96937e22f..00000000000 --- a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/index.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' - -const Writer = require('./writer') -const { - JEST_WORKER_COVERAGE_PAYLOAD_CODE, - JEST_WORKER_TRACE_PAYLOAD_CODE -} = require('../../../plugins/util/test') - -/** - * Lightweight exporter whose writers only do simple JSON serialization - * of trace and coverage payloads, which they send to the jest main process. - */ -class JestWorkerCiVisibilityExporter { - constructor () { - this._writer = new Writer(JEST_WORKER_TRACE_PAYLOAD_CODE) - this._coverageWriter = new Writer(JEST_WORKER_COVERAGE_PAYLOAD_CODE) - } - - export (payload) { - this._writer.append(payload) - } - - exportCoverage (formattedCoverage) { - this._coverageWriter.append(formattedCoverage) - } - - flush () { - this._writer.flush() - this._coverageWriter.flush() - } -} - -module.exports = JestWorkerCiVisibilityExporter diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js new file mode 100644 index 00000000000..f91bdd52090 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js @@ -0,0 +1,56 @@ +'use strict' + +const Writer = require('./writer') +const { + JEST_WORKER_COVERAGE_PAYLOAD_CODE, + JEST_WORKER_TRACE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE +} = require('../../../plugins/util/test') + +function getInterprocessTraceCode () { + if (process.env.JEST_WORKER_ID) { + return JEST_WORKER_TRACE_PAYLOAD_CODE + } + if (process.env.CUCUMBER_WORKER_ID) { + return CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + } + return null +} + +// TODO: make it available with cucumber +function getInterprocessCoverageCode () { + if (process.env.JEST_WORKER_ID) { + return JEST_WORKER_COVERAGE_PAYLOAD_CODE + } + return null +} + +/** + * Lightweight exporter whose writers only do simple JSON serialization + * of trace and coverage payloads, which they send to the test framework's main process. + * Currently used by Jest and Cucumber workers. + */ +class TestWorkerCiVisibilityExporter { + constructor () { + const interprocessTraceCode = getInterprocessTraceCode() + const interprocessCoverageCode = getInterprocessCoverageCode() + + this._writer = new Writer(interprocessTraceCode) + this._coverageWriter = new Writer(interprocessCoverageCode) + } + + export (payload) { + this._writer.append(payload) + } + + exportCoverage (formattedCoverage) { + this._coverageWriter.append(formattedCoverage) + } + + flush () { + this._writer.flush() + this._coverageWriter.flush() + } +} + +module.exports = TestWorkerCiVisibilityExporter diff --git a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js similarity index 76% rename from packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js rename to packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js index 8b4ada0c500..d5004b28273 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/jest-worker/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js @@ -23,11 +23,18 @@ class Writer { } _sendPayload (data) { + // ## Jest // Only available when `child_process` is used for the jest worker. // eslint-disable-next-line // https://github.com/facebook/jest/blob/bb39cb2c617a3334bf18daeca66bd87b7ccab28b/packages/jest-worker/README.md#experimental-worker // If worker_threads is used, this will not work // TODO: make it compatible with worker_threads + + // ## Cucumber + // This reports to the test's main process the same way test data is reported by Cucumber + // See cucumber code: + // eslint-disable-next-line + // https://github.com/cucumber/cucumber-js/blob/5ce371870b677fe3d1a14915dc535688946f734c/src/runtime/parallel/run_worker.ts#L13 if (process.send) { // it only works if process.send is available process.send([this._interprocessCode, data]) } diff --git a/packages/dd-trace/src/exporter.js b/packages/dd-trace/src/exporter.js index 042e43910dc..01bb96ac380 100644 --- a/packages/dd-trace/src/exporter.js +++ b/packages/dd-trace/src/exporter.js @@ -18,7 +18,8 @@ module.exports = name => { case exporters.AGENT_PROXY: return require('./ci-visibility/exporters/agent-proxy') case exporters.JEST_WORKER: - return require('./ci-visibility/exporters/jest-worker') + case exporters.CUCUMBER_WORKER: + return require('./ci-visibility/exporters/test-worker') default: return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent') } diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 0de9bd33388..23ce067670a 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -61,6 +61,8 @@ const CI_APP_ORIGIN = 'ciapp-test' const JEST_TEST_RUNNER = 'test.jest.test_runner' const JEST_DISPLAY_NAME = 'test.jest.display_name' +const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel' + const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped' const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled' const TEST_ITR_SKIPPING_TYPE = 'test.itr.tests_skipping.type' @@ -82,6 +84,9 @@ const TEST_BROWSER_VERSION = 'test.browser.version' const JEST_WORKER_TRACE_PAYLOAD_CODE = 60 const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61 +// cucumber worker variables +const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70 + // Early flake detection util strings const EFD_STRING = "Retried by Datadog's Early Flake Detection" const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') @@ -92,6 +97,7 @@ module.exports = { TEST_FRAMEWORK_VERSION, JEST_TEST_RUNNER, JEST_DISPLAY_NAME, + CUCUMBER_IS_PARALLEL, TEST_TYPE, TEST_NAME, TEST_SUITE, @@ -104,6 +110,7 @@ module.exports = { LIBRARY_VERSION, JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, TEST_SOURCE_START, TEST_SKIPPED_BY_ITR, TEST_CONFIGURATION_BROWSER_NAME, diff --git a/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js deleted file mode 100644 index dd673400d28..00000000000 --- a/packages/dd-trace/test/ci-visibility/exporters/jest-worker/exporter.spec.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict' - -require('../../../../../dd-trace/test/setup/tap') - -const JestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/jest-worker') -const { - JEST_WORKER_TRACE_PAYLOAD_CODE, - JEST_WORKER_COVERAGE_PAYLOAD_CODE -} = require('../../../../src/plugins/util/test') - -describe('CI Visibility Jest Worker Exporter', () => { - let send, originalSend - beforeEach(() => { - send = sinon.spy() - originalSend = process.send - process.send = send - }) - afterEach(() => { - process.send = originalSend - }) - it('can export traces', () => { - const trace = [{ type: 'test' }] - const traceSecond = [{ type: 'test', name: 'other' }] - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.export(trace) - jestWorkerExporter.export(traceSecond) - jestWorkerExporter.flush() - expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) - }) - it('can export coverages', () => { - const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } - const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.exportCoverage(coverage) - jestWorkerExporter.exportCoverage(coverageSecond) - jestWorkerExporter.flush() - expect(send).to.have.been.calledWith( - [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] - ) - }) - it('does not break if process.send is undefined', () => { - delete process.send - const trace = [{ type: 'test' }] - const jestWorkerExporter = new JestWorkerCiVisibilityExporter() - jestWorkerExporter.export(trace) - jestWorkerExporter.flush() - expect(send).not.to.have.been.called - }) -}) diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js new file mode 100644 index 00000000000..cb212eca891 --- /dev/null +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -0,0 +1,83 @@ +'use strict' + +require('../../../../../dd-trace/test/setup/tap') + +const TestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/exporters/test-worker') +const { + JEST_WORKER_TRACE_PAYLOAD_CODE, + JEST_WORKER_COVERAGE_PAYLOAD_CODE, + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE +} = require('../../../../src/plugins/util/test') + +describe('CI Visibility Test Worker Exporter', () => { + let send, originalSend + beforeEach(() => { + send = sinon.spy() + originalSend = process.send + process.send = send + }) + afterEach(() => { + process.send = originalSend + }) + context('when the process is a jest worker', () => { + beforeEach(() => { + process.env.JEST_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.JEST_WORKER_ID + }) + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.export(trace) + jestWorkerExporter.export(traceSecond) + jestWorkerExporter.flush() + expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + it('can export coverages', () => { + const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } + const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.exportCoverage(coverage) + jestWorkerExporter.exportCoverage(coverageSecond) + jestWorkerExporter.flush() + expect(send).to.have.been.calledWith( + [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] + ) + }) + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const jestWorkerExporter = new TestWorkerCiVisibilityExporter() + jestWorkerExporter.export(trace) + jestWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) + context('when the process is a cucumber worker', () => { + beforeEach(() => { + process.env.CUCUMBER_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.CUCUMBER_WORKER_ID + }) + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const cucumberWorkerExporter = new TestWorkerCiVisibilityExporter() + cucumberWorkerExporter.export(trace) + cucumberWorkerExporter.export(traceSecond) + cucumberWorkerExporter.flush() + expect(send).to.have.been.calledWith([CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const cucumberWorkerExporter = new TestWorkerCiVisibilityExporter() + cucumberWorkerExporter.export(trace) + cucumberWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) +})