diff --git a/integration-tests/vitest.config.mjs b/integration-tests/vitest.config.mjs index 9a1572fb499..4e8527646c3 100644 --- a/integration-tests/vitest.config.mjs +++ b/integration-tests/vitest.config.mjs @@ -4,7 +4,8 @@ const config = { test: { include: [ process.env.TEST_DIR || 'ci-visibility/vitest-tests/test-visibility*' - ] + ], + pool: process.env.POOL_CONFIG || 'forks' } } diff --git a/integration-tests/vitest/vitest.spec.js b/integration-tests/vitest/vitest.spec.js index ec3883f9bbd..46f41ac24e2 100644 --- a/integration-tests/vitest/vitest.spec.js +++ b/integration-tests/vitest/vitest.spec.js @@ -50,7 +50,8 @@ const { DD_CAPABILITIES_FAILED_TEST_REPLAY, TEST_RETRY_REASON_TYPES, TEST_IS_MODIFIED, - DD_CAPABILITIES_IMPACTED_TESTS + DD_CAPABILITIES_IMPACTED_TESTS, + VITEST_POOL } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env') const { NODE_MAJOR } = require('../../version') @@ -89,126 +90,138 @@ versions.forEach((version) => { await receiver.stop() }) - it('can run and report tests', (done) => { - receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { - const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + const poolConfig = ['forks', 'threads'] - metadataDicts.forEach(metadata => { - for (const testLevel of TEST_LEVEL_EVENT_TYPES) { - assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') - } - }) + poolConfig.forEach((poolConfig) => { + it(`can run and report tests with pool=${poolConfig}`, (done) => { + receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) - const events = payloads.flatMap(({ payload }) => payload.events) + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) - const testSessionEvent = events.find(event => event.type === 'test_session_end') - const testModuleEvent = events.find(event => event.type === 'test_module_end') - const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') - const testEvents = events.filter(event => event.type === 'test') + const events = payloads.flatMap(({ payload }) => payload.events) - assert.include(testSessionEvent.content.resource, 'test_session.vitest run') - assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') - assert.include(testModuleEvent.content.resource, 'test_module.vitest run') - assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') - assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') - assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') + const testSessionEvent = events.find(event => event.type === 'test_session_end') - const passedSuite = testSuiteEvents.find( - suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' - ) - assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') + if (poolConfig === 'threads') { + assert.equal(testSessionEvent.content.meta[VITEST_POOL], 'worker_threads') + } else { + assert.equal(testSessionEvent.content.meta[VITEST_POOL], 'child_process') + } - const failedSuite = testSuiteEvents.find( - suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' - ) - assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') - const failedSuiteHooks = testSuiteEvents.find( - suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' - ) - assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') - - assert.includeMembers(testEvents.map(test => test.content.resource), - [ - 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + - '.test-visibility-failed-suite-first-describe can report failed test', - 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + - '.test-visibility-failed-suite-first-describe can report more', - 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + - '.test-visibility-failed-suite-second-describe can report passed test', - 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + - '.test-visibility-failed-suite-second-describe can report more', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.no suite', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.skip no suite', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.programmatic skip no suite' - ] - ) + assert.include(testSessionEvent.content.resource, 'test_session.vitest run') + assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail') + assert.include(testModuleEvent.content.resource, 'test_module.vitest run') + assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail') + assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test') + assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test') - const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') - - assert.includeMembers( - failedTests.map(test => test.content.resource), - [ - 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + - '.test-visibility-failed-suite-first-describe can report failed test', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', - 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' - ] - ) + const passedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs' + ) + assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass') - const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') + const failedSuite = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + ) + assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail') - assert.includeMembers( - skippedTests.map(test => test.content.resource), - [ - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', - 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' - ] - ) + const failedSuiteHooks = testSuiteEvents.find( + suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs' + ) + assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail') + + assert.includeMembers(testEvents.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-second-describe can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.no suite', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.skip no suite', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.programmatic skip no suite' + ] + ) - testEvents.forEach(test => { - assert.equal(test.content.meta[TEST_COMMAND], 'vitest run') - assert.exists(test.content.metrics[DD_HOST_CPU_COUNT]) - assert.equal(test.content.meta[DD_TEST_IS_USER_PROVIDED_SERVICE], 'false') - }) + const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail') + + assert.includeMembers( + failedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' + + '.test-visibility-failed-suite-first-describe can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test', + 'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more' + ] + ) + + const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip') - testSuiteEvents.forEach(testSuite => { - assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run') - assert.isTrue( - testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility') + assert.includeMembers( + skippedTests.map(test => test.content.resource), + [ + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo', + 'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip' + ] ) - assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1) - assert.exists(testSuite.content.metrics[DD_HOST_CPU_COUNT]) - }) - // TODO: check error messages - }).then(() => done()).catch(done) - childProcess = exec( - './node_modules/.bin/vitest run', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags - DD_TEST_SESSION_NAME: 'my-test-session', - DD_SERVICE: undefined - }, - stdio: 'pipe' - } - ) + testEvents.forEach(test => { + assert.equal(test.content.meta[TEST_COMMAND], 'vitest run') + assert.exists(test.content.metrics[DD_HOST_CPU_COUNT]) + assert.equal(test.content.meta[DD_TEST_IS_USER_PROVIDED_SERVICE], 'false') + }) + + testSuiteEvents.forEach(testSuite => { + assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run') + assert.isTrue( + testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility') + ) + assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1) + assert.exists(testSuite.content.metrics[DD_HOST_CPU_COUNT]) + }) + // TODO: check error messages + }).then(() => done()).catch(done) + + childProcess = exec( + './node_modules/.bin/vitest run', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags + DD_TEST_SESSION_NAME: 'my-test-session', + POOL_CONFIG: poolConfig, + DD_SERVICE: undefined + }, + stdio: 'pipe' + } + ) + }) }) context('flaky test retries', () => { diff --git a/packages/datadog-instrumentations/src/vitest.js b/packages/datadog-instrumentations/src/vitest.js index a75d65d2195..73b4674ea4f 100644 --- a/packages/datadog-instrumentations/src/vitest.js +++ b/packages/datadog-instrumentations/src/vitest.js @@ -60,6 +60,7 @@ let testManagementAttemptToFixRetries = 0 let isDiEnabled = false let testCodeCoverageLinesTotal let isSessionStarted = false +let vitestPool = null const BREAKPOINT_HIT_GRACE_PERIOD_MS = 400 @@ -389,6 +390,7 @@ function getFinishWrapper (exitOrClose) { isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, isTestManagementTestsEnabled, + vitestPool, onFinish }) @@ -418,11 +420,27 @@ function getCreateCliWrapper (vitestPackage, frameworkVersion) { } function threadHandler (thread) { - if (workerProcesses.has(thread.process)) { + const { runtime } = thread + let workerProcess + if (runtime === 'child_process') { + vitestPool = 'child_process' + workerProcess = thread.process + } else if (runtime === 'worker_threads') { + vitestPool = 'worker_threads' + workerProcess = thread.thread + } else { + vitestPool = 'unknown' + } + if (!workerProcess) { + log.error('Vitest error: could not get process or thread from TinyPool#run') + return + } + + if (workerProcesses.has(workerProcess)) { return } - workerProcesses.add(thread.process) - thread.process.on('message', (message) => { + workerProcesses.add(workerProcess) + workerProcess.on('message', (message) => { if (message.__tinypool_worker_message__ && message.data) { if (message.interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) { workerReportTraceCh.publish(message.data) @@ -433,11 +451,7 @@ function threadHandler (thread) { }) } -addHook({ - name: 'tinypool', - versions: ['>=1.0.0'], - file: 'dist/index.js' -}, (TinyPool) => { +function wrapTinyPoolRun (TinyPool) { shimmer.wrap(TinyPool.prototype, 'run', run => async function () { // We have to do this before and after because the threads list gets recycled, that is, the processes are re-created this.threads.forEach(threadHandler) @@ -445,6 +459,24 @@ addHook({ this.threads.forEach(threadHandler) return runResult }) +} + +addHook({ + name: 'tinypool', + // version from tinypool@0.8 was used in vitest@1.6.0 + versions: ['>=0.8.0 <1.0.0'], + file: 'dist/esm/index.js' +}, (TinyPool) => { + wrapTinyPoolRun(TinyPool) + return TinyPool +}) + +addHook({ + name: 'tinypool', + versions: ['>=1.0.0'], + file: 'dist/index.js' +}, (TinyPool) => { + wrapTinyPoolRun(TinyPool) return TinyPool }) diff --git a/packages/datadog-plugin-vitest/src/index.js b/packages/datadog-plugin-vitest/src/index.js index b3429d062d4..8001d200284 100644 --- a/packages/datadog-plugin-vitest/src/index.js +++ b/packages/datadog-plugin-vitest/src/index.js @@ -6,6 +6,7 @@ const { getEnvironmentVariable } = require('../../dd-trace/src/config-helper') const { TEST_STATUS, + VITEST_POOL, finishAllTraceSpans, getTestSuitePath, getTestSuiteCommonTags, @@ -373,6 +374,7 @@ class VitestPlugin extends CiPlugin { isEarlyFlakeDetectionEnabled, isEarlyFlakeDetectionFaulty, isTestManagementTestsEnabled, + vitestPool, onFinish }) => { this.testSessionSpan.setTag(TEST_STATUS, status) @@ -394,6 +396,9 @@ class VitestPlugin extends CiPlugin { if (isTestManagementTestsEnabled) { this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true') } + if (vitestPool) { + this.testSessionSpan.setTag(VITEST_POOL, vitestPool) + } this.testModuleSpan.finish() this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module') this.testSessionSpan.finish() diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index a872aee9571..9690900c369 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -92,6 +92,8 @@ const CI_APP_ORIGIN = 'ciapp-test' const JEST_TEST_RUNNER = 'test.jest.test_runner' const JEST_DISPLAY_NAME = 'test.jest.display_name' +const VITEST_POOL = 'test.vitest.pool' + const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel' const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel' @@ -203,6 +205,7 @@ module.exports = { TEST_FRAMEWORK_VERSION, JEST_TEST_RUNNER, JEST_DISPLAY_NAME, + VITEST_POOL, CUCUMBER_IS_PARALLEL, MOCHA_IS_PARALLEL, TEST_TYPE,