Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion integration-tests/vitest.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ const config = {
test: {
include: [
process.env.TEST_DIR || 'ci-visibility/vitest-tests/test-visibility*'
]
],
pool: process.env.POOL_CONFIG || 'forks'
}
}

Expand Down
227 changes: 120 additions & 107 deletions integration-tests/vitest/vitest.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ const {
DD_CAPABILITIES_FAILED_TEST_REPLAY,
TEST_RETRY_REASON_TYPES,
TEST_IS_MODIFIED,
DD_CAPABILITIES_IMPACTED_TESTS
DD_CAPABILITIES_IMPACTED_TESTS,
VITEST_POOL
} = require('../../packages/dd-trace/src/plugins/util/test')
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
const { NODE_MAJOR } = require('../../version')
Expand Down Expand Up @@ -89,126 +90,138 @@ versions.forEach((version) => {
await receiver.stop()
})

it('can run and report tests', (done) => {
receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)
const poolConfig = ['forks', 'threads']

metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})
poolConfig.forEach((poolConfig) => {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the test almost did not change, I just added the content.meta[VITEST_POOL] assertions. It looks changed because two pool configs are run now

it(`can run and report tests with pool=${poolConfig}`, (done) => {
receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata)

const events = payloads.flatMap(({ payload }) => payload.events)
metadataDicts.forEach(metadata => {
for (const testLevel of TEST_LEVEL_EVENT_TYPES) {
assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session')
}
})

const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')
const events = payloads.flatMap(({ payload }) => payload.events)

assert.include(testSessionEvent.content.resource, 'test_session.vitest run')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.include(testModuleEvent.content.resource, 'test_module.vitest run')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test')
assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test')
const testSessionEvent = events.find(event => event.type === 'test_session_end')

const passedSuite = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs'
)
assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass')
if (poolConfig === 'threads') {
assert.equal(testSessionEvent.content.meta[VITEST_POOL], 'worker_threads')
} else {
assert.equal(testSessionEvent.content.meta[VITEST_POOL], 'child_process')
}

const failedSuite = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs'
)
assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

const failedSuiteHooks = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs'
)
assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail')

assert.includeMembers(testEvents.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report more',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-second-describe can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-second-describe can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.no suite',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.skip no suite',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.programmatic skip no suite'
]
)
assert.include(testSessionEvent.content.resource, 'test_session.vitest run')
assert.equal(testSessionEvent.content.meta[TEST_STATUS], 'fail')
assert.include(testModuleEvent.content.resource, 'test_module.vitest run')
assert.equal(testModuleEvent.content.meta[TEST_STATUS], 'fail')
assert.equal(testSessionEvent.content.meta[TEST_TYPE], 'test')
assert.equal(testModuleEvent.content.meta[TEST_TYPE], 'test')

const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail')

assert.includeMembers(
failedTests.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more'
]
)
const passedSuite = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-passed-suite.mjs'
)
assert.equal(passedSuite.content.meta[TEST_STATUS], 'pass')

const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip')
const failedSuite = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-suite.mjs'
)
assert.equal(failedSuite.content.meta[TEST_STATUS], 'fail')

assert.includeMembers(
skippedTests.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip'
]
)
const failedSuiteHooks = testSuiteEvents.find(
suite => suite.content.resource === 'test_suite.ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs'
)
assert.equal(failedSuiteHooks.content.meta[TEST_STATUS], 'fail')

assert.includeMembers(testEvents.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report more',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-second-describe can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-second-describe can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report passed test',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.no suite',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.skip no suite',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.programmatic skip no suite'
]
)

testEvents.forEach(test => {
assert.equal(test.content.meta[TEST_COMMAND], 'vitest run')
assert.exists(test.content.metrics[DD_HOST_CPU_COUNT])
assert.equal(test.content.meta[DD_TEST_IS_USER_PROVIDED_SERVICE], 'false')
})
const failedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'fail')

assert.includeMembers(
failedTests.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-failed-suite.mjs' +
'.test-visibility-failed-suite-first-describe can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report failed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.context can report more',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report passed test',
'ci-visibility/vitest-tests/test-visibility-failed-hooks.mjs.other context can report more'
]
)

const skippedTests = testEvents.filter(test => test.content.meta[TEST_STATUS] === 'skip')

testSuiteEvents.forEach(testSuite => {
assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run')
assert.isTrue(
testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility')
assert.includeMembers(
skippedTests.map(test => test.content.resource),
[
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can skip',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can todo',
'ci-visibility/vitest-tests/test-visibility-passed-suite.mjs.other context can programmatic skip'
]
)
assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1)
assert.exists(testSuite.content.metrics[DD_HOST_CPU_COUNT])
})
// TODO: check error messages
}).then(() => done()).catch(done)

childProcess = exec(
'./node_modules/.bin/vitest run',
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags
DD_TEST_SESSION_NAME: 'my-test-session',
DD_SERVICE: undefined
},
stdio: 'pipe'
}
)
testEvents.forEach(test => {
assert.equal(test.content.meta[TEST_COMMAND], 'vitest run')
assert.exists(test.content.metrics[DD_HOST_CPU_COUNT])
assert.equal(test.content.meta[DD_TEST_IS_USER_PROVIDED_SERVICE], 'false')
})

testSuiteEvents.forEach(testSuite => {
assert.equal(testSuite.content.meta[TEST_COMMAND], 'vitest run')
assert.isTrue(
testSuite.content.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/vitest-tests/test-visibility')
)
assert.equal(testSuite.content.metrics[TEST_SOURCE_START], 1)
assert.exists(testSuite.content.metrics[DD_HOST_CPU_COUNT])
})
// TODO: check error messages
}).then(() => done()).catch(done)

childProcess = exec(
'./node_modules/.bin/vitest run',
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init', // ESM requires more flags
DD_TEST_SESSION_NAME: 'my-test-session',
POOL_CONFIG: poolConfig,
DD_SERVICE: undefined
},
stdio: 'pipe'
}
)
})
})

context('flaky test retries', () => {
Expand Down
48 changes: 40 additions & 8 deletions packages/datadog-instrumentations/src/vitest.js
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ let testManagementAttemptToFixRetries = 0
let isDiEnabled = false
let testCodeCoverageLinesTotal
let isSessionStarted = false
let vitestPool = null

const BREAKPOINT_HIT_GRACE_PERIOD_MS = 400

Expand Down Expand Up @@ -389,6 +390,7 @@ function getFinishWrapper (exitOrClose) {
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
isTestManagementTestsEnabled,
vitestPool,
onFinish
})

Expand Down Expand Up @@ -418,11 +420,27 @@ function getCreateCliWrapper (vitestPackage, frameworkVersion) {
}

function threadHandler (thread) {
if (workerProcesses.has(thread.process)) {
const { runtime } = thread
let workerProcess
if (runtime === 'child_process') {
vitestPool = 'child_process'
workerProcess = thread.process
} else if (runtime === 'worker_threads') {
vitestPool = 'worker_threads'
workerProcess = thread.thread
} else {
vitestPool = 'unknown'
}
if (!workerProcess) {
log.error('Vitest error: could not get process or thread from TinyPool#run')
return
}

if (workerProcesses.has(workerProcess)) {
return
}
workerProcesses.add(thread.process)
thread.process.on('message', (message) => {
workerProcesses.add(workerProcess)
workerProcess.on('message', (message) => {
if (message.__tinypool_worker_message__ && message.data) {
if (message.interprocessCode === VITEST_WORKER_TRACE_PAYLOAD_CODE) {
workerReportTraceCh.publish(message.data)
Expand All @@ -433,18 +451,32 @@ function threadHandler (thread) {
})
}

addHook({
name: 'tinypool',
versions: ['>=1.0.0'],
file: 'dist/index.js'
}, (TinyPool) => {
function wrapTinyPoolRun (TinyPool) {
shimmer.wrap(TinyPool.prototype, 'run', run => async function () {
// We have to do this before and after because the threads list gets recycled, that is, the processes are re-created
this.threads.forEach(threadHandler)
const runResult = await run.apply(this, arguments)
this.threads.forEach(threadHandler)
return runResult
})
}

addHook({
name: 'tinypool',
// version from [email protected] was used in [email protected]
versions: ['>=0.8.0 <1.0.0'],
file: 'dist/esm/index.js'
}, (TinyPool) => {
wrapTinyPoolRun(TinyPool)
return TinyPool
})

addHook({
name: 'tinypool',
versions: ['>=1.0.0'],
file: 'dist/index.js'
}, (TinyPool) => {
wrapTinyPoolRun(TinyPool)

return TinyPool
})
Expand Down
5 changes: 5 additions & 0 deletions packages/datadog-plugin-vitest/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const { getEnvironmentVariable } = require('../../dd-trace/src/config-helper')

const {
TEST_STATUS,
VITEST_POOL,
finishAllTraceSpans,
getTestSuitePath,
getTestSuiteCommonTags,
Expand Down Expand Up @@ -373,6 +374,7 @@ class VitestPlugin extends CiPlugin {
isEarlyFlakeDetectionEnabled,
isEarlyFlakeDetectionFaulty,
isTestManagementTestsEnabled,
vitestPool,
onFinish
}) => {
this.testSessionSpan.setTag(TEST_STATUS, status)
Expand All @@ -394,6 +396,9 @@ class VitestPlugin extends CiPlugin {
if (isTestManagementTestsEnabled) {
this.testSessionSpan.setTag(TEST_MANAGEMENT_ENABLED, 'true')
}
if (vitestPool) {
this.testSessionSpan.setTag(VITEST_POOL, vitestPool)
}
this.testModuleSpan.finish()
this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
this.testSessionSpan.finish()
Expand Down
3 changes: 3 additions & 0 deletions packages/dd-trace/src/plugins/util/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ const CI_APP_ORIGIN = 'ciapp-test'
const JEST_TEST_RUNNER = 'test.jest.test_runner'
const JEST_DISPLAY_NAME = 'test.jest.display_name'

const VITEST_POOL = 'test.vitest.pool'

const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel'
const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel'

Expand Down Expand Up @@ -203,6 +205,7 @@ module.exports = {
TEST_FRAMEWORK_VERSION,
JEST_TEST_RUNNER,
JEST_DISPLAY_NAME,
VITEST_POOL,
CUCUMBER_IS_PARALLEL,
MOCHA_IS_PARALLEL,
TEST_TYPE,
Expand Down
Loading