Skip to content

[ci-visibility] Support parallel mode in cucumber #4260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
juan-fernandez merged 9 commits into from
Apr 30, 2024
Merged

[ci-visibility] Support parallel mode in cucumber #4260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
ee5ca87
support for cucumber parallel
juan-fernandez Apr 24, 2024
bd7e9f9
add e2e tests
juan-fernandez Apr 24, 2024
7a7ef61
reorg
juan-fernandez Apr 25, 2024
d325a31
better org
juan-fernandez Apr 25, 2024
f5021cc
fix parallel mode reporting
juan-fernandez Apr 26, 2024
a1b9251
make common exporter
juan-fernandez Apr 26, 2024
8b63219
fix test and simplify instrumentation
juan-fernandez Apr 26, 2024
75db283
better messages
juan-fernandez Apr 29, 2024
64f71f3
Merge branch 'master' into juan-fernandez/support-cucumber-parallel
juan-fernandez Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions ci/init.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ const tracer = require('../packages/dd-trace')
const { isTrue } = require('../packages/dd-trace/src/util')

const isJestWorker = !!process.env.JEST_WORKER_ID
const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID

const options = {
startupLogs: false,
Expand Down Expand Up @@ -37,6 +38,12 @@ if (isJestWorker) {
}
}

if (isCucumberWorker) {
options.experimental = {
exporter: 'cucumber_worker'
}
}

if (shouldInit) {
tracer.init(options)
tracer.use('fs', false)
Expand Down
3 changes: 2 additions & 1 deletion ext/exporters.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ declare const exporters: {
AGENT: 'agent',
DATADOG: 'datadog',
AGENT_PROXY: 'agent_proxy',
JEST_WORKER: 'jest_worker'
JEST_WORKER: 'jest_worker',
CUCUMBER_WORKER: 'cucumber_worker'
}

export = exporters
3 changes: 2 additions & 1 deletion ext/exporters.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@ module.exports = {
AGENT: 'agent',
DATADOG: 'datadog',
AGENT_PROXY: 'agent_proxy',
JEST_WORKER: 'jest_worker'
JEST_WORKER: 'jest_worker',
CUCUMBER_WORKER: 'cucumber_worker'
}
254 changes: 125 additions & 129 deletions integration-tests/cucumber/cucumber.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ const {
TEST_EARLY_FLAKE_ENABLED,
TEST_IS_NEW,
TEST_IS_RETRY,
TEST_NAME
TEST_NAME,
CUCUMBER_IS_PARALLEL
} = require('../../packages/dd-trace/src/plugins/util/test')

const isOldNode = semver.satisfies(process.version, '<=16')
Expand All @@ -44,7 +45,7 @@ const moduleType = [
'./node_modules/nyc/bin/nyc.js -r=text-summary ' +
'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature',
parallelModeCommand: './node_modules/.bin/cucumber-js ' +
'ci-visibility/features/farewell.feature --parallel 2 --publish-quiet',
'ci-visibility/features/*.feature --parallel 2',
featuresPath: 'ci-visibility/features/',
fileExtension: 'js'
}
Expand Down Expand Up @@ -86,35 +87,8 @@ versions.forEach(version => {
childProcess.kill()
await receiver.stop()
})
const reportMethods = ['agentless', 'evp proxy']

it('does not crash with parallel mode', (done) => {
let testOutput
childProcess = exec(
parallelModeCommand,
{
cwd,
env: {
...getCiVisAgentlessConfig(receiver.port),
DD_TRACE_DEBUG: 1,
DD_TRACE_LOG_LEVEL: 'warn'
},
stdio: 'inherit'
}
)
childProcess.stdout.on('data', (chunk) => {
testOutput += chunk.toString()
})
childProcess.stderr.on('data', (chunk) => {
testOutput += chunk.toString()
})
childProcess.on('exit', (code) => {
assert.notInclude(testOutput, 'TypeError')
assert.include(testOutput, 'Unable to initialize CI Visibility because Cucumber is running in parallel mode.')
assert.equal(code, 0)
done()
})
}).timeout(50000)
const reportMethods = ['agentless', 'evp proxy']

reportMethods.forEach((reportMethod) => {
context(`reporting via ${reportMethod}`, () => {
Expand All @@ -123,112 +97,134 @@ versions.forEach(version => {
isAgentless = reportMethod === 'agentless'
envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port)
})
it('can run and report tests', (done) => {
Copy link
Collaborator Author

@juan-fernandez juan-fernandez Apr 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The changes in this file are just for running this test both in serial and parallel mode

receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

const stepEvents = events.filter(event => event.type === 'span')

const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
assert.exists(testModuleEventContent.meta[TEST_MODULE])
assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true)
assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail')
assert.equal(
testModuleEventContent.test_session_id.toString(10),
testSessionEventContent.test_session_id.toString(10)
)
const runModes = ['serial']

assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [
`test_suite.${featuresPath}farewell.feature`,
`test_suite.${featuresPath}greetings.feature`
])
assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [
'pass',
'fail'
])
if (version !== '7.0.0') { // only on latest or 9 if node is old
runModes.push('parallel')
}

testSuiteEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
})
runModes.forEach((runMode) => {
it(`(${runMode}) can run and report tests`, (done) => {
const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand

assert.includeMembers(testEvents.map(test => test.content.resource), [
`${featuresPath}farewell.feature.Say farewell`,
`${featuresPath}greetings.feature.Say greetings`,
`${featuresPath}greetings.feature.Say yeah`,
`${featuresPath}greetings.feature.Say yo`,
`${featuresPath}greetings.feature.Say skip`
])
assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [
'pass',
'pass',
'pass',
'fail',
'skip'
])
const receiverPromise = receiver
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => {
const events = payloads.flatMap(({ payload }) => payload.events)

const testSessionEvent = events.find(event => event.type === 'test_session_end')
const testModuleEvent = events.find(event => event.type === 'test_module_end')
const testSuiteEvents = events.filter(event => event.type === 'test_suite_end')
const testEvents = events.filter(event => event.type === 'test')

const stepEvents = events.filter(event => event.type === 'span')

const { content: testSessionEventContent } = testSessionEvent
const { content: testModuleEventContent } = testModuleEvent

if (runMode === 'parallel') {
assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true')
}

testEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
assert.exists(testSessionEventContent.test_session_id)
assert.exists(testSessionEventContent.meta[TEST_COMMAND])
assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN])
assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true)
assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail')

assert.exists(testModuleEventContent.test_session_id)
assert.exists(testModuleEventContent.test_module_id)
assert.exists(testModuleEventContent.meta[TEST_COMMAND])
assert.exists(testModuleEventContent.meta[TEST_MODULE])
assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true)
assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail')
assert.equal(
testModuleEventContent.test_session_id.toString(10),
testSessionEventContent.test_session_id.toString(10)
)

assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [
`test_suite.${featuresPath}farewell.feature`,
`test_suite.${featuresPath}greetings.feature`
])
assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [
'pass',
'fail'
])

testSuiteEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
})

assert.includeMembers(testEvents.map(test => test.content.resource), [
`${featuresPath}farewell.feature.Say farewell`,
`${featuresPath}greetings.feature.Say greetings`,
`${featuresPath}greetings.feature.Say yeah`,
`${featuresPath}greetings.feature.Say yo`,
`${featuresPath}greetings.feature.Say skip`
])
assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [
'pass',
'pass',
'pass',
'fail',
'skip'
])

testEvents.forEach(({
content: {
meta,
test_suite_id: testSuiteId,
test_module_id: testModuleId,
test_session_id: testSessionId
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true)
// Can read DD_TAGS
assert.propertyVal(meta, 'test.customtag', 'customvalue')
assert.propertyVal(meta, 'test.customtag2', 'customvalue2')
if (runMode === 'parallel') {
assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true')
}
})

stepEvents.forEach(stepEvent => {
assert.equal(stepEvent.content.name, 'cucumber.step')
assert.property(stepEvent.content.meta, 'cucumber.step')
})
}, 5000)

childProcess = exec(
runCommand,
{
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
},
stdio: 'pipe'
}
}) => {
assert.exists(meta[TEST_COMMAND])
assert.exists(meta[TEST_MODULE])
assert.exists(testSuiteId)
assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10))
assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10))
assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true)
// Can read DD_TAGS
assert.propertyVal(meta, 'test.customtag', 'customvalue')
assert.propertyVal(meta, 'test.customtag2', 'customvalue2')
})
)

stepEvents.forEach(stepEvent => {
assert.equal(stepEvent.content.name, 'cucumber.step')
assert.property(stepEvent.content.meta, 'cucumber.step')
childProcess.on('exit', () => {
receiverPromise.then(() => done()).catch(done)
})
}, 5000).then(() => done()).catch(done)

childProcess = exec(
runTestsCommand,
{
cwd,
env: {
...envVars,
DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2'
},
stdio: 'pipe'
}
)
})
})
context('intelligent test runner', () => {
it('can report git metadata', (done) => {
Expand Down
Loading