Skip to content

Commit e6dc87d

Browse files
juan-fernandezStephen Belanger
juan-fernandez
authored and
Stephen Belanger
committed
[ci-visibility] Intelligent test runner: only report skipped flag if tests were actually skipped (#3369)
1 parent 791d33a commit e6dc87d

File tree

7 files changed

+169
-8
lines changed

7 files changed

+169
-8
lines changed

‎integration-tests/ci-visibility.spec.js‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,40 @@ testFrameworks.forEach(({
617617
}
618618
)
619619
})
620+
it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => {
621+
receiver.setSuitesToSkip([{
622+
type: 'suite',
623+
attributes: {
624+
suite: 'ci-visibility/test/not-existing-test.js'
625+
}
626+
}])
627+
const eventsPromise = receiver
628+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
629+
const events = payloads.flatMap(({ payload }) => payload.events)
630+
const testSession = events.find(event => event.type === 'test_session_end').content
631+
assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false')
632+
assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
633+
assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
634+
const testModule = events.find(event => event.type === 'test_module_end').content
635+
assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false')
636+
assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
637+
assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
638+
}, 25000)
639+
640+
childProcess = exec(
641+
runTestsWithCoverageCommand,
642+
{
643+
cwd,
644+
env: getCiVisAgentlessConfig(receiver.port),
645+
stdio: 'inherit'
646+
}
647+
)
648+
childProcess.on('exit', () => {
649+
eventsPromise.then(() => {
650+
done()
651+
}).catch(done)
652+
})
653+
})
620654
})
621655

622656
describe('evp proxy', () => {
@@ -921,6 +955,40 @@ testFrameworks.forEach(({
921955
}
922956
)
923957
})
958+
it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => {
959+
receiver.setSuitesToSkip([{
960+
type: 'suite',
961+
attributes: {
962+
suite: 'ci-visibility/test/not-existing-test.js'
963+
}
964+
}])
965+
const eventsPromise = receiver
966+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
967+
const events = payloads.flatMap(({ payload }) => payload.events)
968+
const testSession = events.find(event => event.type === 'test_session_end').content
969+
assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false')
970+
assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
971+
assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
972+
const testModule = events.find(event => event.type === 'test_module_end').content
973+
assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false')
974+
assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
975+
assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
976+
}, 25000)
977+
978+
childProcess = exec(
979+
runTestsWithCoverageCommand,
980+
{
981+
cwd,
982+
env: getCiVisEvpProxyConfig(receiver.port),
983+
stdio: 'inherit'
984+
}
985+
)
986+
childProcess.on('exit', () => {
987+
eventsPromise.then(() => {
988+
done()
989+
}).catch(done)
990+
})
991+
})
924992
})
925993
})
926994
})

‎integration-tests/cucumber/cucumber.spec.js‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,40 @@ versions.forEach(version => {
485485
}
486486
)
487487
})
488+
it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => {
489+
receiver.setSuitesToSkip([{
490+
type: 'suite',
491+
attributes: {
492+
suite: 'ci-visibility/features/not-existing.feature'
493+
}
494+
}])
495+
const eventsPromise = receiver
496+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
497+
const events = payloads.flatMap(({ payload }) => payload.events)
498+
const testSession = events.find(event => event.type === 'test_session_end').content
499+
assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false')
500+
assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
501+
assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
502+
const testModule = events.find(event => event.type === 'test_module_end').content
503+
assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false')
504+
assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
505+
assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
506+
}, 25000)
507+
508+
childProcess = exec(
509+
runTestsWithCoverageCommand,
510+
{
511+
cwd,
512+
env: envVars,
513+
stdio: 'inherit'
514+
}
515+
)
516+
childProcess.on('exit', () => {
517+
eventsPromise.then(() => {
518+
done()
519+
}).catch(done)
520+
})
521+
})
488522
if (!isAgentless) {
489523
context('if the agent is not event platform proxy compatible', () => {
490524
it('does not do any intelligent test runner request', (done) => {

‎integration-tests/cypress/cypress.spec.js‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ describe(`cypress@${version}`, function () {
364364
}).catch(done)
365365
})
366366
})
367-
it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => {
367+
it('can skip tests received by the intelligent test runner API and still reports code coverage', (done) => {
368368
receiver.setSuitesToSkip([{
369369
type: 'test',
370370
attributes: {
@@ -473,5 +473,55 @@ describe(`cypress@${version}`, function () {
473473
}).catch(done)
474474
})
475475
})
476+
it('sets _dd.ci.itr.tests_skipped to false if the received test is not skipped', (done) => {
477+
receiver.setSuitesToSkip([{
478+
type: 'test',
479+
attributes: {
480+
name: 'fake name',
481+
suite: 'i/dont/exist.spec.js'
482+
}
483+
}])
484+
const eventsPromise = receiver
485+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
486+
const events = payloads.flatMap(({ payload }) => payload.events)
487+
const testSession = events.find(event => event.type === 'test_session_end').content
488+
assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false')
489+
assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
490+
assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
491+
const testModule = events.find(event => event.type === 'test_module_end').content
492+
assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false')
493+
assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true')
494+
assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true')
495+
}, 25000)
496+
497+
const skippableRequestPromise = receiver
498+
.payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable'))
499+
.then(skippableRequest => {
500+
assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1')
501+
assert.propertyVal(skippableRequest.headers, 'dd-application-key', '1')
502+
})
503+
504+
const {
505+
NODE_OPTIONS,
506+
...restEnvVars
507+
} = getCiVisAgentlessConfig(receiver.port)
508+
509+
childProcess = exec(
510+
`./node_modules/.bin/cypress run --quiet ${commandSuffix}`,
511+
{
512+
cwd,
513+
env: {
514+
...restEnvVars,
515+
CYPRESS_BASE_URL: `http://localhost:${webAppPort}`
516+
},
517+
stdio: 'pipe'
518+
}
519+
)
520+
childProcess.on('exit', () => {
521+
Promise.all([eventsPromise, skippableRequestPromise]).then(() => {
522+
done()
523+
}).catch(done)
524+
})
525+
})
476526
})
477527
})

‎packages/datadog-instrumentations/src/cucumber.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const patched = new WeakSet()
3737

3838
let pickleByFile = {}
3939
const pickleResultByFile = {}
40+
let isSuitesSkipped = false
4041

4142
function getSuiteStatusFromTestStatuses (testStatuses) {
4243
if (testStatuses.some(status => status === 'fail')) {
@@ -264,7 +265,9 @@ addHook({
264265
const { err, skippableSuites } = await skippableSuitesPromise
265266

266267
if (!err) {
267-
this.pickleIds = getPicklesToRun(this, skippableSuites)
268+
const newPickleIds = getPicklesToRun(this, skippableSuites)
269+
isSuitesSkipped = newPickleIds.length !== this.pickleIds.length
270+
this.pickleIds = newPickleIds
268271
}
269272

270273
pickleByFile = getPickleByFile(this)
@@ -292,7 +295,7 @@ addHook({
292295
asyncResource.runInAsyncScope(() => {
293296
sessionFinishCh.publish({
294297
status: success ? 'pass' : 'fail',
295-
isSuitesSkipped: skippableSuites ? !!skippableSuites.length : false,
298+
isSuitesSkipped,
296299
testCodeCoverageLinesTotal
297300
})
298301
})

‎packages/datadog-instrumentations/src/jest.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ const jestItrConfigurationCh = channel('ci:jest:itr-configuration')
4242
let skippableSuites = []
4343
let isCodeCoverageEnabled = false
4444
let isSuitesSkippingEnabled = false
45+
let isSuitesSkipped = false
4546

4647
const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
4748

@@ -227,7 +228,6 @@ function cliWrapper (cli, jestVersion) {
227228
log.error(err)
228229
}
229230
}
230-
const isSuitesSkipped = !!skippableSuites.length
231231

232232
const processArgv = process.argv.slice(2).join(' ')
233233
sessionAsyncResource.runInAsyncScope(() => {
@@ -430,6 +430,8 @@ addHook({
430430

431431
const filteredTests = getJestSuitesToRun(skippableSuites, tests, rootDir)
432432

433+
isSuitesSkipped = filteredTests.length !== tests.length
434+
433435
skippableSuites = []
434436

435437
return { ...testPaths, tests: filteredTests }

‎packages/datadog-instrumentations/src/mocha.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ const originalCoverageMap = createCoverageMap()
4646

4747
let suitesToSkip = []
4848
let frameworkVersion
49+
let isSuitesSkipped = false
4950

5051
function getSuitesByTestFile (root) {
5152
const suitesByTestFile = {}
@@ -125,8 +126,6 @@ function mochaHook (Runner) {
125126
}
126127
testFileToSuiteAr.clear()
127128

128-
const isSuitesSkipped = !!suitesToSkip.length
129-
130129
let testCodeCoverageLinesTotal
131130
if (global.__coverage__) {
132131
try {
@@ -360,7 +359,10 @@ addHook({
360359
suitesToSkip = skippableSuites
361360
}
362361
// We remove the suites that we skip through ITR
363-
runner.suite.suites = getSuitesToRun(runner.suite.suites)
362+
const newSuites = getSuitesToRun(runner.suite.suites)
363+
isSuitesSkipped = newSuites.length !== runner.suite.suites.length
364+
runner.suite.suites = newSuites
365+
364366
global.run()
365367
}
366368

‎packages/datadog-plugin-cypress/src/plugin.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ function getSkippableTests (isSuitesSkippingEnabled, tracer, testConfiguration)
119119
}
120120

121121
module.exports = (on, config) => {
122+
let isTestsSkipped = false
122123
const tracer = require('../../dd-trace')
123124
const testEnvironmentMetadata = getTestEnvironmentMetadata(TEST_FRAMEWORK_NAME)
124125

@@ -306,7 +307,7 @@ module.exports = (on, config) => {
306307
testSessionSpan,
307308
testModuleSpan,
308309
{
309-
isSuitesSkipped: !!testsToSkip.length,
310+
isSuitesSkipped: isTestsSkipped,
310311
isSuitesSkippingEnabled,
311312
isCodeCoverageEnabled
312313
}
@@ -352,6 +353,7 @@ module.exports = (on, config) => {
352353
if (testsToSkip.find(test => {
353354
return testName === test.name && testSuite === test.suite
354355
})) {
356+
isTestsSkipped = true
355357
return { shouldSkip: true }
356358
}
357359

0 commit comments

Comments
 (0)