Skip to content

Commit a6b67c3

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Remove expected traces count param from CI Vis snapshot tests (#8311)
1 parent 57a74ad commit a6b67c3

File tree

17 files changed

+446
-416
lines changed

17 files changed

+446
-416
lines changed

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

+38-2
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import datadog.trace.civisibility.test.ExecutionStrategy
4343
import datadog.trace.civisibility.utils.ConcurrentHashMapContextStore
4444
import datadog.trace.civisibility.writer.ddintake.CiTestCovMapperV2
4545
import datadog.trace.civisibility.writer.ddintake.CiTestCycleMapperV1
46+
import datadog.trace.common.writer.ListWriter
4647
import datadog.trace.common.writer.RemoteMapper
4748
import datadog.trace.core.DDSpan
4849
import org.msgpack.jackson.dataformat.MessagePackFactory
@@ -52,6 +53,8 @@ import java.nio.ByteBuffer
5253
import java.nio.file.Files
5354
import java.nio.file.Path
5455
import java.nio.file.Paths
56+
import java.util.concurrent.TimeUnit
57+
import java.util.function.Predicate
5558

5659
abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
5760

@@ -216,6 +219,35 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
216219
"RANDOM"
217220
}
218221

222+
final ListWriter.Filter spanFilter = new ListWriter.Filter() {
223+
private final Object lock = new Object()
224+
private Collection<DDSpan> spans = new ArrayList<>()
225+
226+
@Override
227+
boolean accept(List<DDSpan> trace) {
228+
synchronized (lock) {
229+
spans.addAll(trace)
230+
lock.notifyAll()
231+
}
232+
return true
233+
}
234+
235+
boolean waitForSpan(Predicate<DDSpan> predicate, long timeoutMillis) {
236+
long deadline = System.currentTimeMillis() + timeoutMillis
237+
synchronized (lock) {
238+
while (!spans.stream().anyMatch(predicate)) {
239+
def timeLeft = deadline - System.currentTimeMillis()
240+
if (timeLeft > 0) {
241+
lock.wait(timeLeft)
242+
} else {
243+
return false
244+
}
245+
}
246+
return true
247+
}
248+
}
249+
}
250+
219251
@Override
220252
void setup() {
221253
skippableTests.clear()
@@ -226,6 +258,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
226258
flakyRetryEnabled = false
227259
earlyFlakinessDetectionEnabled = false
228260
impactedTestsDetectionEnabled = false
261+
262+
TEST_WRITER.setFilter(spanFilter)
229263
}
230264

231265
def givenSkippableTests(List<TestIdentifier> tests) {
@@ -280,8 +314,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
280314
Files.deleteIfExists(agentKeyFile)
281315
}
282316

283-
def assertSpansData(String testcaseName, int expectedTracesCount, Map<String, String> replacements = [:]) {
284-
TEST_WRITER.waitForTraces(expectedTracesCount)
317+
def assertSpansData(String testcaseName, Map<String, String> replacements = [:]) {
318+
Predicate<DDSpan> sessionSpan = span -> span.spanType == "test_session_end"
319+
spanFilter.waitForSpan(sessionSpan, TimeUnit.SECONDS.toMillis(20))
320+
285321
def traces = TEST_WRITER.toList()
286322

287323
def events = getEventsAsJson(traces)

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy

+15-11
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ import com.jayway.jsonpath.JsonPath
77
import com.jayway.jsonpath.Option
88
import com.jayway.jsonpath.ReadContext
99
import com.jayway.jsonpath.WriteContext
10+
import freemarker.core.Environment
1011
import freemarker.core.InvalidReferenceException
1112
import freemarker.template.Template
13+
import freemarker.template.TemplateException
1214
import freemarker.template.TemplateExceptionHandler
1315
import org.skyscreamer.jsonassert.JSONAssert
1416
import org.skyscreamer.jsonassert.JSONCompareMode
@@ -114,11 +116,22 @@ abstract class CiVisibilityTestUtils {
114116
}
115117
}
116118

119+
static final TemplateExceptionHandler SUPPRESS_EXCEPTION_HANDLER = new TemplateExceptionHandler() {
120+
@Override
121+
void handleTemplateException(TemplateException e, Environment environment, Writer writer) throws TemplateException {
122+
if (e instanceof InvalidReferenceException) {
123+
writer.write('"<VALUE_MISSING>"')
124+
} else {
125+
throw e
126+
}
127+
}
128+
}
129+
117130
static final freemarker.template.Configuration FREEMARKER = new freemarker.template.Configuration(freemarker.template.Configuration.VERSION_2_3_30) { {
118131
setClassLoaderForTemplateLoading(CiVisibilityTestUtils.classLoader, "")
119132
setDefaultEncoding("UTF-8")
120-
setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER)
121-
setLogTemplateExceptions(true)
133+
setTemplateExceptionHandler(SUPPRESS_EXCEPTION_HANDLER)
134+
setLogTemplateExceptions(false)
122135
setWrapUncheckedExceptions(true)
123136
setFallbackOnNullLoopVariable(false)
124137
setNumberFormat("0.######")
@@ -131,15 +144,6 @@ abstract class CiVisibilityTestUtils {
131144
StringWriter coveragesOut = new StringWriter()
132145
coveragesTemplate.process(replacements, coveragesOut)
133146
return coveragesOut.toString()
134-
} catch (InvalidReferenceException e) {
135-
def missingExpression = e.getBlamedExpressionString()
136-
if (missingExpression != null) {
137-
replacements.put(missingExpression, "<VALUE_MISSING>") // to simplify debugging failures
138-
return getFreemarkerTemplate(templatePath, replacements, replacementsSource)
139-
140-
} else {
141-
throw new RuntimeException("Could not get Freemarker template " + templatePath + "; replacements map: " + replacements + "; replacements source: " + replacementsSource, e)
142-
}
143147

144148
} catch (Exception e) {
145149
throw new RuntimeException("Could not get Freemarker template " + templatePath + "; replacements map: " + replacements + "; replacements source: " + replacementsSource, e)

‎dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

+24-24
Original file line numberDiff line numberDiff line change
@@ -17,37 +17,37 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
1717
def "test #testcaseName"() {
1818
runFeatures(features)
1919

20-
assertSpansData(testcaseName, expectedTracesCount)
20+
assertSpansData(testcaseName)
2121

2222
where:
23-
testcaseName | features | expectedTracesCount
24-
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2
25-
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | 5
26-
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 2
23+
testcaseName | features
24+
"test-succeed" | ["org/example/cucumber/calculator/basic_arithmetic.feature"]
25+
"test-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"]
26+
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]
2727
"test-multiple-features-${version()}" | [
2828
"org/example/cucumber/calculator/basic_arithmetic.feature",
2929
"org/example/cucumber/calculator/basic_arithmetic_failed.feature"
30-
] | 3
31-
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"] | 2
32-
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"] | 2
30+
]
31+
"test-name-with-brackets" | ["org/example/cucumber/calculator/name_with_brackets.feature"]
32+
"test-empty-name-${version()}" | ["org/example/cucumber/calculator/empty_scenario_name.feature"]
3333
}
3434

3535
def "test ITR #testcaseName"() {
3636
givenSkippableTests(skippedTests)
3737

3838
runFeatures(features)
3939

40-
assertSpansData(testcaseName, expectedTracesCount)
40+
assertSpansData(testcaseName)
4141

4242
where:
43-
testcaseName | features | expectedTracesCount | skippedTests
44-
"test-itr-skipping" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2 | [
43+
testcaseName | features | skippedTests
44+
"test-itr-skipping" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [
4545
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null)
4646
]
47-
"test-itr-unskippable" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"] | 2 | [
47+
"test-itr-unskippable" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable.feature"] | [
4848
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_unskippable.feature:Basic Arithmetic", "Addition", null)
4949
]
50-
"test-itr-unskippable-suite" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"] | 2 | [
50+
"test-itr-unskippable-suite" | ["org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature"] | [
5151
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_unskippable_suite.feature:Basic Arithmetic", "Addition", null)
5252
]
5353
}
@@ -58,15 +58,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
5858

5959
runFeatures(features)
6060

61-
assertSpansData(testcaseName, expectedTracesCount)
61+
assertSpansData(testcaseName)
6262

6363
where:
64-
testcaseName | features | expectedTracesCount | retriedTests
65-
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 2 | []
66-
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | 6 | [
64+
testcaseName | features | retriedTests
65+
"test-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
66+
"test-retry-failure" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
6767
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
6868
]
69-
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | 5 | [
69+
"test-retry-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature"] | [
7070
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_with_examples_failed.feature:Basic Arithmetic With Examples", "Many additions", null)
7171
]
7272
}
@@ -77,16 +77,16 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
7777

7878
runFeatures(features)
7979

80-
assertSpansData(testcaseName, expectedTracesCount)
80+
assertSpansData(testcaseName)
8181

8282
where:
83-
testcaseName | features | expectedTracesCount | knownTestsList
84-
"test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 2 | [
83+
testcaseName | features | knownTestsList
84+
"test-efd-known-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | [
8585
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", "Addition", null)
8686
]
87-
"test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | 4 | []
88-
"test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | 9 | []
89-
"test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | 3 | []
87+
"test-efd-new-test" | ["org/example/cucumber/calculator/basic_arithmetic.feature"] | []
88+
"test-efd-new-scenario-outline-${version()}" | ["org/example/cucumber/calculator/basic_arithmetic_with_examples.feature"] | []
89+
"test-efd-new-slow-test" | ["org/example/cucumber/calculator/basic_arithmetic_slow.feature"] | []
9090
}
9191

9292
private String version() {

‎dd-java-agent/instrumentation/junit-4.10/junit-4.13/src/test/groovy/JUnit413Test.groovy

+11-11
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,19 @@ class JUnit413Test extends CiVisibilityInstrumentationTest {
2121
def "test #testcaseName"() {
2222
runTests(tests)
2323

24-
assertSpansData(testcaseName, expectedTracesCount)
24+
assertSpansData(testcaseName)
2525

2626
where:
27-
testcaseName | tests | expectedTracesCount
28-
"test-succeed-before-after" | [TestSucceedBeforeAfter] | 3
29-
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass] | 3
30-
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam] | 2
31-
"test-failed-before-class" | [TestFailedBeforeClass] | 1
32-
"test-failed-after-class" | [TestFailedAfterClass] | 3
33-
"test-failed-before" | [TestFailedBefore] | 3
34-
"test-failed-after" | [TestFailedAfter] | 3
35-
"test-failed-before-param" | [TestFailedBeforeParam] | 2
36-
"test-failed-after-param" | [TestFailedAfterParam] | 2
27+
testcaseName | tests
28+
"test-succeed-before-after" | [TestSucceedBeforeAfter]
29+
"test-succeed-before-class-after-class" | [TestSucceedBeforeClassAfterClass]
30+
"test-succeed-before-param-after-param" | [TestSucceedBeforeParamAfterParam]
31+
"test-failed-before-class" | [TestFailedBeforeClass]
32+
"test-failed-after-class" | [TestFailedAfterClass]
33+
"test-failed-before" | [TestFailedBefore]
34+
"test-failed-after" | [TestFailedAfter]
35+
"test-failed-before-param" | [TestFailedBeforeParam]
36+
"test-failed-after-param" | [TestFailedAfterParam]
3737
}
3838

3939
private void runTests(Collection<Class<?>> tests) {

‎dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy

+23-23
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,14 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
2222
def "test #testcaseName"() {
2323
runTests(tests)
2424

25-
assertSpansData(testcaseName, expectedTracesCount)
25+
assertSpansData(testcaseName)
2626

2727
where:
28-
testcaseName | tests | expectedTracesCount
29-
"test-succeed" | [TestSucceedMUnit] | 2
30-
"test-skipped" | [TestSkippedMUnit] | 2
31-
"test-skipped-suite" | [TestSkippedSuiteMUnit] | 3
32-
"test-failed-assumption-${version()}" | [TestFailedAssumptionMUnit] | 2
28+
testcaseName | tests
29+
"test-succeed" | [TestSucceedMUnit]
30+
"test-skipped" | [TestSkippedMUnit]
31+
"test-skipped-suite" | [TestSkippedSuiteMUnit]
32+
"test-failed-assumption-${version()}" | [TestFailedAssumptionMUnit]
3333
}
3434

3535
def "test flaky retries #testcaseName"() {
@@ -38,13 +38,13 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
3838

3939
runTests(tests)
4040

41-
assertSpansData(testcaseName, expectedTracesCount)
41+
assertSpansData(testcaseName)
4242

4343
where:
44-
testcaseName | tests | expectedTracesCount | retriedTests
45-
"test-failed" | [TestFailedMUnit] | 2 | []
46-
"test-retry-failed" | [TestFailedMUnit] | 6 | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
47-
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | 4 | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
44+
testcaseName | tests | retriedTests
45+
"test-failed" | [TestFailedMUnit] | []
46+
"test-retry-failed" | [TestFailedMUnit] | [new TestIdentifier("org.example.TestFailedMUnit", "Calculator.add", null)]
47+
"test-failed-then-succeed" | [TestFailedThenSucceedMUnit] | [new TestIdentifier("org.example.TestFailedThenSucceedMUnit", "Calculator.add", null)]
4848
}
4949

5050
def "test early flakiness detection #testcaseName"() {
@@ -53,13 +53,13 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
5353

5454
runTests(tests)
5555

56-
assertSpansData(testcaseName, expectedTracesCount)
56+
assertSpansData(testcaseName)
5757

5858
where:
59-
testcaseName | tests | expectedTracesCount | knownTestsList
60-
"test-efd-known-test" | [TestSucceedMUnit] | 2 | [new TestIdentifier("org.example.TestSucceedMUnit", "Calculator.add", null)]
61-
"test-efd-new-test" | [TestSucceedMUnit] | 4 | []
62-
"test-efd-new-slow-test" | [TestSucceedMUnitSlow] | 3 | [] // is executed only twice
59+
testcaseName | tests | knownTestsList
60+
"test-efd-known-test" | [TestSucceedMUnit] | [new TestIdentifier("org.example.TestSucceedMUnit", "Calculator.add", null)]
61+
"test-efd-new-test" | [TestSucceedMUnit] | []
62+
"test-efd-new-slow-test" | [TestSucceedMUnitSlow] | [] // is executed only twice
6363
}
6464

6565
def "test impacted tests detection #testcaseName"() {
@@ -68,15 +68,15 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
6868

6969
runTests(tests)
7070

71-
assertSpansData(testcaseName, expectedTracesCount)
71+
assertSpansData(testcaseName)
7272

7373
where:
74-
testcaseName | tests | expectedTracesCount | prDiff
75-
"test-succeed" | [TestSucceedMUnit] | 2 | LineDiff.EMPTY
76-
"test-succeed" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet())
77-
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
78-
"test-succeed" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
79-
"test-succeed-impacted" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
74+
testcaseName | tests | prDiff
75+
"test-succeed" | [TestSucceedMUnit] | LineDiff.EMPTY
76+
"test-succeed" | [TestSucceedMUnit] | new FileDiff(new HashSet())
77+
"test-succeed-impacted" | [TestSucceedMUnit] | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
78+
"test-succeed" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
79+
"test-succeed-impacted" | [TestSucceedMUnit] | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
8080
}
8181

8282
private void runTests(Collection<Class<?>> tests) {

0 commit comments

Comments
 (0)