Skip to content

Commit 9e2a709

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
committed
Do strict JSON comparison when running CI Vis instrumentation tests locally
1 parent c80ed84 commit 9e2a709

File tree

31 files changed

+450
-10
lines changed

31 files changed

+450
-10
lines changed

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy

+13-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@ import com.jayway.jsonpath.JsonPath
77
import com.jayway.jsonpath.Option
88
import com.jayway.jsonpath.ReadContext
99
import com.jayway.jsonpath.WriteContext
10+
import datadog.trace.api.Config
11+
import datadog.trace.civisibility.ci.CIProviderInfoFactory
12+
import datadog.trace.civisibility.ci.GitLabInfo
13+
import datadog.trace.civisibility.ci.GithubActionsInfo
14+
import datadog.trace.civisibility.ci.env.CiEnvironment
15+
import datadog.trace.civisibility.ci.env.CiEnvironmentImpl
1016
import freemarker.core.Environment
1117
import freemarker.core.InvalidReferenceException
1218
import freemarker.template.Template
@@ -87,18 +93,23 @@ abstract class CiVisibilityTestUtils {
8793
replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"")
8894
}
8995

96+
def environment = System.getenv()
97+
def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null
98+
def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE
99+
90100
def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events)
91101
def actualEvents = JSON_MAPPER.writeValueAsString(events)
102+
92103
try {
93-
JSONAssert.assertEquals(expectedEvents, actualEvents, JSONCompareMode.LENIENT)
104+
JSONAssert.assertEquals(expectedEvents, actualEvents, comparisonMode)
94105
} catch (AssertionError e) {
95106
throw new org.opentest4j.AssertionFailedError("Events mismatch", expectedEvents, actualEvents, e)
96107
}
97108

98109
def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages)
99110
def actualCoverages = JSON_MAPPER.writeValueAsString(coverages)
100111
try {
101-
JSONAssert.assertEquals(expectedCoverages, actualCoverages, JSONCompareMode.LENIENT)
112+
JSONAssert.assertEquals(expectedCoverages, actualCoverages, comparisonMode)
102113
} catch (AssertionError e) {
103114
throw new org.opentest4j.AssertionFailedError("Coverages mismatch", expectedCoverages, actualCoverages, e)
104115
}

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy

+3-3
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
6969

7070
where:
7171
testcaseName | success | features | retriedTests
72-
"test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
72+
"test-no-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
7373
"test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
7474
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
7575
]
@@ -148,13 +148,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
148148
assertSpansData(testcaseName)
149149

150150
where:
151-
testcaseName | features | quarantined | known
151+
testcaseName | features | quarantined | known
152152
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
153153
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
154154
] | [
155155
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
156156
]
157-
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
157+
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
158158
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
159159
] | []
160160
}

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/coverages.ftl

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[ {
2+
"files" : [ {
3+
"filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature"
4+
} ],
5+
"span_id" : ${content_span_id},
6+
"test_session_id" : ${content_test_session_id},
7+
"test_suite_id" : ${content_test_suite_id}
8+
} ]

‎dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/events.ftl

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
[ {
2+
"content" : {
3+
"duration" : ${content_duration},
4+
"error" : 0,
5+
"meta" : {
6+
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"component" : "junit",
8+
"dummy_ci_tag" : "dummy_ci_tag_value",
9+
"env" : "none",
10+
"library_version" : ${content_meta_library_version},
11+
"span.kind" : "test_suite_end",
12+
"test.framework" : "cucumber",
13+
"test.framework_version" : ${content_meta_test_framework_version},
14+
"test.module" : "cucumber-junit-5",
15+
"test.status" : "fail",
16+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
17+
"test.type" : "test",
18+
"test_session.name" : "session-name"
19+
},
20+
"metrics" : {
21+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
22+
},
23+
"name" : "junit.test_suite",
24+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
25+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
26+
"start" : ${content_start},
27+
"test_module_id" : ${content_test_module_id},
28+
"test_session_id" : ${content_test_session_id},
29+
"test_suite_id" : ${content_test_suite_id}
30+
},
31+
"type" : "test_suite_end",
32+
"version" : 1
33+
}, {
34+
"content" : {
35+
"duration" : ${content_duration_2},
36+
"error" : 1,
37+
"meta" : {
38+
"_dd.profiling.ctx" : "test",
39+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
40+
"component" : "junit",
41+
"dummy_ci_tag" : "dummy_ci_tag_value",
42+
"env" : "none",
43+
"error.message" : ${content_meta_error_message},
44+
"error.stack" : ${content_meta_error_stack},
45+
"error.type" : "java.lang.AssertionError",
46+
"language" : "jvm",
47+
"library_version" : ${content_meta_library_version},
48+
"runtime-id" : ${content_meta_runtime_id},
49+
"span.kind" : "test",
50+
"test.framework" : "cucumber",
51+
"test.framework_version" : ${content_meta_test_framework_version},
52+
"test.module" : "cucumber-junit-5",
53+
"test.name" : "Addition",
54+
"test.status" : "fail",
55+
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
56+
"test.traits" : "{\"category\":[\"foo\"]}",
57+
"test.type" : "test",
58+
"test_session.name" : "session-name"
59+
},
60+
"metrics" : {
61+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
62+
"_dd.profiling.enabled" : 0,
63+
"_dd.trace_span_attribute_schema" : 0,
64+
"process_id" : ${content_metrics_process_id}
65+
},
66+
"name" : "junit.test",
67+
"parent_id" : ${content_parent_id},
68+
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
69+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
70+
"span_id" : ${content_span_id},
71+
"start" : ${content_start_2},
72+
"test_module_id" : ${content_test_module_id},
73+
"test_session_id" : ${content_test_session_id},
74+
"test_suite_id" : ${content_test_suite_id},
75+
"trace_id" : ${content_trace_id}
76+
},
77+
"type" : "test",
78+
"version" : 2
79+
}, {
80+
"content" : {
81+
"duration" : ${content_duration_3},
82+
"error" : 0,
83+
"meta" : {
84+
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
85+
"_dd.profiling.ctx" : "test",
86+
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
87+
"component" : "junit",
88+
"dummy_ci_tag" : "dummy_ci_tag_value",
89+
"env" : "none",
90+
"language" : "jvm",
91+
"library_version" : ${content_meta_library_version},
92+
"runtime-id" : ${content_meta_runtime_id},
93+
"span.kind" : "test_session_end",
94+
"test.command" : "cucumber-junit-5",
95+
"test.framework" : "cucumber",
96+
"test.framework_version" : ${content_meta_test_framework_version},
97+
"test.status" : "fail",
98+
"test.type" : "test",
99+
"test_session.name" : "session-name"
100+
},
101+
"metrics" : {
102+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
103+
"_dd.profiling.enabled" : 0,
104+
"_dd.trace_span_attribute_schema" : 0,
105+
"process_id" : ${content_metrics_process_id}
106+
},
107+
"name" : "junit.test_session",
108+
"resource" : "cucumber-junit-5",
109+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
110+
"start" : ${content_start_3},
111+
"test_session_id" : ${content_test_session_id}
112+
},
113+
"type" : "test_session_end",
114+
"version" : 1
115+
}, {
116+
"content" : {
117+
"duration" : ${content_duration_4},
118+
"error" : 0,
119+
"meta" : {
120+
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
121+
"component" : "junit",
122+
"dummy_ci_tag" : "dummy_ci_tag_value",
123+
"env" : "none",
124+
"library_version" : ${content_meta_library_version},
125+
"span.kind" : "test_module_end",
126+
"test.framework" : "cucumber",
127+
"test.framework_version" : ${content_meta_test_framework_version},
128+
"test.module" : "cucumber-junit-5",
129+
"test.status" : "fail",
130+
"test.type" : "test",
131+
"test_session.name" : "session-name"
132+
},
133+
"metrics" : {
134+
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
135+
},
136+
"name" : "junit.test_module",
137+
"resource" : "cucumber-junit-5",
138+
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
139+
"start" : ${content_start_4},
140+
"test_module_id" : ${content_test_module_id},
141+
"test_session_id" : ${content_test_session_id}
142+
},
143+
"type" : "test_module_end",
144+
"version" : 1
145+
} ]

‎dd-smoke-tests/gradle/src/test/resources/test-corrupted-config-legacy-instrumentation/events.ftl

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"error" : 1,
55
"meta" : {
66
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"_dd.test.is_user_provided_service" : "true",
78
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
89
"ci.workspace_path" : ${content_meta_ci_workspace_path},
910
"component" : "gradle",

‎dd-smoke-tests/gradle/src/test/resources/test-corrupted-config-new-instrumentation/events.ftl

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"error" : 1,
55
"meta" : {
66
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"_dd.test.is_user_provided_service" : "true",
78
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
89
"ci.workspace_path" : ${content_meta_ci_workspace_path},
910
"component" : "gradle",

‎dd-smoke-tests/gradle/src/test/resources/test-failed-flaky-retries/events.ftl

+15
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"error" : 1,
55
"meta" : {
66
"_dd.p.tid" : ${content_meta__dd_p_tid},
7+
"_dd.test.is_user_provided_service" : "true",
78
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
89
"ci.workspace_path" : ${content_meta_ci_workspace_path},
910
"component" : "gradle",
@@ -55,6 +56,7 @@
5556
"error" : 1,
5657
"meta" : {
5758
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
59+
"_dd.test.is_user_provided_service" : "true",
5860
"ci.workspace_path" : ${content_meta_ci_workspace_path},
5961
"component" : "gradle",
6062
"env" : "integration-test",
@@ -103,6 +105,7 @@
103105
"error" : 0,
104106
"meta" : {
105107
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
108+
"_dd.test.is_user_provided_service" : "true",
106109
"env" : "integration-test",
107110
"language" : "jvm",
108111
"library_version" : ${content_meta_library_version},
@@ -131,6 +134,7 @@
131134
"error" : 0,
132135
"meta" : {
133136
"_dd.p.tid" : ${content_meta__dd_p_tid_4},
137+
"_dd.test.is_user_provided_service" : "true",
134138
"env" : "integration-test",
135139
"language" : "jvm",
136140
"library_version" : ${content_meta_library_version},
@@ -159,6 +163,7 @@
159163
"error" : 0,
160164
"meta" : {
161165
"_dd.p.tid" : ${content_meta__dd_p_tid_5},
166+
"_dd.test.is_user_provided_service" : "true",
162167
"env" : "integration-test",
163168
"language" : "jvm",
164169
"library_version" : ${content_meta_library_version},
@@ -187,6 +192,7 @@
187192
"error" : 0,
188193
"meta" : {
189194
"_dd.p.tid" : ${content_meta__dd_p_tid_6},
195+
"_dd.test.is_user_provided_service" : "true",
190196
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
191197
"ci.workspace_path" : ${content_meta_ci_workspace_path},
192198
"component" : "junit",
@@ -234,6 +240,7 @@
234240
"error" : 1,
235241
"meta" : {
236242
"_dd.p.tid" : ${content_meta__dd_p_tid_7},
243+
"_dd.test.is_user_provided_service" : "true",
237244
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
238245
"ci.workspace_path" : ${content_meta_ci_workspace_path},
239246
"component" : "junit",
@@ -288,6 +295,7 @@
288295
"error" : 1,
289296
"meta" : {
290297
"_dd.p.tid" : ${content_meta__dd_p_tid_8},
298+
"_dd.test.is_user_provided_service" : "true",
291299
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
292300
"ci.workspace_path" : ${content_meta_ci_workspace_path},
293301
"component" : "junit",
@@ -344,6 +352,7 @@
344352
"error" : 1,
345353
"meta" : {
346354
"_dd.p.tid" : ${content_meta__dd_p_tid_9},
355+
"_dd.test.is_user_provided_service" : "true",
347356
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
348357
"ci.workspace_path" : ${content_meta_ci_workspace_path},
349358
"component" : "junit",
@@ -400,6 +409,7 @@
400409
"error" : 1,
401410
"meta" : {
402411
"_dd.p.tid" : ${content_meta__dd_p_tid_10},
412+
"_dd.test.is_user_provided_service" : "true",
403413
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
404414
"ci.workspace_path" : ${content_meta_ci_workspace_path},
405415
"component" : "junit",
@@ -456,6 +466,7 @@
456466
"error" : 1,
457467
"meta" : {
458468
"_dd.p.tid" : ${content_meta__dd_p_tid_11},
469+
"_dd.test.is_user_provided_service" : "true",
459470
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
460471
"ci.workspace_path" : ${content_meta_ci_workspace_path},
461472
"component" : "junit",
@@ -474,6 +485,7 @@
474485
"span.kind" : "test",
475486
"test.framework" : "junit4",
476487
"test.framework_version" : "4.10",
488+
"test.has_failed_all_retries" : "true",
477489
"test.is_retry" : "true",
478490
"test.module" : ":test",
479491
"test.name" : "test_failed",
@@ -512,6 +524,7 @@
512524
"error" : 0,
513525
"meta" : {
514526
"_dd.p.tid" : ${content_meta__dd_p_tid_12},
527+
"_dd.test.is_user_provided_service" : "true",
515528
"env" : "integration-test",
516529
"language" : "jvm",
517530
"library_version" : ${content_meta_library_version},
@@ -540,6 +553,7 @@
540553
"error" : 0,
541554
"meta" : {
542555
"_dd.p.tid" : ${content_meta__dd_p_tid_13},
556+
"_dd.test.is_user_provided_service" : "true",
543557
"env" : "integration-test",
544558
"language" : "jvm",
545559
"library_version" : ${content_meta_library_version},
@@ -568,6 +582,7 @@
568582
"error" : 0,
569583
"meta" : {
570584
"_dd.p.tid" : ${content_meta__dd_p_tid_14},
585+
"_dd.test.is_user_provided_service" : "true",
571586
"env" : "integration-test",
572587
"language" : "jvm",
573588
"library_version" : ${content_meta_library_version},

0 commit comments

Comments
 (0)