Skip to content

Commit

Permalink
Do strict JSON comparison when running CI Vis instrumentation tests l…
Browse files Browse the repository at this point in the history
…ocally (#8327)
  • Loading branch information
nikita-tkachenko-datadog authored Feb 3, 2025
1 parent 38b8211 commit 5d18c24
Show file tree
Hide file tree
Showing 37 changed files with 795 additions and 121 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -341,13 +341,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
] + replacements

// uncomment to generate expected data templates
// def clazz = this.getClass()
// def resourceName = clazz.name.replace('.', '/') + ".class"
// def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
// def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
// def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
// CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
// return [:]
// def clazz = this.getClass()
// def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
// def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
// def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
// def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
// CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
// return [:]

return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import com.jayway.jsonpath.JsonPath
import com.jayway.jsonpath.Option
import com.jayway.jsonpath.ReadContext
import com.jayway.jsonpath.WriteContext
import datadog.trace.api.Config
import datadog.trace.civisibility.ci.CIProviderInfoFactory
import datadog.trace.civisibility.ci.GitLabInfo
import datadog.trace.civisibility.ci.GithubActionsInfo
import datadog.trace.civisibility.ci.env.CiEnvironment
import datadog.trace.civisibility.ci.env.CiEnvironmentImpl
import freemarker.core.Environment
import freemarker.core.InvalidReferenceException
import freemarker.template.Template
Expand Down Expand Up @@ -87,18 +93,23 @@ abstract class CiVisibilityTestUtils {
replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"")
}

def environment = System.getenv()
def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null
def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE

def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events)
def actualEvents = JSON_MAPPER.writeValueAsString(events)

try {
JSONAssert.assertEquals(expectedEvents, actualEvents, JSONCompareMode.LENIENT)
JSONAssert.assertEquals(expectedEvents, actualEvents, comparisonMode)
} catch (AssertionError e) {
throw new org.opentest4j.AssertionFailedError("Events mismatch", expectedEvents, actualEvents, e)
}

def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages)
def actualCoverages = JSON_MAPPER.writeValueAsString(coverages)
try {
JSONAssert.assertEquals(expectedCoverages, actualCoverages, JSONCompareMode.LENIENT)
JSONAssert.assertEquals(expectedCoverages, actualCoverages, comparisonMode)
} catch (AssertionError e) {
throw new org.opentest4j.AssertionFailedError("Coverages mismatch", expectedCoverages, actualCoverages, e)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {

where:
testcaseName | success | features | retriedTests
"test-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-no-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | []
"test-retry-failed" | false | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
Expand Down Expand Up @@ -148,13 +148,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
assertSpansData(testcaseName)

where:
testcaseName | features | quarantined | known
testcaseName | features | quarantined | known
"test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
]
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
"test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
] | []
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public void a_calculator_I_just_turned_on() {

@Given("a flaky calculator I just turned on")
public void a_flaky_calculator_I_just_turned_on() {
calc = ++flakyCounter >= 3 ? new Calculator() : null;
calc = new Calculator(++flakyCounter < 3);
}

@When("I add {int} and {int}")
Expand Down Expand Up @@ -94,6 +94,15 @@ public void setOperation(String operation) {
static class Calculator {
private static final List<String> OPS = asList("-", "+", "*", "/");
private final Deque<Number> stack = new LinkedList<>();
private final boolean broken;

public Calculator() {
this(false);
}

public Calculator(boolean broken) {
this.broken = broken;
}

public void push(Object arg) {
if (OPS.contains(arg)) {
Expand All @@ -116,6 +125,9 @@ public void push(Object arg) {
}

public Number value() {
if (broken) {
throw new RuntimeException("This test is flaky");
}
return stack.getLast();
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"error.message" : ${content_meta_error_message},
"error.stack" : ${content_meta_error_stack},
"error.type" : "java.lang.NullPointerException",
"error.type" : "java.lang.RuntimeException",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
Expand Down Expand Up @@ -85,8 +86,9 @@
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"error.message" : ${content_meta_error_message},
"error.stack" : ${content_meta_error_stack_2},
"error.type" : "java.lang.NullPointerException",
"error.type" : "java.lang.RuntimeException",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[ {
"files" : [ {
"filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature"
} ],
"span_id" : ${content_span_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id}
} ]
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
[ {
"content" : {
"duration" : ${content_duration},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"library_version" : ${content_meta_library_version},
"span.kind" : "test_suite_end",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.status" : "fail",
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
},
"name" : "junit.test_suite",
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id}
},
"type" : "test_suite_end",
"version" : 1
}, {
"content" : {
"duration" : ${content_duration_2},
"error" : 1,
"meta" : {
"_dd.profiling.ctx" : "test",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"error.message" : ${content_meta_error_message},
"error.stack" : ${content_meta_error_stack},
"error.type" : "java.lang.AssertionError",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
"span.kind" : "test",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.name" : "Addition",
"test.status" : "fail",
"test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
"test.traits" : "{\"category\":[\"foo\"]}",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
"_dd.profiling.enabled" : 0,
"_dd.trace_span_attribute_schema" : 0,
"process_id" : ${content_metrics_process_id}
},
"name" : "junit.test",
"parent_id" : ${content_parent_id},
"resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"span_id" : ${content_span_id},
"start" : ${content_start_2},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id},
"test_suite_id" : ${content_test_suite_id},
"trace_id" : ${content_trace_id}
},
"type" : "test",
"version" : 2
}, {
"content" : {
"duration" : ${content_duration_3},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_2},
"_dd.profiling.ctx" : "test",
"_dd.tracer_host" : ${content_meta__dd_tracer_host},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"language" : "jvm",
"library_version" : ${content_meta_library_version},
"runtime-id" : ${content_meta_runtime_id},
"span.kind" : "test_session_end",
"test.command" : "cucumber-junit-5",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.status" : "fail",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
"_dd.profiling.enabled" : 0,
"_dd.trace_span_attribute_schema" : 0,
"process_id" : ${content_metrics_process_id}
},
"name" : "junit.test_session",
"resource" : "cucumber-junit-5",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start_3},
"test_session_id" : ${content_test_session_id}
},
"type" : "test_session_end",
"version" : 1
}, {
"content" : {
"duration" : ${content_duration_4},
"error" : 0,
"meta" : {
"_dd.p.tid" : ${content_meta__dd_p_tid_3},
"component" : "junit",
"dummy_ci_tag" : "dummy_ci_tag_value",
"env" : "none",
"library_version" : ${content_meta_library_version},
"span.kind" : "test_module_end",
"test.framework" : "cucumber",
"test.framework_version" : ${content_meta_test_framework_version},
"test.module" : "cucumber-junit-5",
"test.status" : "fail",
"test.type" : "test",
"test_session.name" : "session-name"
},
"metrics" : {
"_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
},
"name" : "junit.test_module",
"resource" : "cucumber-junit-5",
"service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
"start" : ${content_start_4},
"test_module_id" : ${content_test_module_id},
"test_session_id" : ${content_test_session_id}
},
"type" : "test_module_end",
"version" : 1
} ]
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[ ]
Loading

0 comments on commit 5d18c24

Please sign in to comment.