diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index e47c51c99e86..e9362935bf44 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -24,6 +24,7 @@ import datadog.trace.civisibility.coverage.file.FileCoverageStore import datadog.trace.civisibility.coverage.percentage.NoOpCoverageCalculator import datadog.trace.civisibility.decorator.TestDecorator import datadog.trace.civisibility.decorator.TestDecoratorImpl +import datadog.trace.civisibility.diff.Diff import datadog.trace.civisibility.diff.LineDiff import datadog.trace.civisibility.domain.BuildSystemSession import datadog.trace.civisibility.domain.TestFrameworkModule @@ -52,7 +53,6 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -// FIXME nikita: add integration/smoke tests abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { static String dummyModule @@ -70,9 +70,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { private static final List skippableTests = new ArrayList<>() private static final List flakyTests = new ArrayList<>() private static final List knownTests = new ArrayList<>() + private static volatile Diff diff = LineDiff.EMPTY + private static volatile boolean itrEnabled = false private static volatile boolean flakyRetryEnabled = false private static volatile boolean earlyFlakinessDetectionEnabled = false + private static volatile boolean impactedTestsDetectionEnabled = false + public static final int SLOW_TEST_THRESHOLD_MILLIS = 1000 public static final int VERY_SLOW_TEST_THRESHOLD_MILLIS = 2000 @@ -118,14 +122,14 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { false, itrEnabled, flakyRetryEnabled, - false, + impactedTestsDetectionEnabled, earlyFlakinessDetectionSettings, itrEnabled ? "itrCorrelationId" : null, skippableTestsWithMetadata, [:], flakyTests, earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null, - LineDiff.EMPTY) + diff) } } @@ -202,8 +206,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { TestFrameworkSession testSession = testFrameworkSessionFactory.startSession(dummyModule, component, null) TestFrameworkModule testModule = testSession.testModuleStart(dummyModule, null) new TestEventsHandlerImpl(metricCollector, testSession, testModule, - suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(), - testStore != null ? testStore : new ConcurrentHashMapContextStore<>()) + suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(), + testStore != null ? testStore : new ConcurrentHashMapContextStore<>()) } } @@ -217,9 +221,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { skippableTests.clear() flakyTests.clear() knownTests.clear() + diff = LineDiff.EMPTY itrEnabled = false flakyRetryEnabled = false earlyFlakinessDetectionEnabled = false + impactedTestsDetectionEnabled = false } def givenSkippableTests(List tests) { @@ -235,6 +241,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { knownTests.addAll(tests) } + def givenDiff(Diff diff) { + this.diff = diff + } + def givenFlakyRetryEnabled(boolean flakyRetryEnabled) { this.flakyRetryEnabled = flakyRetryEnabled } @@ -243,6 +253,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled } + def givenImpactedTestsDetectionEnabled(boolean impactedTestsDetectionEnabled) { + this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled + } + def givenTestsOrder(String testsOrder) { injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder) } @@ -351,4 +365,12 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { abstract String instrumentedLibraryName() abstract String instrumentedLibraryVersion() + + BitSet lines(int ... setBits) { + BitSet bitSet = new BitSet() + for (int bit : setBits) { + bitSet.set(bit) + } + return bitSet + } } diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy index b6ac00dd7c9d..f47dd127b369 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy @@ -109,6 +109,7 @@ abstract class CiVisibilityTestUtils { static final ObjectMapper JSON_MAPPER = new ObjectMapper() { { enable(SerializationFeature.INDENT_OUTPUT) + enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) } } diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index 43ac216a46b6..670da04d98e7 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -1,6 +1,8 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.junit4.MUnitTracingListener import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder import org.example.TestFailedAssumptionMUnit @@ -60,6 +62,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest { "test-efd-new-slow-test" | [TestSucceedMUnitSlow] | 3 | [] // is executed only twice } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceedMUnit] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceedMUnit] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceedMUnit] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + private void runTests(Collection> tests) { TestEventsHandlerHolder.start() try { diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..5fa9c5b5e535 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,149 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "munit-junit-4", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "munit-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "munit-junit-4", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "munit-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "munit-junit-4", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceedMUnit", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceedMUnit", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "munit", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "munit-junit-4", + "test.name" : "Calculator.add", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceedMUnit", + "test.traits" : "{\"category\":[\"myTag\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceedMUnit.Calculator.add", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index 38f51a328507..c3f7792ed0a9 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -1,6 +1,8 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder import junit.runner.Version import org.example.TestAssumption @@ -133,6 +135,23 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | 8 | [] } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceed] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceed] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceed] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + private void runTests(Collection> tests) { TestEventsHandlerHolder.start() try { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..d3f0ae0144ae --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,151 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-4.10", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "junit-4.10", + "test.name" : "test_succeed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_succeed()V", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceed.test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index 6d316ee4de1e..5957544b95af 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -2,6 +2,8 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.api.civisibility.CIConstants import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder import org.example.TestFailedParameterizedSpock import org.example.TestFailedSpock @@ -113,6 +115,23 @@ class SpockTest extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestSucceedAndFailedSpock] | 8 | [] } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceedSpock] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceedSpock] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceedSpock] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceedSpock] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceedSpock] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + private static void runTests(List> classes) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..547148c63980 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,152 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceedSpock", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.is_new" : "true", + "test.module" : "spock-junit-5", + "test.name" : "test success", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test success()V", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceedSpock.test success", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "spock-junit-5", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index 8f58d5f18ec3..bc658745ad94 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -1,6 +1,8 @@ import datadog.trace.api.DisableTestTrace import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.junit5.TestEventsHandlerHolder import org.example.TestAssumption import org.example.TestAssumptionAndSucceed @@ -140,6 +142,23 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | 8 | [] } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceed] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceed] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceed] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + private static void runTests(List> tests) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..61fbd06bc116 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,151 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-5.3", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestSucceed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "junit-5.3", + "test.name" : "test_succeed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_succeed()V", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceed.test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index 1646f5173ac0..d63ec8f5085c 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -1,5 +1,7 @@ import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import datadog.trace.instrumentation.scalatest.ScalatestUtils import org.example.TestFailed import org.example.TestFailedParameterized @@ -84,6 +86,23 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestSucceedMoreCases] | 8 | [] } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceed] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceed] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceed] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + @Override String instrumentedLibraryName() { return "scalatest" diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..dc3476e6a550 --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,148 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "scalatest.test_suite", + "resource" : "org.example.TestSucceed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "scalatest", + "test.name" : "Example.add adds two numbers", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceed.Example.add adds two numbers", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "scalatest", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test_session", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "scalatest.test_module", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 28a892fd5423..89c7b8f505f0 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -2,6 +2,8 @@ package datadog.trace.instrumentation.testng import datadog.trace.api.civisibility.config.TestIdentifier import datadog.trace.civisibility.CiVisibilityInstrumentationTest +import datadog.trace.civisibility.diff.FileDiff +import datadog.trace.civisibility.diff.LineDiff import org.example.TestError import org.example.TestFailed import org.example.TestFailedAndSucceed @@ -151,6 +153,23 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | 8 | [] } + def "test impacted tests detection #testcaseName"() { + givenImpactedTestsDetectionEnabled(true) + givenDiff(prDiff) + + runTests(tests) + + assertSpansData(testcaseName, expectedTracesCount) + + where: + testcaseName | tests | expectedTracesCount | prDiff + "test-succeed" | [TestSucceed] | 2 | LineDiff.EMPTY + "test-succeed" | [TestSucceed] | 2 | new FileDiff(new HashSet()) + "test-succeed-impacted" | [TestSucceed] | 2 | new FileDiff(new HashSet([DUMMY_SOURCE_PATH])) + "test-succeed" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines()]) + "test-succeed-impacted" | [TestSucceed] | 2 | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)]) + } + private static boolean isEFDSupported() { TracingListener.FRAMEWORK_VERSION >= "7.5" } diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java b/dd-java-agent/instrumentation/testng/testng-6/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java index c2bdc0cd3c05..2568b8b532fe 100644 --- a/dd-java-agent/instrumentation/testng/testng-6/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/testng-6/src/main/java/datadog/trace/instrumentation/testng6/TestNGClassListenerInstrumentation.java @@ -65,6 +65,7 @@ public String[] helperClassNames() { } public static class InvokeBeforeClassAdvice { + @SuppressWarnings("bytebuddy-exception-suppression") @Advice.OnMethodEnter public static void invokeBeforeClass( @Advice.FieldValue("m_testContext") final ITestContext testContext, @@ -82,6 +83,7 @@ public static String muzzleCheck(final DataProvider dataProvider) { } public static class InvokeAfterClassAdvice { + @SuppressWarnings("bytebuddy-exception-suppression") @Advice.OnMethodExit public static void invokeAfterClass( @Advice.FieldValue("m_testContext") final ITestContext testContext, diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..262fe32c32d9 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,151 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestSucceed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "testng-6", + "test.name" : "test_succeed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_succeed()V", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceed.test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-6", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "testng.test_module", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/coverages.ftl new file mode 100644 index 000000000000..8878e547a798 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/events.ftl new file mode 100644 index 000000000000..e5e7aa402525 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-succeed-impacted/events.ftl @@ -0,0 +1,151 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestSucceed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.is_modified" : "true", + "test.module" : "testng-7", + "test.name" : "test_succeed", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_succeed()V", + "test.status" : "pass", + "test.suite" : "org.example.TestSucceed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestSucceed.test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-7", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.status" : "pass", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "testng.test_module", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file