Add instrumentation tests for impacted tests detection

DataDog · Jan 22, 2025 · e6a5ff4 · e6a5ff4
1 parent 3b7cc1e
commit e6a5ff4
Show file tree

Hide file tree

Showing 23 changed files with 1,204 additions and 5 deletions.
diff --git a/...src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/...src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy
@@ -24,6 +24,7 @@ import datadog.trace.civisibility.coverage.file.FileCoverageStore
 import datadog.trace.civisibility.coverage.percentage.NoOpCoverageCalculator
 import datadog.trace.civisibility.decorator.TestDecorator
 import datadog.trace.civisibility.decorator.TestDecoratorImpl
+import datadog.trace.civisibility.diff.Diff
 import datadog.trace.civisibility.diff.LineDiff
 import datadog.trace.civisibility.domain.BuildSystemSession
 import datadog.trace.civisibility.domain.TestFrameworkModule
@@ -52,7 +53,6 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 
-// FIXME nikita: add integration/smoke tests
 abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
 
   static String dummyModule
@@ -70,9 +70,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
   private static final List<TestIdentifier> skippableTests = new ArrayList<>()
   private static final List<TestIdentifier> flakyTests = new ArrayList<>()
   private static final List<TestIdentifier> knownTests = new ArrayList<>()
+  private static volatile Diff diff = LineDiff.EMPTY
+
   private static volatile boolean itrEnabled = false
   private static volatile boolean flakyRetryEnabled = false
   private static volatile boolean earlyFlakinessDetectionEnabled = false
+  private static volatile boolean impactedTestsDetectionEnabled = false
+
   public static final int SLOW_TEST_THRESHOLD_MILLIS = 1000
   public static final int VERY_SLOW_TEST_THRESHOLD_MILLIS = 2000
 
@@ -118,14 +122,14 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
         false,
         itrEnabled,
         flakyRetryEnabled,
-        false,
+        impactedTestsDetectionEnabled,
         earlyFlakinessDetectionSettings,
         itrEnabled ? "itrCorrelationId" : null,
         skippableTestsWithMetadata,
         [:],
         flakyTests,
         earlyFlakinessDetectionEnabled || CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(Config.get().ciVisibilityTestOrder) ? knownTests : null,
-        LineDiff.EMPTY)
+        diff)
       }
     }
 
@@ -202,8 +206,8 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
       TestFrameworkSession testSession = testFrameworkSessionFactory.startSession(dummyModule, component, null)
       TestFrameworkModule testModule = testSession.testModuleStart(dummyModule, null)
       new TestEventsHandlerImpl(metricCollector, testSession, testModule,
-        suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(),
-        testStore != null ? testStore : new ConcurrentHashMapContextStore<>())
+      suiteStore != null ? suiteStore : new ConcurrentHashMapContextStore<>(),
+      testStore != null ? testStore : new ConcurrentHashMapContextStore<>())
     }
   }
 
@@ -217,9 +221,11 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     skippableTests.clear()
     flakyTests.clear()
     knownTests.clear()
+    diff = LineDiff.EMPTY
     itrEnabled = false
     flakyRetryEnabled = false
     earlyFlakinessDetectionEnabled = false
+    impactedTestsDetectionEnabled = false
   }
 
   def givenSkippableTests(List<TestIdentifier> tests) {
@@ -235,6 +241,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     knownTests.addAll(tests)
   }
 
+  def givenDiff(Diff diff) {
+    this.diff = diff
+  }
+
   def givenFlakyRetryEnabled(boolean flakyRetryEnabled) {
     this.flakyRetryEnabled = flakyRetryEnabled
   }
@@ -243,6 +253,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     this.earlyFlakinessDetectionEnabled = earlyFlakinessDetectionEnabled
   }
 
+  def givenImpactedTestsDetectionEnabled(boolean impactedTestsDetectionEnabled) {
+    this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled
+  }
+
   def givenTestsOrder(String testsOrder) {
     injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder)
   }
@@ -351,4 +365,12 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
   abstract String instrumentedLibraryName()
 
   abstract String instrumentedLibraryVersion()
+
+  BitSet lines(int ... setBits) {
+    BitSet bitSet = new BitSet()
+    for (int bit : setBits) {
+      bitSet.set(bit)
+    }
+    return bitSet
+  }
 }
diff --git a/...isibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy b/...isibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy
@@ -109,6 +109,7 @@ abstract class CiVisibilityTestUtils {
 
   static final ObjectMapper JSON_MAPPER = new ObjectMapper() { {
       enable(SerializationFeature.INDENT_OUTPUT)
+      enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
     }
   }
 

diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy
@@ -1,6 +1,8 @@
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.diff.FileDiff
+import datadog.trace.civisibility.diff.LineDiff
 import datadog.trace.instrumentation.junit4.MUnitTracingListener
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
 import org.example.TestFailedAssumptionMUnit
@@ -60,6 +62,23 @@ class MUnitTest extends CiVisibilityInstrumentationTest {
     "test-efd-new-slow-test" | [TestSucceedMUnitSlow] | 3                   | [] // is executed only twice
   }
 
+  def "test impacted tests detection #testcaseName"() {
+    givenImpactedTestsDetectionEnabled(true)
+    givenDiff(prDiff)
+
+    runTests(tests)
+
+    assertSpansData(testcaseName, expectedTracesCount)
+
+    where:
+    testcaseName            | tests              | expectedTracesCount | prDiff
+    "test-succeed"          | [TestSucceedMUnit] | 2                   | LineDiff.EMPTY
+    "test-succeed"          | [TestSucceedMUnit] | 2                   | new FileDiff(new HashSet())
+    "test-succeed-impacted" | [TestSucceedMUnit] | 2                   | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
+    "test-succeed"          | [TestSucceedMUnit] | 2                   | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
+    "test-succeed-impacted" | [TestSucceedMUnit] | 2                   | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
+  }
+
   private void runTests(Collection<Class<?>> tests) {
     TestEventsHandlerHolder.start()
     try {

diff --git a/...mentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl b/...mentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/coverages.ftl
@@ -0,0 +1 @@
+[ ]
diff --git a/...trumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl b/...trumentation/junit-4.10/munit-junit-4/src/test/resources/test-succeed-impacted/events.ftl
@@ -0,0 +1,149 @@
+[ {
+  "content" : {
+    "duration" : ${content_duration},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid},
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test_session_end",
+      "test.command" : "munit-junit-4",
+      "test.framework" : "munit",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.status" : "pass",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test_session",
+    "resource" : "munit-junit-4",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_session_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_2},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_2},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_module_end",
+      "test.framework" : "munit",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "munit-junit-4",
+      "test.status" : "pass",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}
+    },
+    "name" : "junit.test_module",
+    "resource" : "munit-junit-4",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_2},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_module_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_3},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_3},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_suite_end",
+      "test.codeowners" : "[\"owner1\",\"owner2\"]",
+      "test.framework" : "munit",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "munit-junit-4",
+      "test.source.file" : "dummy_source_path",
+      "test.status" : "pass",
+      "test.suite" : "org.example.TestSucceedMUnit",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
+      "test.source.end" : 19,
+      "test.source.start" : 11
+    },
+    "name" : "junit.test_suite",
+    "resource" : "org.example.TestSucceedMUnit",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_3},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id}
+  },
+  "type" : "test_suite_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_4},
+    "error" : 0,
+    "meta" : {
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test",
+      "test.codeowners" : "[\"owner1\",\"owner2\"]",
+      "test.framework" : "munit",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.is_modified" : "true",
+      "test.module" : "munit-junit-4",
+      "test.name" : "Calculator.add",
+      "test.source.file" : "dummy_source_path",
+      "test.status" : "pass",
+      "test.suite" : "org.example.TestSucceedMUnit",
+      "test.traits" : "{\"category\":[\"myTag\"]}",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test",
+    "parent_id" : ${content_parent_id},
+    "resource" : "org.example.TestSucceedMUnit.Calculator.add",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "span_id" : ${content_span_id},
+    "start" : ${content_start_4},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id},
+    "trace_id" : ${content_trace_id}
+  },
+  "type" : "test",
+  "version" : 2
+} ]
diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy
@@ -1,6 +1,8 @@
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
+import datadog.trace.civisibility.diff.FileDiff
+import datadog.trace.civisibility.diff.LineDiff
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
 import junit.runner.Version
 import org.example.TestAssumption
@@ -133,6 +135,23 @@ class JUnit4Test extends CiVisibilityInstrumentationTest {
     "test-efd-faulty-session-threshold" | [TestFailedAndSucceed] | 8                   | []
   }
 
+  def "test impacted tests detection #testcaseName"() {
+    givenImpactedTestsDetectionEnabled(true)
+    givenDiff(prDiff)
+
+    runTests(tests)
+
+    assertSpansData(testcaseName, expectedTracesCount)
+
+    where:
+    testcaseName            | tests         | expectedTracesCount | prDiff
+    "test-succeed"          | [TestSucceed] | 2                   | LineDiff.EMPTY
+    "test-succeed"          | [TestSucceed] | 2                   | new FileDiff(new HashSet())
+    "test-succeed-impacted" | [TestSucceed] | 2                   | new FileDiff(new HashSet([DUMMY_SOURCE_PATH]))
+    "test-succeed"          | [TestSucceed] | 2                   | new LineDiff([(DUMMY_SOURCE_PATH): lines()])
+    "test-succeed-impacted" | [TestSucceed] | 2                   | new LineDiff([(DUMMY_SOURCE_PATH): lines(DUMMY_TEST_METHOD_START)])
+  }
+
   private void runTests(Collection<Class<?>> tests) {
     TestEventsHandlerHolder.start()
     try {

diff --git a/...a-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl b/...a-agent/instrumentation/junit-4.10/src/test/resources/test-succeed-impacted/coverages.ftl
@@ -0,0 +1 @@
+[ ]
-Original file line number
+Diff line change
@@ Expand Up / @@ -109,6 +109,7 @@ abstract class CiVisibilityTestUtils { @@
       static final ObjectMapper JSON_MAPPER = new ObjectMapper() { {
           enable(SerializationFeature.INDENT_OUTPUT)
+          enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
         }
       }
@@ Expand Down @@