Nikita tkachenko/test env 19d8832a9d074b4e378be52f31eb1627522341a6

.circleci/config.continue.yml.j2

@@ -696,6 +696,8 @@ jobs:
     <<: *tests
 
     resource_class: medium
+    environment:
+      - CI_AGENT_HOST=localhost
 
     docker:
       - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8

.gitlab-ci.yml

@@ -25,6 +25,9 @@ variables:
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
   JAVA_BUILD_IMAGE_VERSION: "v25.01"
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
+  PROFILE_TESTS:
+    description: "Enable profiling of tests"
+    value: "false"
 
 default:
   tags: [ "arch:amd64" ]
@@ -35,6 +38,10 @@ default:
   script:
     - echo "done"
 
+.set_datadog_api_keys: &set_datadog_api_keys
+  - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
+  - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
   stage: build
@@ -177,12 +184,76 @@ muzzle-dep-report:
       - ./reports
       - '.gradle/daemon/*/*.out.log'
 
+# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
+# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
+.prepare_test_env: &prepare_test_env
+  - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
+  - '[ ! -e pretest.env ] || rm pretest.env'
+  - |
+    for VARIABLE in "${gitlabVariables[@]}"
+    do
+      echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
+      unset "$VARIABLE"
+    done
+
+.restore_pretest_env: &restore_pretest_env
+  - source pretest.env
+
+.test_job:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: lib
+    GRADLE_PARAMS: ""
+    CONTINUE_ON_FAILURE: "false"
+  script:
+    - > 
+      if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
+      then
+      export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
+      fi
+    - *prepare_test_env
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
+  after_script:
+    - *restore_pretest_env
+    - *set_datadog_api_keys
+    - .circleci/collect_reports.sh
+    - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
+    - .circleci/collect_results.sh
+    - .circleci/upload_ciapp.sh tests $testJvm
+# TODO Get APM Test Agent Trace Check Results
+  artifacts:
+    when: always
+    paths:
+      - ./reports.tar
+      - ./profiles.tar
+      - ./results
+      - '.gradle/daemon/*/*.out.log'
+
+agent_integration_tests:
+  extends: .test_job
+  variables:
+    testJvm: "8"
+    CI_AGENT_HOST: local-agent
+    GRADLE_TARGET: "traceAgentTest"
+  services:
+    - name: datadog/agent:7.34.0
+      alias: local-agent
+      variables:
+        DD_APM_ENABLED: "true"
+        DD_BIND_HOST: "0.0.0.0"
+        DD_API_KEY: "invalid_key_but_this_is_fine"
+
 required:
   extends: .fan_in
   needs:
     - spotless
     - muzzle
     - test_published_artifacts
+    - agent_integration_tests
 
 deploy_to_profiling_backend:
   stage: publish

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -358,8 +358,10 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     //          def clazz = this.getClass()
     //          def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
     //          def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
-    //          def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
-    //          def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
+    //          def searchIndex = classfilePath.indexOf("/build/classes/groovy")
+    //          def modulePath = classfilePath.substring(0, searchIndex)
+    //          def submoduleName = classfilePath.substring(searchIndex + "/build/classes/groovy".length()).split("/")[1]
+    //          def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
     //          CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
     //          return [:]
 

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java

@@ -44,24 +44,28 @@ public void testSuiteStarted(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
-          suiteDescriptor,
-          testSuiteName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          Collections.emptyList(),
-          false,
-          TestFrameworkInstrumentation.CUCUMBER,
-          null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteStart(
+              suiteDescriptor,
+              testSuiteName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              Collections.emptyList(),
+              false,
+              TestFrameworkInstrumentation.CUCUMBER,
+              null);
     }
   }
 
   @Override
   public void testSuiteFinished(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor, null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFinish(suiteDescriptor, null);
     }
   }
 
@@ -71,17 +75,19 @@ public void testStarted(final Description description) {
     String testName = CucumberUtils.getTestNameForScenario(description);
     List<String> categories = getCategories(description);
 
-    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart(
-        new TestSuiteDescriptor(testSuiteName, null),
-        CucumberUtils.toTestDescriptor(description),
-        testName,
-        FRAMEWORK_NAME,
-        FRAMEWORK_VERSION,
-        null,
-        categories,
-        TestSourceData.UNKNOWN,
-        null,
-        executionHistories.get(description));
+    TestEventsHandlerHolder.HANDLERS
+        .get(TestFrameworkInstrumentation.CUCUMBER)
+        .onTestStart(
+            new TestSuiteDescriptor(testSuiteName, null),
+            CucumberUtils.toTestDescriptor(description),
+            testName,
+            FRAMEWORK_NAME,
+            FRAMEWORK_VERSION,
+            null,
+            categories,
+            TestSourceData.UNKNOWN,
+            null,
+            executionHistories.get(description));
 
     recordFeatureFileCodeCoverage(description);
   }
@@ -100,8 +106,9 @@ private static void recordFeatureFileCodeCoverage(Description scenarioDescriptio
   public void testFinished(final Description description) {
     TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
     TestExecutionHistory executionHistory = executionHistories.get(description);
-    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(
-        testDescriptor, null, executionHistory);
+    TestEventsHandlerHolder.HANDLERS
+        .get(TestFrameworkInstrumentation.CUCUMBER)
+        .onTestFinish(testDescriptor, null, executionHistory);
   }
 
   // same callback is executed both for test cases and test suites (for setup/teardown errors)
@@ -111,11 +118,15 @@ public void testFailure(final Failure failure) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       Throwable throwable = failure.getException();
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(suiteDescriptor, throwable);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFailure(suiteDescriptor, throwable);
     } else {
       TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
       Throwable throwable = failure.getException();
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, throwable);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestFailure(testDescriptor, throwable);
     }
   }
 
@@ -132,10 +143,14 @@ public void testAssumptionFailure(final Failure failure) {
     Description description = failure.getDescription();
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteSkip(suiteDescriptor, reason);
     } else {
       TestDescriptor testDescriptor = CucumberUtils.toTestDescriptor(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSkip(testDescriptor, reason);
     }
   }
 
@@ -147,32 +162,40 @@ public void testIgnored(final Description description) {
     if (isFeature(description)) {
       TestSuiteDescriptor suiteDescriptor = CucumberUtils.toSuiteDescriptor(description);
       String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
-          suiteDescriptor,
-          testSuiteName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          Collections.emptyList(),
-          false,
-          TestFrameworkInstrumentation.CUCUMBER,
-          null);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, reason);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor, null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteStart(
+              suiteDescriptor,
+              testSuiteName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              Collections.emptyList(),
+              false,
+              TestFrameworkInstrumentation.CUCUMBER,
+              null);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteSkip(suiteDescriptor, reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestSuiteFinish(suiteDescriptor, null);
     } else {
       String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
       String testName = CucumberUtils.getTestNameForScenario(description);
       List<String> categories = getCategories(description);
-      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore(
-          new TestSuiteDescriptor(testSuiteName, null),
-          CucumberUtils.toTestDescriptor(description),
-          testName,
-          FRAMEWORK_NAME,
-          FRAMEWORK_VERSION,
-          null,
-          categories,
-          TestSourceData.UNKNOWN,
-          reason);
+      TestEventsHandlerHolder.HANDLERS
+          .get(TestFrameworkInstrumentation.CUCUMBER)
+          .onTestIgnore(
+              new TestSuiteDescriptor(testSuiteName, null),
+              CucumberUtils.toTestDescriptor(description),
+              testName,
+              FRAMEWORK_NAME,
+              FRAMEWORK_VERSION,
+              null,
+              categories,
+              TestSourceData.UNKNOWN,
+              reason);
     }
   }
 

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberInstrumentation.java

@@ -8,6 +8,7 @@
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.agent.tooling.muzzle.Reference;
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.InstrumentationContext;
 import java.util.Collections;
 import java.util.List;
@@ -82,6 +83,8 @@ public static void addTracingListener(
         }
       }
 
+      TestEventsHandlerHolder.start(TestFrameworkInstrumentation.CUCUMBER);
+
       replacedNotifier.addListener(
           new CucumberTracingListener(
               InstrumentationContext.get(Description.class, TestExecutionHistory.class), children));

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.civisibility.InstrumentationBridge;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import io.cucumber.core.gherkin.Pickle;
 import java.util.List;
@@ -83,7 +84,10 @@ public static Boolean run(
         @Advice.Argument(0) RunNotifier notifier) {
 
       TestIdentifier test = CucumberUtils.toTestIdentifier(description);
-      SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(test);
+      SkipReason skipReason =
+          TestEventsHandlerHolder.HANDLERS
+              .get(TestFrameworkInstrumentation.CUCUMBER)
+              .skipReason(test);
       if (skipReason == null) {
         return null;
       }

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/execution/Cucumber4ExecutionInstrumentation.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.execution.TestExecutionHistory;
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
 import datadog.trace.bootstrap.InstrumentationContext;
 import datadog.trace.instrumentation.junit4.CucumberUtils;
 import datadog.trace.instrumentation.junit4.JUnit4Utils;
@@ -97,8 +98,9 @@ public static Boolean execute(
       Description description = CucumberUtils.getPickleRunnerDescription(pickleRunner);
       TestIdentifier testIdentifier = CucumberUtils.toTestIdentifier(description);
       TestExecutionPolicy executionPolicy =
-          TestEventsHandlerHolder.TEST_EVENTS_HANDLER.executionPolicy(
-              testIdentifier, TestSourceData.UNKNOWN);
+          TestEventsHandlerHolder.HANDLERS
+              .get(TestFrameworkInstrumentation.CUCUMBER)
+              .executionPolicy(testIdentifier, TestSourceData.UNKNOWN);
       if (!executionPolicy.applicable()) {
         // retries not applicable, run original method
         return null;

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy

@@ -1,6 +1,7 @@
 import datadog.trace.api.DisableTestTrace
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation
 import datadog.trace.civisibility.CiVisibilityInstrumentationTest
 import datadog.trace.instrumentation.junit4.CucumberTracingListener
 import datadog.trace.instrumentation.junit4.TestEventsHandlerHolder
@@ -211,7 +212,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     .map(f -> "classpath:" + f).
     collect(Collectors.joining(",")))
 
-    TestEventsHandlerHolder.start()
+    TestEventsHandlerHolder.start(TestFrameworkInstrumentation.CUCUMBER)
     try {
       def result = runner.run(TestSucceedCucumber)
       if (expectSuccess) {
@@ -224,7 +225,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
         }
       }
     } finally {
-      TestEventsHandlerHolder.stop()
+      TestEventsHandlerHolder.stop(TestFrameworkInstrumentation.CUCUMBER)
     }
   }