Migrate agent integration tests to gitlab

.circleci/config.continue.yml.j2

@@ -696,6 +696,8 @@ jobs:
     <<: *tests
 
     resource_class: medium
+    environment:
+      - CI_AGENT_HOST=localhost
 
     docker:
       - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8

.gitlab-ci.yml

@@ -25,6 +25,9 @@ variables:
   GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties
   JAVA_BUILD_IMAGE_VERSION: "v25.01"
   REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
+  PROFILE_TESTS:
+    description: "Enable profiling of tests"
+    value: "false"
 
 default:
   tags: [ "arch:amd64" ]
@@ -35,6 +38,10 @@ default:
   script:
     - echo "done"
 
+.set_datadog_api_keys: &set_datadog_api_keys
+  - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)
+  - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text)
+
 .gradle_build: &gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base
   stage: build
@@ -185,12 +192,76 @@ muzzle-dep-report:
       - ./reports
       - '.gradle/daemon/*/*.out.log'
 
+# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
+# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
+.prepare_test_env: &prepare_test_env
+  - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
+  - '[ ! -e pretest.env ] || rm pretest.env'
+  - |
+    for VARIABLE in "${gitlabVariables[@]}"
+    do
+      echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
+      unset "$VARIABLE"
+    done
+
+.restore_pretest_env: &restore_pretest_env
+  - source pretest.env
+
+.test_job:
+  extends: .gradle_build
+  image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm
+  needs: [ build ]
+  stage: tests
+  variables:
+    BUILD_CACHE_TYPE: lib
+    GRADLE_PARAMS: ""
+    CONTINUE_ON_FAILURE: "false"
+  script:
+    - > 
+      if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
+      then
+      export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
+      fi
+    - *prepare_test_env
+    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M"
+    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
+  after_script:
+    - *restore_pretest_env
+    - *set_datadog_api_keys
+    - .circleci/collect_reports.sh
+    - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi
+    - .circleci/collect_results.sh
+    - .circleci/upload_ciapp.sh tests $testJvm
+# TODO Get APM Test Agent Trace Check Results
+  artifacts:
+    when: always
+    paths:
+      - ./reports.tar
+      - ./profiles.tar
+      - ./results
+      - '.gradle/daemon/*/*.out.log'
+
+agent_integration_tests:
+  extends: .test_job
+  variables:
+    testJvm: "8"
+    CI_AGENT_HOST: local-agent
+    GRADLE_TARGET: "traceAgentTest"
+  services:
+    - name: datadog/agent:7.34.0
+      alias: local-agent
+      variables:
+        DD_APM_ENABLED: "true"
+        DD_BIND_HOST: "0.0.0.0"
+        DD_API_KEY: "invalid_key_but_this_is_fine"
+
 required:
   extends: .fan_in
   needs:
     - spotless
     - muzzle
     - test_published_artifacts
+    - agent_integration_tests
 
 deploy_to_profiling_backend:
   stage: publish

dd-trace-core/src/traceAgentTest/groovy/AbstractTraceAgentTest.groovy

@@ -0,0 +1,59 @@
+import datadog.trace.api.ConfigDefaults
+import datadog.trace.api.config.TracerConfig
+import datadog.trace.test.util.DDSpecification
+import org.testcontainers.containers.GenericContainer
+import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy
+import spock.lang.Shared
+
+import java.time.Duration
+
+abstract class AbstractTraceAgentTest extends DDSpecification {
+  @Shared
+  def agentContainer
+
+  def setupSpec() {
+    /*
+     CI will provide us with agent container running along side our build.
+     When building locally, however, we need to take matters into our own hands
+     and we use 'testcontainers' for this.
+     */
+    if ("true" != System.getenv("CI")) {
+      agentContainer = new GenericContainer("datadog/agent:7.34.0")
+        .withEnv(["DD_APM_ENABLED": "true",
+          "DD_BIND_HOST"  : "0.0.0.0",
+          "DD_API_KEY"    : "invalid_key_but_this_is_fine",
+          "DD_LOGS_STDOUT": "yes"])
+        .withExposedPorts(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
+        .withStartupTimeout(Duration.ofSeconds(120))
+        // Apparently we need to sleep for a bit so agent's response `{"service:,env:":1}` in rate_by_service.
+        // This is clearly a race-condition and maybe we should avoid verifying complete response
+        .withStartupCheckStrategy(new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(10)))
+      agentContainer.start()
+    }
+  }
+
+  def setup() {
+    injectSysConfig(TracerConfig.AGENT_HOST, getAgentContainerHost())
+    injectSysConfig(TracerConfig.TRACE_AGENT_PORT, getAgentContainerPort())
+  }
+
+  String getAgentContainerHost() {
+    if (agentContainer) {
+      return (String) agentContainer.getHost()
+    }
+
+    return System.getenv("CI_AGENT_HOST")
+  }
+
+  String getAgentContainerPort() {
+    if (agentContainer) {
+      return (String) agentContainer.getMappedPort(ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
+    }
+
+    return ConfigDefaults.DEFAULT_TRACE_AGENT_PORT
+  }
+
+  def cleanupSpec() {
+    agentContainer?.stop()
+  }
+}

dd-trace-core/src/traceAgentTest/groovy/DDApiIntegrationTest.groovy

@@ -3,6 +3,7 @@ import datadog.communication.http.OkHttpUtils
 import datadog.communication.serialization.ByteBufferConsumer
 import datadog.communication.serialization.FlushingBuffer
 import datadog.communication.serialization.msgpack.MsgPackWriter
+import datadog.trace.api.Config
 import datadog.trace.api.StatsDClient
 import datadog.trace.common.writer.ListWriter
 import datadog.trace.common.writer.Payload
@@ -15,39 +16,22 @@ import datadog.trace.common.writer.ddagent.TraceMapperV0_5
 import datadog.trace.core.CoreTracer
 import datadog.trace.core.DDSpan
 import datadog.trace.core.monitor.MonitoringImpl
-import datadog.trace.test.util.DDSpecification
 import okhttp3.HttpUrl
 import okhttp3.OkHttpClient
-import org.testcontainers.containers.GenericContainer
-import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy
-import spock.lang.Requires
 import spock.lang.Shared
 
 import java.nio.ByteBuffer
-import java.time.Duration
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicReference
 
-// It is fine to run on CI because CI provides agent externally, not through testcontainers
-@Requires({ "true" == System.getenv("CI") })
-class DDApiIntegrationTest extends DDSpecification {
-  def tracer = CoreTracer.builder().writer(new ListWriter()).build()
+class DDApiIntegrationTest extends AbstractTraceAgentTest {
+  def tracer
   DDSpan span
 
   // Looks like okHttp needs to resolve this, even for connection over socket
   static final SOMEHOST = "datadoghq.com"
   static final SOMEPORT = 123
 
-  /*
-   Note: type here has to stay undefined, otherwise tests will fail in CI in Java 7 because
-   'testcontainers' are built for Java 8 and Java 7 cannot load this class.
-   */
-  @Shared
-  def agentContainer
-  @Shared
-  def agentContainerHost = "localhost"
-  @Shared
-  def agentContainerPort = 8126
   @Shared
   Process process
   @Shared
@@ -69,30 +53,6 @@ class DDApiIntegrationTest extends DDSpecification {
   }
 
   def setupSpec() {
-    /*
-     CI will provide us with agent container running along side our build.
-     When building locally, however, we need to take matters into our own hands
-     and we use 'testcontainers' for this.
-     */
-    if ("true" != System.getenv("CI")) {
-      agentContainer = new GenericContainer("datadog/agent:7.22.0")
-        .withEnv(["DD_APM_ENABLED": "true",
-          "DD_BIND_HOST"  : "0.0.0.0",
-          "DD_API_KEY"    : "invalid_key_but_this_is_fine",
-          "DD_LOGS_STDOUT": "yes"])
-        .withExposedPorts(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
-        .withStartupTimeout(Duration.ofSeconds(120))
-        // Apparently we need to sleep for a bit so agent's response `{"service:,env:":1}` in rate_by_service.
-        // This is clearly a race-condition and maybe we should avoid verifying complete response
-        .withStartupCheckStrategy(new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(10)))
-      //        .withLogConsumer { output ->
-      //        print output.utf8String
-      //      }
-      agentContainer.start()
-      agentContainerHost = agentContainer.getHost()
-      agentContainerPort = agentContainer.getMappedPort(datadog.trace.api.ConfigDefaults.DEFAULT_TRACE_AGENT_PORT)
-    }
-
     File tmpDir = File.createTempDir()
     tmpDir.deleteOnExit()
     socketPath = new File(tmpDir, "socket")
@@ -101,6 +61,7 @@ class DDApiIntegrationTest extends DDSpecification {
   }
 
   def setup() {
+    tracer = CoreTracer.builder().writer(new ListWriter()).build()
     span = tracer.buildSpan("fakeOperation").start()
     Thread.sleep(1)
     span.finish()
@@ -111,15 +72,12 @@ class DDApiIntegrationTest extends DDSpecification {
   }
 
   def cleanupSpec() {
-    if (agentContainer) {
-      agentContainer.stop()
-    }
-    process.destroy()
+    process?.destroy()
   }
 
   def beforeTest(boolean enableV05) {
     MonitoringImpl monitoring = new MonitoringImpl(StatsDClient.NO_OP, 1, TimeUnit.SECONDS)
-    HttpUrl agentUrl = HttpUrl.get(String.format("http://%s:%d", agentContainerHost, agentContainerPort))
+    HttpUrl agentUrl = HttpUrl.get(Config.get().getAgentUrl())
     OkHttpClient httpClient = OkHttpUtils.buildHttpClient(agentUrl, 5000)
     discovery = new DDAgentFeaturesDiscovery(httpClient, monitoring, agentUrl, enableV05, true)
     api = new DDAgentApi(httpClient, agentUrl, discovery, monitoring, false)
@@ -143,7 +101,7 @@ class DDApiIntegrationTest extends DDSpecification {
     assert 200 == response.status()
     assert response.success()
     assert discovery.getTraceEndpoint() == "${version}/traces"
-    assert endpoint.get() == "http://${agentContainerHost}:${agentContainerPort}/${version}/traces"
+    assert endpoint.get() == "${Config.get().getAgentUrl()}/${version}/traces"
     assert agentResponse.get()["rate_by_service"] instanceof Map
 
     where:
@@ -166,7 +124,7 @@ class DDApiIntegrationTest extends DDSpecification {
     assert 200 == response.status()
     assert response.success()
     assert discovery.getTraceEndpoint() == "${version}/traces"
-    assert endpoint.get() == "http://${agentContainerHost}:${agentContainerPort}/${version}/traces"
+    assert endpoint.get() == "${Config.get().getAgentUrl()}/${version}/traces"
     assert agentResponse.get()["rate_by_service"] instanceof Map
 
     where:

dd-trace-core/src/traceAgentTest/groovy/DataStreamsIntegrationTest.groovy

@@ -8,21 +8,16 @@ import datadog.trace.api.datastreams.StatsPoint
 import datadog.trace.common.metrics.EventListener
 import datadog.trace.common.metrics.OkHttpSink
 import datadog.trace.core.datastreams.DefaultDataStreamsMonitoring
-import datadog.trace.test.util.DDSpecification
 import okhttp3.HttpUrl
 import spock.lang.Ignore
-import spock.lang.Requires
 import spock.util.concurrent.PollingConditions
 
 import java.util.concurrent.CopyOnWriteArrayList
 
 import static datadog.trace.common.metrics.EventListener.EventType.OK
 
-@Requires({
-  "true" == System.getenv("CI")
-})
 @Ignore("The agent in CI doesn't have a valid API key. Unlike metrics and traces, data streams fails in this case")
-class DataStreamsIntegrationTest extends DDSpecification {
+class DataStreamsIntegrationTest extends AbstractTraceAgentTest {
 
   def "Sending stats bucket to agent should notify with OK event"() {
     given:
@@ -49,10 +44,10 @@ class DataStreamsIntegrationTest extends DDSpecification {
     }
 
     when:
-    def dataStreams = new DefaultDataStreamsMonitoring(sink, sharedCommunicationObjects.featuresDiscovery, timeSource, { traceConfig }, Config.get())
+    def dataStreams = new DefaultDataStreamsMonitoring(sink, sharedCommunicationObjects.featuresDiscovery(Config.get()), timeSource, { traceConfig }, Config.get())
     dataStreams.start()
-    dataStreams.accept(new StatsPoint("testType", "testGroup", "testTopic", 1, 2, timeSource.currentTimeNanos, 0, 0, null))
-    timeSource.advance(DEFAULT_BUCKET_DURATION_NANOS)
+    dataStreams.add(new StatsPoint(["type:testType", "group:testGroup", "topic:testTopic"], 1, 2, 5, timeSource.currentTimeNanos, 0, 0, 0, null))
+    timeSource.advance(Config.get().getDataStreamsBucketDurationNanoseconds())
     dataStreams.report()
 
     then:

dd-trace-core/src/traceAgentTest/groovy/MetricsIntegrationTest.groovy

@@ -1,34 +1,29 @@
-
 import datadog.communication.ddagent.DDAgentFeaturesDiscovery
 import datadog.communication.http.OkHttpUtils
+import datadog.trace.api.Config
 import datadog.trace.api.WellKnownTags
 import datadog.trace.common.metrics.AggregateMetric
 import datadog.trace.common.metrics.EventListener
 import datadog.trace.common.metrics.MetricKey
 import datadog.trace.common.metrics.OkHttpSink
 import datadog.trace.common.metrics.SerializingMetricWriter
-import datadog.trace.test.util.DDSpecification
-import spock.lang.Requires
+import okhttp3.HttpUrl
 
 import java.util.concurrent.CopyOnWriteArrayList
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.atomic.AtomicLongArray
 
 import static datadog.trace.common.metrics.EventListener.EventType.OK
 import static java.util.concurrent.TimeUnit.SECONDS
-import okhttp3.HttpUrl
 
-@Requires({
-  "true" == System.getenv("CI")
-})
-class MetricsIntegrationTest extends DDSpecification {
+class MetricsIntegrationTest extends AbstractTraceAgentTest {
 
 
   def "send metrics to trace agent should notify with OK event"() {
     setup:
     def latch = new CountDownLatch(1)
     def listener = new BlockingListener(latch)
-    def agentUrl = "http://localhost:8126"
+    def agentUrl = Config.get().getAgentUrl()
     OkHttpSink sink = new OkHttpSink(OkHttpUtils.buildHttpClient(HttpUrl.parse(agentUrl), 5000L), agentUrl, DDAgentFeaturesDiscovery.V6_METRICS_ENDPOINT, true, false, [:])
     sink.register(listener)