Merge branch 'master' into smola/host-tests

Author: smola

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 315bc8a32cc888834726397f088336ba8038277f
+default_system_tests_commit: &default_system_tests_commit 344e4abc912ada8004f2d0828a09efb65b5ee1e1
 
 parameters:
   nightly:
@@ -365,6 +365,21 @@ jobs:
 
       - display_memory_usage
 
+  spotless:
+    <<: *defaults
+    resource_class: medium+
+
+    steps:
+      - setup_code
+
+      - run:
+          name: Run spotless
+          command: >-
+            JAVA_HOME=$JAVA_11_HOME
+            ./gradlew spotlessCheck
+            << pipeline.parameters.gradle_flags >>
+            --max-workers=8
+
   check:
     <<: *defaults
 
@@ -681,6 +696,8 @@ jobs:
     <<: *tests
 
     resource_class: medium
+    environment:
+      - CI_AGENT_HOST=localhost
 
     docker:
       - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}8
@@ -690,6 +707,117 @@ jobs:
           - DD_BIND_HOST=0.0.0.0
           - DD_API_KEY=invalid_key_but_this_is_fine
 
+  test_published_artifacts:
+    <<: *defaults
+    resource_class: medium
+    docker:
+      - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}7
+
+    steps:
+      - setup_code
+      - restore_dependency_cache:
+          cacheType: lib
+      - restore_build_cache:
+          cacheType: lib
+
+      - run:
+          name: Publish Artifacts Locally
+          command: |
+            mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
+            rm -rf "${mvn_local_repo}/com/datadoghq"
+            export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+            ./gradlew publishToMavenLocal << pipeline.parameters.gradle_flags >> --max-workers=3
+
+      - run:
+          name: Test Published Artifacts
+          command: |
+            cd test-published-dependencies
+            export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+            ./gradlew check --info --max-workers=3
+
+      - run:
+          name: Collect Reports
+          when: on_fail
+          command: .circleci/collect_reports.sh
+
+      - store_artifacts:
+          path: ./reports
+
+      - display_memory_usage
+  muzzle-dep-report:
+    <<: *defaults
+    resource_class: medium
+    steps:
+      - setup_code
+      - skip_unless_matching_files_changed:
+          pattern: "dd-java-agent/instrumentation"
+      - restore_dependency_cache:
+          cacheType: inst
+      - restore_build_cache:
+          cacheType: inst
+      - run:
+          name: Generate muzzle dep report
+          command: >-
+            SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+            ./gradlew generateMuzzleReport muzzleInstrumentationReport
+      - run:
+          name: Collect Reports
+          command: .circleci/collect_muzzle_deps.sh
+      - store_artifacts:
+          path: ./reports
+
+  muzzle:
+    <<: *defaults
+    resource_class: medium+
+    parallelism: 4
+    steps:
+      - setup_code
+
+      - skip_unless_matching_files_changed:
+          pattern: "dd-java-agent/instrumentation"
+
+      # We are not running with a separate cache of all muzzle artifacts here because it gets very big and
+      # ends up taking more time restoring/saving than the actual increase in time it takes just
+      # downloading the artifacts each time.
+      #
+      # Let's at least restore the build cache to have something to start from.
+      - restore_dependency_cache:
+          cacheType: inst
+      - restore_build_cache:
+          cacheType: inst
+
+      - run:
+          name: Gather muzzle tasks
+          command: >-
+            SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+            ./gradlew writeMuzzleTasksToFile
+            << pipeline.parameters.gradle_flags >>
+            --max-workers=3
+
+      - run:
+          name: Verify Muzzle
+          command: >-
+            SKIP_BUILDSCAN="true"
+            GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx3G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
+            ./gradlew `circleci tests split --split-by=timings workspace/build/muzzleTasks | xargs`
+            << pipeline.parameters.gradle_flags >>
+            --max-workers=4
+
+      - run:
+          name: Collect Reports
+          when: on_fail
+          command: .circleci/collect_reports.sh
+
+      - store_artifacts:
+          path: ./reports
+
+      - store_test_results:
+          path: workspace/build/muzzle-test-results
+
+      - display_memory_usage
+
   system-tests:
     machine:
       image: ubuntu-2404:current
@@ -731,6 +859,9 @@ jobs:
               APPSEC_REQUEST_BLOCKING
               APPSEC_RASP
               APPSEC_RUNTIME_ACTIVATION
+              APPSEC_API_SECURITY
+              APPSEC_API_SECURITY_RC
+              APPSEC_API_SECURITY_WITH_SAMPLING
               APPSEC_WAF_TELEMETRY
               APPSEC_STANDALONE_V2
               IAST_STANDALONE_V2
@@ -760,6 +891,7 @@ jobs:
             for log_dir in logs*; do
               tar -cvzf ../artifacts/${log_dir}_<< parameters.weblog-variant >>.tar.gz $log_dir
             done
+          when: always
 
       - store_artifacts:
           path: artifacts
@@ -849,6 +981,7 @@ jobs:
             for dir in system-tests/logs*/; do
                 cp -r "$dir" logs_debugger
             done
+          when: always
 
       - store_test_results:
           path: logs_debugger
@@ -943,6 +1076,7 @@ build_test_jobs: &build_test_jobs
       name: build_profiling
       gradleTarget: :profilingTest
       cacheType: profiling
+  - spotless
 
   - fan_in:
       requires:
@@ -951,6 +1085,7 @@ build_test_jobs: &build_test_jobs
         - build_inst
         - build_smoke
         - build_profiling
+        - spotless
       name: ok_to_test
       stage: ok_to_test
 
@@ -1299,6 +1434,24 @@ build_test_jobs: &build_test_jobs
       cacheType: base
       testJvm: "8"
 
+  - test_published_artifacts:
+      requires:
+        - ok_to_test
+
+  - muzzle:
+      requires:
+        - ok_to_test
+      filters:
+        branches:
+          ignore:
+            - master
+            - project/*
+            - release/*
+
+  - muzzle-dep-report:
+      requires:
+        - ok_to_test
+
   - system-tests:
       requires:
         - ok_to_test
@@ -1319,6 +1472,7 @@ build_test_jobs: &build_test_jobs
 
   - fan_in:
       requires:
+        - test_published_artifacts
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}_profiling"
 {% endfor %}
@@ -1327,6 +1481,7 @@ build_test_jobs: &build_test_jobs
 
   - fan_in:
       requires:
+        - test_published_artifacts
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}_debugger"
 {% endfor %}
@@ -1338,11 +1493,13 @@ build_test_jobs: &build_test_jobs
   - fan_in:
       requires:
         - check
+        - test_published_artifacts
         - agent_integration_tests
 {% for jdk in all_jdks %}
         - "test_{{ jdk }}"
 {% endfor %}
         - test_inst_latest
+        - muzzle
         - profiling
         - debugger
         - system-tests

.github/CODEOWNERS

@@ -65,3 +65,10 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 # @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark-executor/   @DataDog/data-jobs-monitoring
+
+# @DataDog/data-streams-monitoring
+dd-java-agent/testing/src/main/groovy/datadog/trace/agent/test/datastreams  @DataDog/data-streams-monitoring
+dd-trace-core/src/main/java/datadog/trace/core/datastreams                  @DataDog/data-streams-monitoring
+dd-trace-core/src/test/groovy/datadog/trace/core/datastreams                @DataDog/data-streams-monitoring
+internal-api/src/main/java/datadog/trace/api/datastreams                    @DataDog/data-streams-monitoring
+internal-api/src/test/groovy/datadog/trace/api/datastreams                  @DataDog/data-streams-monitoring

.github/workflows/README.md

@@ -87,7 +87,7 @@ _Recovery:_ Check at the milestone for the related issues and update them manual
 
 ### prune-github-container-registry [🔗](prune-github-container-registry.yaml)
 
-_Trigger:_ Every week or manually.
+_Trigger:_ Every day or manually.
 
 _Action:_ Clean up old lib-injection OCI images from GitHub Container Registry.
 

.github/workflows/add-release-to-cloudfoundry.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout "cloudfoundry" branch
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           ref: cloudfoundry
       - name: Get release version
@@ -43,7 +43,7 @@ jobs:
         run: |
           echo "${{ steps.get-release-version.outputs.VERSION }}: ${{ steps.get-release-url.outputs.URL }}" >> index.yml
       - name: Commit and push changes
-        uses: planetscale/ghcommit-action@b68767a2e130a71926b365322e62b583404a5e09 # v0.1.43
+        uses: planetscale/ghcommit-action@9400254a26464337cbe5af17c5f25075134e0089 # v0.2.7
         with:
           commit_message: "chore: Add version ${{ steps.get-release-version.outputs.VERSION }} to Cloud Foundry"
           repo: ${{ github.repository }}

.github/workflows/analyze-changes.yaml

@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
       # Run the static analysis on the staging environment to benefit from the new features not yet released
       - name: Check code meets quality standards (staging)
         id: datadog-static-analysis-staging
-        uses: DataDog/datadog-static-analyzer-github-action@c74aff158c8cc1c3e285660713bcaa5f9c6d696e # v1
+        uses: DataDog/datadog-static-analyzer-github-action@1297a546e6bb268e2ac5bc98a1477d22be335822 # v1
         with:
           dd_app_key: ${{ secrets.DATADOG_APP_KEY_STAGING }}
           dd_api_key: ${{ secrets.DATADOG_API_KEY_STAGING }}
@@ -44,12 +44,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.gradle/caches
@@ -59,7 +59,7 @@ jobs:
             ${{ runner.os }}-gradle-
         
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/init@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         with:
           languages: 'java'
           build-mode: 'manual'
@@ -76,7 +76,7 @@ jobs:
             --build-cache --parallel --stacktrace --no-daemon --max-workers=4
 
       - name: Perform CodeQL Analysis and upload results to GitHub Security tab
-        uses: github/codeql-action/analyze@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/analyze@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
 
       # For now, CodeQL SARIF results are not supported by Datadog CI
       # - name: Upload results to Datadog CI Static Analysis
@@ -106,12 +106,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # 4.1.6
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
         with:
           submodules: 'recursive'
 
       - name: Cache Gradle dependencies
-        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             ~/.gradle/caches
@@ -144,7 +144,7 @@ jobs:
           ls -laR "./workspace/.trivy"
 
       - name: Run Trivy security scanner
-        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
         with:
           scan-type: rootfs
           scan-ref: './workspace/.trivy/'
@@ -157,7 +157,7 @@ jobs:
           TRIVY_JAVA_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-java-db,public.ecr.aws/aquasecurity/trivy-java-db
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@4dd16135b69a43b6c8efb853346f8437d92d3c93 # v3.26.6
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         if: always()
         with:
           sarif_file: 'trivy-results.sarif'

.github/workflows/prune-github-container-registry.yaml

@@ -20,8 +20,6 @@ jobs:
           container: dd-trace-java/dd-lib-java-init
           keep-younger-than: 7 # days
           keep-last: 10
-          keep-tags: |
-            latest_snapshot
           prune-tags-regexes: |
-            ^[a-z0-9]{40}$
+            .+
           prune-untagged: true

.github/workflows/prune-old-pull-requests.yaml

@@ -13,7 +13,7 @@ jobs:
       pull-requests: write
     steps:
     - name: Prune old pull requests
-      uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+      uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
       with:
         days-before-stale: -1 # Disable general stale bot
         days-before-pr-stale: 90 # Only enable stale bot for PRs with no activity for 90 days