From 10ecbb9f19dc503974d05b08fce2b23f06d67232 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 27 Feb 2025 11:34:30 -0500 Subject: [PATCH 1/7] first pass checks --- .gitlab-ci.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bca4b89eed3..d8080952c9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,6 +143,47 @@ test_published_artifacts: paths: - ./check_reports +.check_job: + extends: .gradle_build + needs: [ build ] + stage: tests + variables: + BUILD_CACHE_TYPE: lib + script: + - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh --destination ./check_reports --move + artifacts: + when: always + paths: + - ./check_reports + - '.gradle/daemon/*/*.out.log' + +check_base: + extends: .check_job + variables: + GRADLE_TARGET: ":baseCheck" + +check_inst: + extends: .check_job + variables: + GRADLE_TARGET: ":instrumentationCheck" + +check_smoke: + extends: .check_job + variables: + GRADLE_TARGET: ":smokeCheck" + +check_profiling: + extends: .check_job + variables: + GRADLE_TARGET: ":profilingCheck" + +check_debugger: + extends: .check_job + variables: + GRADLE_TARGET: ":debuggerCheck" + muzzle: extends: .gradle_build needs: [ build ] From 26a7adbd04374c8959ba0b252c85a7a40dda7fa0 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 27 Feb 2025 13:17:15 -0500 Subject: [PATCH 2/7] add some debug --- .circleci/config.continue.yml.j2 | 5 ++++- .gitlab-ci.yml | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index a44d1f95e6d..1bffad9a553 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -410,6 +410,10 @@ jobs: - restore_build_cache: cacheType: << parameters.cacheType >> + - run: + name: Print some vars + command: echo "-PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}" + - run: name: Check Project command: >- @@ -1101,7 +1105,6 @@ build_test_jobs: &build_test_jobs parallelism: 4 gradleTarget: ":instrumentationCheck" cacheType: inst - triggeredBy: *instrumentation_modules - check: requires: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8080952c9a..c9a3deea589 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -150,7 +150,7 @@ test_published_artifacts: variables: BUILD_CACHE_TYPE: lib script: - - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless $GRADLE_ARGS + - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$CI_NODE_TOTAL -PtaskPartition=$CI_NODE_INDEX $GRADLE_ARGS after_script: - .circleci/collect_reports.sh --destination ./check_reports --move artifacts: @@ -166,11 +166,13 @@ check_base: check_inst: extends: .check_job + parallel: 4 variables: GRADLE_TARGET: ":instrumentationCheck" check_smoke: extends: .check_job + parallel: 4 variables: GRADLE_TARGET: ":smokeCheck" From f8448c965d1f0fa18a829d35003c94f3a0633f06 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Fri, 28 Feb 2025 10:40:13 -0500 Subject: [PATCH 3/7] Revert "add some debug" This reverts commit 26a7adbd04374c8959ba0b252c85a7a40dda7fa0. --- .circleci/config.continue.yml.j2 | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 1bffad9a553..a44d1f95e6d 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -410,10 +410,6 @@ jobs: - restore_build_cache: cacheType: << parameters.cacheType >> - - run: - name: Print some vars - command: echo "-PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}" - - run: name: Check Project command: >- @@ -1105,6 +1101,7 @@ build_test_jobs: &build_test_jobs parallelism: 4 gradleTarget: ":instrumentationCheck" cacheType: inst + triggeredBy: *instrumentation_modules - check: requires: From 0b8643a0f8f111dc50a66ca8ad872067757b9f77 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Fri, 28 Feb 2025 10:49:16 -0500 Subject: [PATCH 4/7] Normalize node index and node total --- .gitlab-ci.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c9a3deea589..a33983dd30a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,6 +42,13 @@ default: - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text) - export DATADOG_API_KEY_DDSTAGING=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.dd_api_key --with-decryption --query "Parameter.Value" --out text) +# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs +.normalize_node_index: &normalize_node_index + - echo "CI_NODE_TOTAL=$CI_NODE_TOTAL , CI_NODE_INDEX=$CI_NODE_INDEX" + - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1} + - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((CI_NODE_INDEX - 1)) + - echo "NORMALIZED_NODE_TOTAL=$NORMALIZED_NODE_TOTAL , NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX" + .gradle_build: &gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base stage: build @@ -68,6 +75,7 @@ default: before_script: - export GRADLE_USER_HOME=`pwd`/.gradle - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS" + - *normalize_node_index # for weird reasons, gradle will always "chmod 700" the .gradle folder # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails # This dance is a hack to have .gradle owned by the Gitlab runner user @@ -195,13 +203,10 @@ muzzle: BUILD_CACHE_TYPE: test script: - export SKIP_BUILDSCAN="true" - - echo CI_NODE_INDEX=$CI_NODE_INDEX - - echo CI_NODE_TOTAL=CI_NODE_TOTAL - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS - sort workspace/build/muzzleTasks > sortedMuzzleTasks - - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit - - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1)) - - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS + - split --number=l/$NORMALIZED_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit + - ./gradlew `cat muzzleSplit${NORMALIZED_NODE_INDEX} | xargs` $GRADLE_ARGS after_script: - .circleci/collect_reports.sh artifacts: From 65d41ac5f828a5723b54fc1304a857b60e5d2ea0 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Fri, 28 Feb 2025 13:00:56 -0500 Subject: [PATCH 5/7] fix normalization --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a33983dd30a..41f0555ad28 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -46,7 +46,7 @@ default: .normalize_node_index: &normalize_node_index - echo "CI_NODE_TOTAL=$CI_NODE_TOTAL , CI_NODE_INDEX=$CI_NODE_INDEX" - export NORMALIZED_NODE_TOTAL=${CI_NODE_TOTAL:-1} - - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((CI_NODE_INDEX - 1)) + - ONE_INDEXED_NODE_INDEX=${CI_NODE_INDEX:-1}; export NORMALIZED_NODE_INDEX=$((ONE_INDEXED_NODE_INDEX - 1)) - echo "NORMALIZED_NODE_TOTAL=$NORMALIZED_NODE_TOTAL , NORMALIZED_NODE_INDEX=$NORMALIZED_NODE_INDEX" .gradle_build: &gradle_build From 661749deea7e6f899132ebbf71833cbd448d4a31 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Mon, 3 Mar 2025 10:12:10 -0500 Subject: [PATCH 6/7] actually use the normalized values --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 41f0555ad28..6148e218395 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,7 +158,7 @@ test_published_artifacts: variables: BUILD_CACHE_TYPE: lib script: - - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$CI_NODE_TOTAL -PtaskPartition=$CI_NODE_INDEX $GRADLE_ARGS + - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS after_script: - .circleci/collect_reports.sh --destination ./check_reports --move artifacts: From 5870de682cb58f68fb50c7569599251a8f444e2e Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Mon, 3 Mar 2025 11:58:14 -0500 Subject: [PATCH 7/7] add to required --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 08a667862de..da8ffd29fc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -308,6 +308,16 @@ required: optional: true - job: agent_integration_tests optional: true + - job: check_base + optional: true + - job: check_inst + optional: true + - job: check_smoke + optional: true + - job: check_profiling + optional: true + - job: check_debugger + optional: true deploy_to_profiling_backend: stage: publish