[9.0](backport #3347) DRA: separate arm64 (#3350)

DRA: separate arm64 (#3347)

* DRA: separate arm64

* more logs

* more logs

* fix artifact download

* linter fix

* cleanup

* Update .buildkite/scripts/publish.sh



---------


(cherry picked from commit db161e1)

Co-authored-by: Kostas Stamatakis <[email protected]>
Co-authored-by: Orestis Floros <[email protected]>

Author: 3 people

.buildkite/pipeline.yml

@@ -1,36 +1,88 @@
 env:
   BRANCH: "${BUILDKITE_BRANCH}"
-agents:
-  provider: "gcp" # needed for running docker commands
-  image: "family/platform-ingest-beats-ubuntu-2204"
+
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-standard-8"
+  GCP_IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  PLATFORMS_AMD64: "linux/amd64"
+  PLATFORMS_ARM64: "linux/arm64"
+
+  DEV: false
 
 steps:
-  - label: ":package: Package Cloudbeat - Snapshot"
+
+  - group: "Package Snapshot"
     if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true"
-    env:
-      WORKFLOW: "snapshot"
     key: "package-snapshot"
-    command: "./.buildkite/scripts/package.sh"
-    artifact_paths: "build/distributions/*"
+    steps:
+      - label: ":package: Package Cloudbeat linux/amd64 - Snapshot"
+        env:
+          PLATFORMS: "${PLATFORMS_AMD64}"
+          WORKFLOW: "snapshot"
+        command: "./.buildkite/scripts/package.sh"
+        artifact_paths: "build/distributions/*"
+        agents:
+          provider: gcp
+          image: "${GCP_IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+
+      - label: ":package: Package Cloudbeat linux/arm64 - Snapshot"
+        env:
+          PLATFORMS: "${PLATFORMS_ARM64}"
+          WORKFLOW: "snapshot"
+        command: "./.buildkite/scripts/package.sh"
+        artifact_paths: "build/distributions/*"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
 
   - label: ":rocket: Publishing Snapshot DRA artifacts"
     if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true"
     depends_on: "package-snapshot"
+    agents:
+      provider: gcp
+      image: "${GCP_IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
     command: "./.buildkite/scripts/publish.sh"
     env:
       WORKFLOW: "snapshot"
 
-  - label: ":package: Package Cloudbeat - Staging"
+  - group: "Package Staging"
     if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
-    env:
-      WORKFLOW: "staging"
     key: "package-staging"
-    command: "./.buildkite/scripts/package.sh"
-    artifact_paths: "build/distributions/*"
+    steps:
+      - label: ":package: Package Cloudbeat linux/amd64 - Staging"
+        env:
+          PLATFORMS: "${PLATFORMS_AMD64}"
+          WORKFLOW: "staging"
+        command: "./.buildkite/scripts/package.sh"
+        artifact_paths: "build/distributions/*"
+        agents:
+          provider: gcp
+          image: "${GCP_IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+
+      - label: ":package: Package Cloudbeat linux/arm64 - Staging"
+        env:
+          PLATFORMS: "${PLATFORMS_ARM64}"
+          WORKFLOW: "staging"
+        command: "./.buildkite/scripts/package.sh"
+        artifact_paths: "build/distributions/*"
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
 
   - label: ":rocket: Publishing Staging DRA artifacts"
     if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
     depends_on: "package-staging"
+    agents:
+      provider: gcp
+      image: "${GCP_IMAGE_UBUNTU_X86_64}"
+      machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
     command: "./.buildkite/scripts/publish.sh"
     env:
       WORKFLOW: "staging"

.buildkite/scripts/package.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -uox pipefail
 
-export PLATFORMS="linux/amd64,linux/arm64"
 export TYPES="tar.gz"
 source ./bin/activate-hermit
 
@@ -17,9 +16,6 @@ if [ "$WORKFLOW" = "snapshot" ]; then
     export SNAPSHOT="true"
 fi
 
-# debug command to verify
-ls -lah /proc/sys/fs/binfmt_misc/ || true
-
 mage pythonEnv
 mage package
 
@@ -32,3 +28,6 @@ fi
 echo "Generating $CSV_FILE.csv"
 $PYTHON ./.buildkite/scripts/generate_notice.py --csv "$CSV_FILE.csv"
 cp build/dependencies-*.csv build/distributions/.
+
+echo "Produced artifacts:"
+ls -lahR build/distributions/

.buildkite/scripts/publish.sh

@@ -1,5 +1,13 @@
 #!/usr/bin/env bash
 
+if [[ "${DRY_RUN}" == "true" ]]; then
+    echo "~~~ Running in dry-run mode -- will NOT publish artifacts"
+    DRY_RUN="--dry-run"
+else
+    echo "~~~ Running in publish mode"
+    DRY_RUN=""
+fi
+
 # Allow other users write access to create checksum files
 
 # The "branch" here selects which "$BRANCH.gradle" file of release manager is used
@@ -19,9 +27,12 @@ echo "VERSION_QUALIFIER: ${VERSION_QUALIFIER}"
 
 # Download artifacts from other stages
 echo "Downloading artifacts..."
-buildkite-agent artifact download "build/distributions/*" "." --step package-"${WORKFLOW}"
+buildkite-agent artifact download "build/distributions/*" "."
 chmod -R 777 build/distributions
 
+echo "Downloaded artifacts:"
+ls -lahR build/distributions/
+
 # Shared secret path containing the dra creds for project teams
 DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role)
 
@@ -42,4 +53,13 @@ docker run --rm \
     --workflow "${WORKFLOW}" \
     --version "${VERSION}" \
     --artifact-set main \
-    --qualifier "${VERSION_QUALIFIER}"
+    --qualifier "${VERSION_QUALIFIER}" ${DRY_RUN} | tee rm-output.txt
+
+if [[ "$DRY_RUN" != "--dry-run" ]]; then
+    # extract the summary URL from a release manager output line like:
+    SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}')
+    # builkite annotation
+    printf "**${WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
+fi
+
+rm -f rm-output.txt