diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a3bd24aa8c..6e446ff14d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,36 +1,88 @@ env: BRANCH: "${BUILDKITE_BRANCH}" -agents: - provider: "gcp" # needed for running docker commands - image: "family/platform-ingest-beats-ubuntu-2204" + + GCP_DEFAULT_MACHINE_TYPE: "c2d-standard-8" + GCP_IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + + PLATFORMS_AMD64: "linux/amd64" + PLATFORMS_ARM64: "linux/arm64" + + DEV: false steps: - - label: ":package: Package Cloudbeat - Snapshot" + + - group: "Package Snapshot" if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true" - env: - WORKFLOW: "snapshot" key: "package-snapshot" - command: "./.buildkite/scripts/package.sh" - artifact_paths: "build/distributions/*" + steps: + - label: ":package: Package Cloudbeat linux/amd64 - Snapshot" + env: + PLATFORMS: "${PLATFORMS_AMD64}" + WORKFLOW: "snapshot" + command: "./.buildkite/scripts/package.sh" + artifact_paths: "build/distributions/*" + agents: + provider: gcp + image: "${GCP_IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - label: ":package: Package Cloudbeat linux/arm64 - Snapshot" + env: + PLATFORMS: "${PLATFORMS_ARM64}" + WORKFLOW: "snapshot" + command: "./.buildkite/scripts/package.sh" + artifact_paths: "build/distributions/*" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" - label: ":rocket: Publishing Snapshot DRA artifacts" if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true" depends_on: "package-snapshot" + agents: + provider: gcp + image: "${GCP_IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" command: "./.buildkite/scripts/publish.sh" env: WORKFLOW: "snapshot" - - label: ":package: Package Cloudbeat - Staging" + - group: "Package Staging" if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true" - env: - WORKFLOW: "staging" key: "package-staging" - command: "./.buildkite/scripts/package.sh" - artifact_paths: "build/distributions/*" + steps: + - label: ":package: Package Cloudbeat linux/amd64 - Staging" + env: + PLATFORMS: "${PLATFORMS_AMD64}" + WORKFLOW: "staging" + command: "./.buildkite/scripts/package.sh" + artifact_paths: "build/distributions/*" + agents: + provider: gcp + image: "${GCP_IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - label: ":package: Package Cloudbeat linux/arm64 - Staging" + env: + PLATFORMS: "${PLATFORMS_ARM64}" + WORKFLOW: "staging" + command: "./.buildkite/scripts/package.sh" + artifact_paths: "build/distributions/*" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" - label: ":rocket: Publishing Staging DRA artifacts" if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true" depends_on: "package-staging" + agents: + provider: gcp + image: "${GCP_IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" command: "./.buildkite/scripts/publish.sh" env: WORKFLOW: "staging" diff --git a/.buildkite/scripts/package.sh b/.buildkite/scripts/package.sh index 71cc7fa2d3..272beda608 100644 --- a/.buildkite/scripts/package.sh +++ b/.buildkite/scripts/package.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -uox pipefail -export PLATFORMS="linux/amd64,linux/arm64" export TYPES="tar.gz" source ./bin/activate-hermit @@ -13,9 +12,6 @@ if [ "$WORKFLOW" = "snapshot" ]; then export SNAPSHOT="true" fi -# debug command to verify -ls -lah /proc/sys/fs/binfmt_misc/ || true - mage pythonEnv mage package @@ -25,3 +21,6 @@ CSV_FILE="build/dependencies-${CLOUDBEAT_VERSION}" echo "Generating $CSV_FILE.csv" $PYTHON ./.buildkite/scripts/generate_notice.py --csv "$CSV_FILE.csv" cp build/dependencies-*.csv build/distributions/. + +echo "Produced artifacts:" +ls -lahR build/distributions/ diff --git a/.buildkite/scripts/publish.sh b/.buildkite/scripts/publish.sh index 4d7684b96d..2d5f4ad8a6 100644 --- a/.buildkite/scripts/publish.sh +++ b/.buildkite/scripts/publish.sh @@ -1,5 +1,13 @@ #!/usr/bin/env bash +if [[ "${DRY_RUN}" == "true" ]]; then + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" + DRY_RUN="--dry-run" +else + echo "~~~ Running in publish mode" + DRY_RUN="" +fi + # Allow other users write access to create checksum files # The "branch" here selects which "$BRANCH.gradle" file of release manager is used @@ -16,9 +24,12 @@ fi # Download artifacts from other stages echo "Downloading artifacts..." -buildkite-agent artifact download "build/distributions/*" "." --step package-"${WORKFLOW}" +buildkite-agent artifact download "build/distributions/*" "." chmod -R 777 build/distributions +echo "Downloaded artifacts:" +ls -lahR build/distributions/ + # Shared secret path containing the dra creds for project teams DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role) @@ -38,4 +49,13 @@ docker run --rm \ --commit "${BUILDKITE_COMMIT}" \ --workflow "${WORKFLOW}" \ --version "${VERSION}" \ - --artifact-set main + --artifact-set main ${DRY_RUN} | tee rm-output.txt + +if [[ "$DRY_RUN" != "--dry-run" ]]; then + # extract the summary URL from a release manager output line like: + SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') + # builkite annotation + printf "**${WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append +fi + +rm -f rm-output.txt