From 723e0caf0917a0c598c43ad306d58317418f71a6 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 24 Feb 2025 11:59:55 +0200 Subject: [PATCH] Added binfmt to beats packaging step (#42825) Added binfmt docker to run before packaging for multiarch builds --------- Co-authored-by: Dimitrios Liappis Co-authored-by: Victor Martinez (cherry picked from commit 5409677da6682c0a4aae8b675658f3a893099a96) --- .buildkite/auditbeat/auditbeat-pipeline.yml | 8 ++------ .buildkite/filebeat/filebeat-pipeline.yml | 6 ++---- .buildkite/heartbeat/heartbeat-pipeline.yml | 6 ++---- .buildkite/metricbeat/pipeline.yml | 6 ++---- .buildkite/packetbeat/pipeline.packetbeat.yml | 6 ++---- .buildkite/scripts/packaging/packaging.sh | 19 +++++++++++++++++++ .buildkite/winlogbeat/pipeline.winlogbeat.yml | 3 +-- .../x-pack/pipeline.xpack.agentbeat.yml | 4 +--- .../x-pack/pipeline.xpack.auditbeat.yml | 6 ++---- .../x-pack/pipeline.xpack.dockerlogbeat.yml | 6 ++---- .buildkite/x-pack/pipeline.xpack.filebeat.yml | 6 ++---- .../x-pack/pipeline.xpack.heartbeat.yml | 6 ++---- .../x-pack/pipeline.xpack.metricbeat.yml | 6 ++---- .../x-pack/pipeline.xpack.osquerybeat.yml | 3 +-- .../x-pack/pipeline.xpack.packetbeat.yml | 6 ++---- .../x-pack/pipeline.xpack.winlogbeat.yml | 3 +-- 16 files changed, 45 insertions(+), 55 deletions(-) create mode 100755 .buildkite/scripts/packaging/packaging.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index b47ee4f15f2c..2d104a6b79a0 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -382,9 +382,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - set -euo pipefail - cd auditbeat - mage package + .buildkite/scripts/packaging/packaging.sh auditbeat retry: automatic: - limit: 1 @@ -403,9 +401,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - set -euo pipefail - cd auditbeat - mage package + .buildkite/scripts/packaging/packaging.sh auditbeat retry: automatic: - limit: 1 diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 13f79cd04193..83c3c1091673 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -333,8 +333,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd filebeat - mage package + .buildkite/scripts/packaging/packaging.sh filebeat retry: automatic: - limit: 1 @@ -355,8 +354,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd filebeat - mage package + .buildkite/scripts/packaging/packaging.sh filebeat retry: automatic: - limit: 1 diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 13b0c8035b4f..7d07f804f2e1 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -343,8 +343,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd heartbeat - mage package + .buildkite/scripts/packaging/packaging.sh heartbeat retry: automatic: - limit: 1 @@ -363,8 +362,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd heartbeat - mage package + .buildkite/scripts/packaging/packaging.sh heartbeat retry: automatic: - limit: 1 diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index fab2c734ceee..2d2b0d871cd3 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -339,8 +339,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd metricbeat - mage package + .buildkite/scripts/packaging/packaging.sh metricbeat retry: automatic: - limit: 1 @@ -361,8 +360,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd metricbeat - mage package + .buildkite/scripts/packaging/packaging.sh metricbeat retry: automatic: - limit: 1 diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index fe36831cfc4a..1abaa33fc6ed 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -306,8 +306,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd packetbeat - mage package + .buildkite/scripts/packaging/packaging.sh packetbeat retry: automatic: - limit: 1 @@ -328,8 +327,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd packetbeat - mage package + .buildkite/scripts/packaging/packaging.sh packetbeat retry: automatic: - limit: 1 diff --git a/.buildkite/scripts/packaging/packaging.sh b/.buildkite/scripts/packaging/packaging.sh new file mode 100755 index 000000000000..43d3ae19bfa5 --- /dev/null +++ b/.buildkite/scripts/packaging/packaging.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Centralise the mage package for a given beat in Buildkite. +# It enables multi-arch builds to avoid the exec format errors when +# attempting to build arm64 inside arm64 workers. +# For further details, see https://github.com/elastic/elastic-agent/pull/6948 +# and https://github.com/elastic/golang-crossbuild/pull/507 +# + +set -ueo pipefail + + +BEAT_DIR=${1:?-"Error: Beat directory must be specified."} + +#Use newer multiarch support for packaging +docker run --privileged --rm tonistiigi/binfmt:master --install all + +cd $BEAT_DIR +mage package diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index d8986a72a54e..e205fb9e8354 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -209,8 +209,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd winlogbeat - mage package + .buildkite/scripts/packaging/packaging.sh winlogbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml index 092cfdf9f236..14f139881af9 100644 --- a/.buildkite/x-pack/pipeline.xpack.agentbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.agentbeat.yml @@ -70,9 +70,7 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true command: | - set -euo pipefail - cd x-pack/agentbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/agentbeat artifact_paths: - x-pack/agentbeat/build/distributions/**/* - "x-pack/agentbeat/build/*.xml" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 18d2702d231c..76d089428a91 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -314,8 +314,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/auditbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/auditbeat retry: automatic: - limit: 1 @@ -336,8 +335,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/auditbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/auditbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index 54b3451b23e0..85b8634b90f0 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -117,8 +117,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/dockerlogbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/dockerlogbeat retry: automatic: - limit: 1 @@ -137,8 +136,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/dockerlogbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/dockerlogbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 20087debbeed..d744b9040d09 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -358,8 +358,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/filebeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/filebeat retry: automatic: - limit: 1 @@ -380,8 +379,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/filebeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/filebeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index e719f0107dfa..192c87582a3f 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -307,8 +307,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/heartbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/heartbeat retry: automatic: - limit: 1 @@ -329,8 +328,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/heartbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/heartbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index d37278f65faf..5adec8e73238 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -342,8 +342,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/metricbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/metricbeat retry: automatic: - limit: 1 @@ -364,8 +363,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/metricbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/metricbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index c06f473c000c..89021df0e166 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -283,8 +283,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/osquerybeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/osquerybeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index db47a5b62dfe..4454f803e313 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -373,8 +373,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/packetbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/packetbeat retry: automatic: - limit: 1 @@ -395,8 +394,7 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: | - cd x-pack/packetbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/packetbeat retry: automatic: - limit: 1 diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index f0a1b30c2536..582f4d1f277a 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -210,8 +210,7 @@ steps: env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" command: | - cd x-pack/winlogbeat - mage package + .buildkite/scripts/packaging/packaging.sh x-pack/winlogbeat retry: automatic: - limit: 1