From cfc069455dee430b6926bdeeffeb488e0c118fde Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 11:13:05 +0200 Subject: [PATCH 1/7] Don't package arm64 on amd64 workers This commit is the counterpart of https://github.com/elastic/beats/pull/43019 for the DRA packaging pipeline. It ensures that arm64 packages get built without qemu emulation on dedicated arm64 workers. --- .buildkite/packaging.pipeline.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 34b103b23736..204cd635fa0b 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -8,8 +8,8 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-standard-8" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - PLATFORMS_ARM: "linux/arm64" + PLATFORMS: "+all linux/amd64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS_ARM: "+all linux/arm64" steps: # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) @@ -124,7 +124,6 @@ steps: - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" - PACKAGES: "docker" SNAPSHOT: true # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 DEV: false @@ -151,6 +150,7 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat + - x-pack/osquerybeat ## Agentbeat needs more CPUs because it builds many other beats - label: "SNAPSHOT: x-pack/agentbeat" @@ -242,6 +242,7 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat + - x-pack/osquerybeat ## Agentbeat needs more CPUs because it builds many other beats - label: "STAGING: x-pack/agentbeat" From c144eea7a758a0716c0a69cef5ba5db174242142 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 13:27:13 +0200 Subject: [PATCH 2/7] Use dedicate build for x-pack/osquerybeat and x-pack/agentbeat on arm64 to produce all artifacts --- .buildkite/packaging.pipeline.yml | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 204cd635fa0b..bafa56e125e2 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -124,6 +124,7 @@ steps: - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" SNAPSHOT: true # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 DEV: false @@ -150,6 +151,26 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat + + - label: "SNAPSHOT: {{matrix}} all artifacts for Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + SNAPSHOT: true + # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat - x-pack/osquerybeat ## Agentbeat needs more CPUs because it builds many other beats @@ -244,6 +265,27 @@ steps: - x-pack/packetbeat - x-pack/osquerybeat + - label: "STAGING: {{matrix} all artifacts for Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + SNAPSHOT: false + DEV: false + command: | + source .buildkite/scripts/version_qualifier.sh + .buildkite/scripts/packaging/package-dra.sh x-pack/osquerybeat + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 + artifact_paths: + - build/distributions/**/* + matrix: + - x-pack/agentbeat + ## Agentbeat needs more CPUs because it builds many other beats - label: "STAGING: x-pack/agentbeat" env: From 435bdc0488fe5f07212524002365358f285711d7 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 13:57:12 +0200 Subject: [PATCH 3/7] bug fix --- .buildkite/packaging.pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index bafa56e125e2..5843310636b5 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -272,7 +272,7 @@ steps: DEV: false command: | source .buildkite/scripts/version_qualifier.sh - .buildkite/scripts/packaging/package-dra.sh x-pack/osquerybeat + .buildkite/scripts/packaging/package-dra.sh {{matrix}} agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -285,6 +285,7 @@ steps: - build/distributions/**/* matrix: - x-pack/agentbeat + - x-pack/osquerybeat ## Agentbeat needs more CPUs because it builds many other beats - label: "STAGING: x-pack/agentbeat" From 94fc16232cd6c062c769f9d48201d60c0c053172 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 14:10:42 +0200 Subject: [PATCH 4/7] Final bug fixes --- .buildkite/packaging.pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 5843310636b5..91b39b164dc7 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -263,9 +263,8 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat - - x-pack/osquerybeat - - label: "STAGING: {{matrix} all artifacts for Linux/arm64" + - label: "STAGING: {{matrix}} all artifacts for Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" SNAPSHOT: false From 8cc9251af50a2c425d6775c87af959c4884a3ee4 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 14:27:42 +0200 Subject: [PATCH 5/7] Rely on multiarch fpm --- .buildkite/packaging.pipeline.yml | 53 ++++--------------------------- 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index 91b39b164dc7..6063cd8c7164 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -121,10 +121,9 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" + - label: "SNAPSHOT: {{matrix}} Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" - PACKAGES: "docker" SNAPSHOT: true # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 DEV: false @@ -151,30 +150,11 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat - - - label: "SNAPSHOT: {{matrix}} all artifacts for Linux/arm64" - env: - PLATFORMS: "${PLATFORMS_ARM}" - SNAPSHOT: true - # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270 - DEV: false - command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - timeout_in_minutes: 40 - retry: - automatic: - - limit: 1 - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - x-pack/osquerybeat + - x-pack/agentbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "SNAPSHOT: x-pack/agentbeat" + - label: "SNAPSHOT: x-pack/agentbeat all artifacts apart from linux/arm64" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: true @@ -232,10 +212,9 @@ steps: - x-pack/packetbeat - x-pack/winlogbeat - - label: "STAGING: {{matrix}} docker Linux/arm64" + - label: "STAGING: {{matrix}} Linux/arm64" env: PLATFORMS: "${PLATFORMS_ARM}" - PACKAGES: "docker" SNAPSHOT: false DEV: false command: | @@ -263,31 +242,11 @@ steps: - x-pack/heartbeat - x-pack/metricbeat - x-pack/packetbeat - - - label: "STAGING: {{matrix}} all artifacts for Linux/arm64" - env: - PLATFORMS: "${PLATFORMS_ARM}" - SNAPSHOT: false - DEV: false - command: | - source .buildkite/scripts/version_qualifier.sh - .buildkite/scripts/packaging/package-dra.sh {{matrix}} - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - timeout_in_minutes: 40 - retry: - automatic: - - limit: 1 - artifact_paths: - - build/distributions/**/* - matrix: - - x-pack/agentbeat - x-pack/osquerybeat + - x-pack/agentbeat ## Agentbeat needs more CPUs because it builds many other beats - - label: "STAGING: x-pack/agentbeat" + - label: "STAGING: x-pack/agentbeat all artifacts apart from linux/arm64" env: PLATFORMS: "${PLATFORMS}" SNAPSHOT: false From 73c9491f731352b610f5dd29721d804f4fa356c7 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 14:42:54 +0200 Subject: [PATCH 6/7] Allow testing with multiplatform fpm --- dev-tools/mage/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 22c081351e8b..dfad6d12ad8c 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -41,7 +41,7 @@ const ( fpmVersion = "1.13.1" // Docker images. See https://github.com/elastic/golang-crossbuild. - beatsFPMImage = "docker.elastic.co/beats-dev/fpm" + beatsFPMImage = "docker.elastic.co/observability-ci/fpm" // BeatsCrossBuildImage is the image used for crossbuilding Beats. BeatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild" From 88b610f08eda2083b9616c8f84c36a60ca98ec68 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 5 Mar 2025 18:11:01 +0200 Subject: [PATCH 7/7] Revert "Allow testing with multiplatform fpm" This reverts commit 73c9491f731352b610f5dd29721d804f4fa356c7. --- dev-tools/mage/settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index dfad6d12ad8c..22c081351e8b 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -41,7 +41,7 @@ const ( fpmVersion = "1.13.1" // Docker images. See https://github.com/elastic/golang-crossbuild. - beatsFPMImage = "docker.elastic.co/observability-ci/fpm" + beatsFPMImage = "docker.elastic.co/beats-dev/fpm" // BeatsCrossBuildImage is the image used for crossbuilding Beats. BeatsCrossBuildImage = "docker.elastic.co/beats-dev/golang-crossbuild"