[8.16](backport #6972) [CI] Fixed container images packaging

.buildkite/bk.integration.pipeline.yml

@@ -27,6 +27,8 @@ steps:
       - integration-ess
     steps:
       - label: "Win2022:sudo:{{matrix}}"
+        depends_on:
+          - packaging-windows
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
@@ -50,6 +52,8 @@ steps:
           - install-uninstall
 
       - label: "Win2022:non-sudo:{{matrix}}"
+        depends_on:
+          - packaging-windows
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
@@ -67,6 +71,8 @@ steps:
           - default
 
       - label: "Win2025:sudo:{{matrix}}"
+        depends_on:
+          - packaging-windows
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
@@ -77,6 +83,9 @@ steps:
           provider: "gcp"
           machineType: "n1-standard-8"
           image: "family/platform-ingest-elastic-agent-windows-2025"
+        retry:
+          automatic:
+            limit: 1
         matrix:
           - default
           - fleet
@@ -88,12 +97,17 @@ steps:
           - install-uninstall
 
       - label: "Win2025:non-sudo:{{matrix}}"
+        depends_on:
+          - packaging-windows
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
           - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
         agents:
           provider: "gcp"
           machineType: "n1-standard-8"
@@ -107,7 +121,7 @@ steps:
       - integration-ess
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
-        # only packaging-ubuntu-x86-64 artifact dependency is required
+        depends_on: packaging-ubuntu-x86-64
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -125,9 +139,11 @@ steps:
           - default
 
       - label: "x86_64:sudo: {{matrix}}"
+        depends_on:
+          - packaging-ubuntu-x86-64
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** .
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64          
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -155,6 +171,8 @@ steps:
           - container
 
       - label: "arm:sudo: {{matrix}}"
+        depends_on:
+          - packaging-ubuntu-arm64
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
@@ -186,6 +204,8 @@ steps:
 
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
+        depends_on:
+          - packaging-ubuntu-arm64
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
@@ -208,8 +228,10 @@ steps:
       - integration-ess
     steps:
       - label: "x86_64:sudo:rpm"
+        depends_on:
+          - packaging-ubuntu-x86-64
         command: |
-          buildkite-agent artifact download build/distributions/** .
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -8,38 +8,42 @@ steps:
   - group: "Integration tests: packaging"
     key: "int-packaging"
     steps:
-      - label: "Packaging: Ubuntu x86_64"
-        key: "packaging-ubuntu-x86-64"
+      # Build matrix is not used for packaging in favor to unique step keys
+      # Packaging linux/amd64
+      - label: "Packaging: linux/amd64 rpm"
+        key: packaging-ubuntu-x86-64
         env:
-          PACKAGES: "tar.gz"
           PLATFORMS: "linux/amd64"
+          PACKAGES: "tar.gz,rpm,deb"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
-        - build/distributions/**
+          - build/distributions/**
         retry:
           automatic:
             limit: 1
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
 
-      - label: "Packaging: Ubuntu arm64"
-        key: "packaging-ubuntu-arm64"
+      # Packaging linux/arm64
+      - label: "Packaging: linux/arm64 tar.gz"
+        key: packaging-ubuntu-arm64
         env:
-          PACKAGES: "tar.gz"
           PLATFORMS: "linux/arm64"
+          PACKAGES: "tar.gz"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
           - build/distributions/**
         retry:
           automatic:
             limit: 1
         agents:
-          provider: "gcp"
-          machineType: "n2-standard-8"
+          provider: "aws"
+          instanceType: "c6g.4xlarge"
+          imagePrefix: "core-ubuntu-2204-aarch64"
 
-      - label: "Packaging: Windows"
-        key: "packaging-windows"
+      - label: "Packaging: windows/amd64 zip"
+        key: packaging-windows
         env:
           PACKAGES: "zip"
           PLATFORMS: "windows/amd64"
@@ -53,28 +57,31 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
 
-      - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
-        key: "packaging-containers"
+      - label: "Packaging: Containers linux/amd64"
+        key: packaging-containers-x86-64
         env:
-          PACKAGES: "{{matrix.ext}}"
-          PLATFORMS: "{{matrix.arch}}"
+          PACKAGES: "docker"
+          PLATFORMS: "linux/amd64"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
           - build/distributions/**
-        retry:
-          automatic:
-            limit: 1
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
-        matrix:
-          setup:
-            arch:
-              - linux/amd64
-              - linux/arm64
-            ext:
-              - rpm
-              - deb
+
+      - label: "Packaging: Containers linux/arm64"
+        key: packaging-containers-arm64
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/arm64"
+        command: |
+          .buildkite/scripts/steps/integration-package.sh
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          instanceType: "c6g.4xlarge"
+          imagePrefix: "core-ubuntu-2204-aarch64"
 
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
@@ -131,9 +138,8 @@ steps:
     command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
 
   - label: "Serverless Beats Tests"
-    # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
     depends_on:
-      - packaging-ubuntu-x86-64
+      - int-packaging
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8

dev-tools/mage/crossbuild.go

@@ -229,7 +229,7 @@ func CrossBuildImage(platform string) (string, error) {
 	case platform == "darwin/arm64" || platform == "darwin/universal":
 		tagSuffix = "darwin-arm64-debian10"
 	case platform == "linux/arm64":
-		tagSuffix = "arm"
+		tagSuffix = "base-arm-debian9"
 	case platform == "linux/armv5" || platform == "linux/armv6":
 		tagSuffix = "armel"
 	case platform == "linux/armv7":