[CI] Fixed container images packaging

.buildkite/bk.integration.pipeline.yml

@@ -102,9 +102,8 @@ steps:
       - integration-ess
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
-        # only packaging-ubuntu-x86-64 artifact dependency is required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-tar'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -119,7 +118,8 @@ steps:
       - label: "x86_64:sudo: {{matrix}}"
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** .
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64-tar
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64-deb
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -147,7 +147,8 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-tar'
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-deb'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -175,7 +176,7 @@ steps:
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-tar'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -194,7 +195,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** .
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64-rpm
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -8,32 +8,83 @@ steps:
   - group: "Integration tests: packaging"
     key: "int-packaging"
     steps:
-      - label: "Packaging: Ubuntu x86_64"
-        key: "packaging-ubuntu-x86-64"
+      # Build matrix is not used for packaging in favor to unique step keys
+      # Packaging linux/amd64
+      - label: "Packaging: linux/amd64 rpm"
+        key: packaging-ubuntu-x86-64-rpm
+        env:
+          PLATFORMS: "linux/amd64"
+          PACKAGES: "rpm"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: linux/amd64 deb"
+        key: packaging-ubuntu-x86-64-deb
         env:
-          PACKAGES: "tar.gz"
           PLATFORMS: "linux/amd64"
+          PACKAGES: "deb"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
-        - build/distributions/**
+          - build/distributions/**
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
 
-      - label: "Packaging: Ubuntu arm64"
-        key: "packaging-ubuntu-arm64"
+      - label: "Packaging: linux/amd64 tar.gz"
+        key: packaging-ubuntu-x86-64-tar
         env:
+          PLATFORMS: "linux/amd64"
           PACKAGES: "tar.gz"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      # Packaging linux/arm64
+      - label: "Packaging: linux/arm64 rpm"
+        key: packaging-ubuntu-arm64-rpm
+        env:
           PLATFORMS: "linux/arm64"
+          PACKAGES: "rpm"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
           - build/distributions/**
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
 
-      - label: "Packaging: Windows"
-        key: "packaging-windows"
+      - label: "Packaging: linux/arm64 deb"
+        key: packaging-ubuntu-arm64-deb
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "deb"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: linux/arm64 tar.gz"
+        key: packaging-ubuntu-arm64-tar
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "tar.gz"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: windows/amd64 zip"
+        key: packaging-windows
         env:
           PACKAGES: "zip"
           PLATFORMS: "windows/amd64"
@@ -44,25 +95,34 @@ steps:
           provider: "gcp"
           machineType: "n2-standard-8"
 
-      - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
-        key: "packaging-containers"
+      - label: "Packaging: Containers linux/amd64"
+        key: packaging-containers-x86-64
         env:
-          PACKAGES: "{{matrix.ext}}"
-          PLATFORMS: "{{matrix.arch}}"
+          PACKAGES: "docker"
+          PLATFORMS: "linux/amd64"
         command: ".buildkite/scripts/steps/integration-package.sh"
         artifact_paths:
           - build/distributions/**
         agents:
           provider: "gcp"
           machineType: "n2-standard-8"
-        matrix:
-          setup:
-            arch:
-              - linux/amd64
-              - linux/arm64
-            ext:
-              - rpm
-              - deb
+
+      - label: "Packaging: Containers linux/arm64"
+        key: packaging-containers-arm64
+        env:
+          PACKAGES: "docker"
+          PLATFORMS: "linux/arm64"
+        command: |
+          # echo "Add support for multiarch"
+          # docker run --privileged --rm tonistiigi/binfmt:qemu-v9.2.0-51 --uninstall qemu-*
+          # docker run --privileged --rm tonistiigi/binfmt:qemu-v9.2.0-51 --install all
+          .buildkite/scripts/steps/integration-package.sh
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          instanceType: "c6g.4xlarge"
+          imagePrefix: "core-ubuntu-2204-aarch64"
 
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
@@ -113,9 +173,8 @@ steps:
     command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
 
   - label: "Serverless Beats Tests"
-    # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
     depends_on:
-      - packaging-ubuntu-x86-64
+      - int-packaging
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8