[CI] BK Integration tests: Run non-sudo tests

.buildkite/bk.integration.pipeline.yml

@@ -32,15 +32,16 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
 
-  - group: "Stateful(Sudo): Windows"
+  - group: "Stateful: Windows"
     key: integration-tests-win
     depends_on:
       - integration-ess
+      - package-it
     steps:
-      - label: "Win2022:{{matrix}}"
+      - label: "Win2022:sudo:{{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/integration-tests.ps1 {{matrix}}
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
         agents:
@@ -53,15 +54,42 @@ steps:
           - fleet-privileged
           - upgrade
 
-  - group: "Stateful(Sudo): Ubuntu"
+      - label: "Win2022:non-sudo:{{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} false
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-windows-2022"
+        matrix:
+          - default
+
+  - group: "Stateful:Ubuntu"
     key: integration-tests-ubuntu
     depends_on:
       - integration-ess
+      - package-it
     steps:
-      - label: "x86_64:Group: {{matrix}}"
+      - label: "x86_64:non-sudo: {{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-ubuntu-2404"
+        matrix:
+          - default
+
+      - label: "x86_64:sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
         agents:
@@ -80,10 +108,10 @@ steps:
           - fleet-privileged
           - fleet-airgapped-privileged
 
-      - label: "arm:Group: {{matrix}}"
+      - label: "arm:sudo: {{matrix}}"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh {{matrix}}
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
         agents:
@@ -102,16 +130,29 @@ steps:
           - fleet-privileged
           - fleet-airgapped-privileged
 
-  - group: "Stateful(Sudo): RHEL8"
+      - label: "arm:non-sudo: {{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'package-it'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
+        artifact_paths:
+          - build/**
+        agents:
+          provider: "aws"
+          imagePrefix: "platform-ingest-beats-ubuntu-2404-aarch64"
+          instanceType: "m6g.xlarge"
+        matrix:
+          - default
+
+  - group: "Stateful(Sudo):RHEL8"
     key: integration-tests-rhel8
     depends_on:
       - package-it
       - integration-ess
     steps:
-      - label: "x86_64:Group: rpm"
+      - label: "x86_64:sudo:rpm"
         command: |
           buildkite-agent artifact download build/distributions/** . --step 'package-it'
-          .buildkite/scripts/steps/integration_tests_tf_sudo.sh rpm
+          .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**
         agents:

.buildkite/scripts/sudo-integration-tests.sh → .buildkite/scripts/buildkite-integration-tests.sh

@@ -1,27 +1,42 @@
 #!/usr/bin/env bash
 
-# Fixes source asdf.sh, otherwise it relies on unexisting /root/.asdf directory
-export HOME=/opt/buildkite-agent
 
-# The script is used to run integration tests with sudo
-source /opt/buildkite-agent/hooks/pre-command
-source .buildkite/hooks/pre-command || echo "No pre-command hook found"
+GROUP_NAME=$1
+TEST_SUDO=$2
+
+if [ -z "$GROUP_NAME" ]; then
+  echo "Error: Specify the group name: sudo-integration-tests.sh [group_name]" >&2
+  exit 1
+fi
+
+if [ -z "$TEST_SUDO" ]; then
+  echo "Error: Specify the test sudo: sudo-integration-tests.sh [group_name] [test_sudo]" >&2
+  exit 1
+fi
+
+if [ "$TEST_SUDO" == "true" ]; then
+  echo "Re-initializing ASDF. The user is changed to root..."  
+  export HOME=/opt/buildkite-agent
+  source /opt/buildkite-agent/hooks/pre-command 
+  source .buildkite/hooks/pre-command || echo "No pre-command hook found"
+fi
 
 # Make sure that all tools are installed
 asdf install
 
-GROUP_NAME=$1
-
 echo "~~~ Running integration tests as $USER"
 echo "~~~ Integration tests: ${GROUP_NAME}"
+
 go install gotest.tools/gotestsum
 gotestsum --version
+
 PACKAGE_VERSION="$(cat .package-version)"
 if [[ -n "$PACKAGE_VERSION" ]]; then
     PACKAGE_VERSION=${PACKAGE_VERSION}"-SNAPSHOT"
 fi
+
 set +e
-TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo=true
+TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/scripts/integration-tests.ps1

@@ -1,5 +1,6 @@
 param (
-    [string]$GROUP_NAME
+    [string]$GROUP_NAME,
+    [string]$TEST_SUDO
 )
 
 echo "~~~ Preparing environment"
@@ -26,7 +27,7 @@ mage build:testBinaries
 try {
     Get-Ess-Stack -StackVersion $PACKAGE_VERSION    
     Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
-    gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=true"    
+    gotestsum --no-color -f standard-quiet --junitfile "build/${GROUP_NAME}.integration.xml" --jsonfile "build/${GROUP_NAME}.integration.out.json" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=$TEST_SUDO"    
 } finally {
     ess_down
     # Generate HTML report if XML output exists

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -2,20 +2,57 @@
 set -euo pipefail
 
 source .buildkite/scripts/common2.sh
+
 source .buildkite/scripts/steps/ess.sh
 
+# Make sure that all tools are installed
+asdf install
+
+GROUP_NAME=$1
+TEST_SUDO=$2
+if [ -z "$GROUP_NAME" ]; then
+  echo "Error: Specify the group name: integration_tests_tf.sh [group_name]" >&2
+  exit 1
+fi
+
+if [ -z "$TEST_SUDO" ]; then
+  echo "Error: Specify the test sudo: integration_tests_tf.sh [group_name] [test_sudo]" >&2
+  exit 1
+fi
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
 # There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
 # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
-OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
-OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
+OVERRIDE_STACK_VERSION="$(cat .package-version)"
+OVERRIDE_STACK_VERSION=${OVERRIDE_STACK_VERSION}"-SNAPSHOT"
 
-echo "~~~ Bulding test binaries"
+echo "~~~ Building test binaries"
 mage build:testBinaries
 
-ess_up $OVERRIDE_TEST_AGENT_VERSION || echo "Failed to start ESS stack" >&2
-trap 'ess_down' EXIT
+# If the step is retried, we start the stack again.
+# BUILDKITE_RETRY_COUNT == "0" for the first run
+# BUILDKITE_RETRY_COUNT > 0 for the retries
+if [[ "${BUILDKITE_RETRY_COUNT}" -gt 0 ]]; then
+  echo "~~~ The steps is retried, starting the ESS stack again"
+  trap 'ess_down' EXIT
+  ess_up $OVERRIDE_STACK_VERSION || echo "Failed to start ESS stack" >&2
+else 
+  # For the first run, we start the stack in the start_ess.sh step and it sets the meta-data
+  echo "~~~ Receiving ESS stack metadata"
+  export ELASTICSEARCH_HOST=$(buildkite-agent meta-data get "es.host")
+  export ELASTICSEARCH_USERNAME=$(buildkite-agent meta-data get "es.username")
+  export ELASTICSEARCH_PASSWORD=$(buildkite-agent meta-data get "es.pwd")
+  export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host")
+  export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username")
+  export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd")
+fi
 
+# Run integration tests
 echo "~~~ Running integration tests"
-AGENT_VERSION="8.16.0-SNAPSHOT" SNAPSHOT=true TEST_DEFINE_PREFIX=non_sudo_linux gotestsum --no-color -f standard-verbose --junitfile build/TEST-go-integration.xml --jsonfile build/TEST-go-integration.out.json -- -tags integration github.com/elastic/elastic-agent/testing/integration
+
+if [ "$TEST_SUDO" == "true" ]; then
+  sudo -E .buildkite/scripts/buildkite-integration-tests.sh $@
+else
+  .buildkite/scripts/buildkite-integration-tests.sh $@
+fi
+

test_infra/ess/deployment.tf

@@ -70,6 +70,13 @@ resource "ec_deployment" "integration-testing" {
   kibana = {
     size       = "1g"
     zone_count = 1
+    config = {
+      user_settings_json = jsonencode({
+        "xpack.fleet.enableExperimental"                          = ["agentTamperProtectionEnabled"]
+        "xpack.fleet.internal.registry.kibanaVersionCheckEnabled" = false
+        "server.restrictInternalApis"                             = false
+      })
+    }
   }
 
   integrations_server = {