[7.x](backport #707) feat: add support for running the tests in ARM

.ci/.e2e-tests.yaml

@@ -12,12 +12,15 @@ SUITES:
         tags: "metricbeat"
   - suite: "fleet"
     platforms:
+      - "arm64"
       - "ubuntu-18.04"
     scenarios:
       - name: "Fleet"
         pullRequestFilter: " && ~debian"
         tags: "fleet_mode_agent"
       - name: "Endpoint Integration"
+        platforms:
+          - "ubuntu-18.04"
         pullRequestFilter: " && ~debian"
         tags: "agent_endpoint_integration"
       - name: "Linux Integration"

.ci/Jenkinsfile

@@ -249,16 +249,23 @@ def checkTestSuite(Map parallelTasks = [:], Map item = [:]) {
   def suite = item.suite
   def platforms = item.platforms
   item.scenarios.each { scenario ->
+    def name = scenario.name
+    def platformsValue = platforms
+    def scenarioPlatforms = scenario.platforms
+    if (scenarioPlatforms?.size() > 0) {
+      // scenario platforms take precedence over suite platforms, overriding them
+      platformsValue = scenarioPlatforms
+    }
     def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
     def tags = scenario.tags
     def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
     if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) {
-      platforms.each { platform ->
+      platformsValue.each { platform ->
         log(level: 'INFO', text: "Adding ${platform}:${suite}:${tags} test suite to the build execution")
-        parallelTasks["${platform}_${suite}_${tags}"] = generateFunctionalTestStep(platform: "${platform}", suite: "${suite}", tags: "${tags}", pullRequestFilter: "${pullRequestFilter}")
+        parallelTasks["${platform}_${suite}_${tags}"] = generateFunctionalTestStep(name: "${name}", platform: "${platform}", suite: "${suite}", tags: "${tags}", pullRequestFilter: "${pullRequestFilter}")
       }
     } else {
-      log(level: 'WARN', text: "The ${suite}:${tags} test suite won't be executed because there are no modified files")
+      log(level: 'WARN', text: "The ${platform}:${suite}:${tags} test suite won't be executed because there are no modified files")
     }
   }
 }
@@ -281,6 +288,7 @@ def doNotifyBuildResult(boolean slackNotify) {
 }
 
 def generateFunctionalTestStep(Map args = [:]){
+  def name = args.get('name')
   def platform = args.get('platform')
   def suite = args.get('suite')
   def tags = args.get('tags')
@@ -298,7 +306,17 @@ def generateFunctionalTestStep(Map args = [:]){
   if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
     tags += pullRequestFilter
   }
-  def workerLabels = "${platform} && immutable && docker"
+
+  def goArch = "amd64"
+  def workerLabels = "${platform} && immutable"
+  if (platform == "arm64") {
+    goArch = "arm64"
+    // ARM is not ready as immutable workers
+    workerLabels = "arm"
+  }
+
+  // Skip scenarios per platform
+  tags += " && ~@skip:${goArch}"
 
   return {
     withNode(labels: "${workerLabels}", sleepMax: 20, forceWorkspace: true){
@@ -309,10 +327,12 @@ def generateFunctionalTestStep(Map args = [:]){
           if(isInstalled(tool: 'docker', flag: '--version')) {
             dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
           }
-          filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
+          filebeat(image: 'docker.elastic.co/beats/filebeat:7.13.0-SNAPSHOT', output: "docker_logs_${goArch}_${suite}_${name}.log", workdir: "${env.WORKSPACE}"){
             dir("${BASE_DIR}"){
-              withSecretVault(secret: "${VAULT_INSTRUMENTATION_SECRET}", user_key: "apmServerToken", user_var_name: "APM_SECRET_TOKEN", pass_key: "apmServerUrl", pass_var_name: "APM_SERVER_URL"){
-                sh script: """.ci/scripts/functional-test.sh "${suite}" "${tags}" "${STACK_VERSION}" "${BEAT_VERSION}" """, label: "Run functional tests for ${suite}:${tags}"
+              withEnv(["GOARCH=${goArch}"]) {
+                withSecretVault(secret: "${VAULT_INSTRUMENTATION_SECRET}", user_key: "apmServerToken", user_var_name: "APM_SECRET_TOKEN", pass_key: "apmServerUrl", pass_var_name: "APM_SERVER_URL"){
+                  sh script: """.ci/scripts/functional-test.sh "${suite}" "${tags}" "${STACK_VERSION}" "${BEAT_VERSION}" """, label: "Run functional tests for ${suite}:${tags}"
+                }
               }
             }
           }

.ci/scripts/clean-docker.sh

@@ -12,6 +12,9 @@ set -euxo pipefail
 readonly VERSION="$(cat $(pwd)/.stack-version)"
 
 main() {
+  # remove running containers
+  docker container rm -fv $(docker container ls -a --quiet) || true
+
   # refresh docker images
   cat <<EOF >.tmp_images
 docker.elastic.co/beats/elastic-agent:${VERSION}

.ci/scripts/functional-test.sh

@@ -22,13 +22,14 @@ SUITE=${1:-''}
 TAGS=${2:-''}
 STACK_VERSION=${3:-"${BASE_VERSION}"}
 BEAT_VERSION=${4:-"${BASE_VERSION}"}
+GOARCH=${GOARCH:-"amd64"}
 
 ## Install the required dependencies for the given SUITE
 .ci/scripts/install-test-dependencies.sh "${SUITE}"
 
 rm -rf outputs || true
 mkdir -p outputs
 
-REPORT="$(pwd)/outputs/TEST-${SUITE}"
+REPORT="$(pwd)/outputs/TEST-${GOARCH}-${SUITE}"
 
-TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test
+TAGS="${TAGS}" FORMAT=junit:${REPORT}.xml GOARCH=${GOARCH} STACK_VERSION=${STACK_VERSION} BEAT_VERSION=${BEAT_VERSION} make --no-print-directory -C e2e/_suites/${SUITE} functional-test

.ci/scripts/install-helm-test-dependencies.sh

@@ -14,32 +14,33 @@ set -euxo pipefail
 #   - KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
 #
 
+GOARCH=${GOARCH:-"amd64"}
 MSG="parameter missing."
 HOME=${HOME:?$MSG}
 
 HELM_VERSION="${HELM_VERSION:-"3.5.2"}"
-HELM_TAR_GZ_FILE="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
+HELM_TAR_GZ_FILE="helm-v${HELM_VERSION}-linux-${GOARCH}.tar.gz"
 KIND_VERSION="v${KIND_VERSION:-"0.10.0"}"
 KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.18.2"}"
 
 HELM_CMD="${HOME}/bin/helm"
 KBC_CMD="${HOME}/bin/kubectl"
 
 # Install kind as a Go binary
-GO111MODULE="on" go get sigs.k8s.io/kind@${KIND_VERSION}
+GOARCH=${GOARCH} GO111MODULE="on" go get sigs.k8s.io/kind@${KIND_VERSION}
 
 mkdir -p "${HOME}/bin" "${HOME}/.kube"
 touch "${HOME}/.kube/config"
 
 # Install kubectl
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/${GOARCH}/kubectl"
 chmod +x "${KBC_CMD}"
 ${KBC_CMD} version --client
 
 # Install Helm
 curl -o ${HELM_TAR_GZ_FILE} "https://get.helm.sh/${HELM_TAR_GZ_FILE}"
 tar -xvf ${HELM_TAR_GZ_FILE}
-mv linux-amd64/helm ${HELM_CMD}
+mv linux-${GOARCH}/helm ${HELM_CMD}
 chmod +x "${HELM_CMD}"
 ${HELM_CMD} version --client
-rm -fr linux-amd64 ${HELM_TAR_GZ_FILE}
+rm -fr linux-${GOARCH} ${HELM_TAR_GZ_FILE}

.stack-version

@@ -1 +1 @@
-8.0.0-SNAPSHOT
+7.x-SNAPSHOT

cli/config/compose/profiles/fleet/configurations/kibana.config.yml

@@ -12,7 +12,7 @@ monitoring.ui.container.elasticsearch.enabled: true
 xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012"
 
 xpack.fleet.enabled: true
-xpack.fleet.registryUrl: http://package-registry:8080
+xpack.fleet.registryUrl: https://epr-staging.elastic.co
 xpack.fleet.agents.enabled: true
 xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200
 xpack.fleet.agents.fleet_server.hosts:

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -17,30 +17,23 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-7.x-SNAPSHOT}"
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
   kibana:
     depends_on:
       elasticsearch:
         condition: service_healthy
-      package-registry:
-        condition: service_healthy
     healthcheck:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
     image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-7.x-SNAPSHOT}"
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "5601:5601"
     volumes:
       - ./configurations/kibana.config.yml:/usr/share/kibana/config/kibana.yml
-  package-registry:
-    image: docker.elastic.co/package-registry/distribution:staging
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080"]
-      retries: 300
-      interval: 1s
-
   fleet-server:
     image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-7.x-SNAPSHOT}"
     depends_on:

cli/config/compose/profiles/metricbeat/docker-compose.yml

@@ -10,5 +10,6 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-7.x-SNAPSHOT}"
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"

cli/config/compose/services/elastic-agent/centos/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   elastic-agent:
-    image: centos/systemd:${centos_systemdTag:-latest} 
+    image: docker.elastic.co/observability-ci/centos-systemd:latest
     container_name: ${centos_systemdContainerName}
     entrypoint: "/usr/sbin/init"
     platform: ${stackPlatform:-linux/amd64}

cli/config/compose/services/elastic-agent/cloud/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       - "DATA_PATH=/apm-legacy/data/"
       - "LOGS_PATH=/apm-legacy/logs/"
       - "HOME_PATH=/apm-legacy/"
+    platform: ${stackPlatform:-linux/amd64}
     volumes:
       - "${apmVolume}:/apm-legacy"
     ports:

cli/config/compose/services/elastic-agent/debian/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   elastic-agent:
-    image: alehaa/debian-systemd:${debian_systemdTag:-stretch}
+    image: docker.elastic.co/observability-ci/debian-systemd:latest
     container_name: ${debian_systemdContainerName} 
     entrypoint: "/sbin/init"
     platform: ${stackPlatform:-linux/amd64}

cli/config/compose/services/elastic-agent/docker-compose.yml

@@ -11,6 +11,6 @@ services:
     environment:
       - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}"
       - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}"
-    platform: ${elasticAgentPlatform:-linux/amd64}
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "127.0.0.1:8220:8220"

cli/config/compose/services/elasticsearch/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag}"
+    platform: ${elasticsearchPlatform:-linux/amd64}
     ports:
       - "9200:9200"
       - "9300:9300"

cli/config/compose/services/fleet-server-centos/docker-compose.yml

@@ -1,9 +1,10 @@
 version: '2.4'
 services:
   fleet-server-centos:
-    image: centos/systemd:${fleet_server_centosTag:-latest} 
+    image: docker.elastic.co/observability-ci/centos-systemd:latest
     container_name: ${fleet_server_centosContainerName}
     entrypoint: "/usr/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro 

cli/config/compose/services/fleet-server-debian/docker-compose.yml

@@ -1,9 +1,10 @@
 version: '2.4'
 services:
   fleet-server-debian:
-    image: alehaa/debian-systemd:${fleet_server_debianTag:-stretch}
+    image: docker.elastic.co/observability-ci/debian-systemd:latest
     container_name: ${fleet_server_debianContainerName} 
     entrypoint: "/sbin/init"
+    platform: ${stackPlatform:-linux/amd64}
     privileged: true
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:ro

cli/config/compose/services/metricbeat/docker-compose.yml

@@ -17,6 +17,6 @@ services:
     image: "docker.elastic.co/${metricbeatDockerNamespace:-beats}/metricbeat:${metricbeatTag:-7.x-SNAPSHOT}"
     labels:
       co.elastic.logs/module: "${serviceName}"
-    platform: ${metricbeatPlatform:-linux/amd64}
+    platform: ${stackPlatform:-linux/amd64}
     volumes:
       - "${metricbeatConfigFile}:/usr/share/metricbeat/metricbeat.yml"

e2e/_suites/fleet/features/agent_endpoint_integration.feature

@@ -1,8 +1,8 @@
 @agent_endpoint_integration
+@skip:arm64
 Feature: Agent Endpoint Integration
   Scenarios for Agent to deploy Endpoint and sending data to Fleet and Elasticsearch.
 
-@deploy-endpoint-with-agent
 Scenario Outline: Adding the Endpoint Integration to an Agent makes the host to show in Security App
   Given a "<os>" agent is deployed to Fleet with "tar" installer
     And the agent is listed in Fleet as "online"
@@ -20,7 +20,6 @@ Examples: Debian
 | os     |
 | debian |
 
-@endpoint-policy-check
 Scenario Outline: Deploying an Endpoint makes policies to appear in the Security App
   When an "Endpoint" is successfully deployed with a "<os>" Agent using "tar" installer
   Then the policy response will be shown in the Security App
@@ -35,7 +34,6 @@ Examples: Debian
 | os     |
 | debian |
 
-@set-policy-and-check-changes
 Scenario Outline: Changing an Agent policy is reflected in the Security App
   Given an "Endpoint" is successfully deployed with a "<os>" Agent using "tar" installer
   When the policy is updated to have "malware" in "detect" mode
@@ -51,7 +49,6 @@ Examples: Debian
 | os     |
 | debian |
 
-@deploy-endpoint-then-unenroll-agent
 Scenario Outline: Un-enrolling Elastic Agent stops Elastic Endpoint
   Given an "Endpoint" is successfully deployed with a "<os>" Agent using "tar" installer
   When the agent is un-enrolled
@@ -68,7 +65,6 @@ Examples: Debian
 | os     |
 | debian |
 
-@deploy-endpoint-then-remove-it-from-policy
 Scenario Outline: Removing Endpoint from Agent policy stops the connected Endpoint
   Given an "Endpoint" is successfully deployed with a "<os>" Agent using "tar" installer
   When the "Endpoint Security" integration is "removed" in the policy

e2e/_suites/fleet/ingest_manager_test.go

@@ -66,8 +66,6 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 	ctx.BeforeSuite(func() {
 		setUpSuite()
 
-		log.Trace("Bootstrapping Fleet Server")
-
 		if !shell.GetEnvBool("SKIP_PULL") {
 			images := []string{
 				"docker.elastic.co/beats/elastic-agent:" + common.BeatVersion,

e2e/_suites/fleet/stand-alone.go

@@ -112,7 +112,9 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en
 		agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName)
 		dockerInstaller, _ := installer.Attach(fts.deployer, agentService, "docker")
 		dockerInstaller.Preinstall()
-		dockerImageTag += "-amd64"
+
+		arch := utils.GetArchitecture()
+		dockerImageTag += "-" + arch
 	}
 
 	common.ProfileEnv["elasticAgentDockerImageSuffix"] = ""
@@ -125,7 +127,6 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, en
 	containerName := fmt.Sprintf("%s_%s_%d", common.FleetProfileName, common.ElasticAgentServiceName, 1)
 
 	common.ProfileEnv["elasticAgentContainerName"] = containerName
-	common.ProfileEnv["elasticAgentPlatform"] = "linux/amd64"
 	common.ProfileEnv["elasticAgentTag"] = dockerImageTag
 
 	for k, v := range env {

e2e/_suites/metricbeat/metricbeat_test.go

@@ -231,7 +231,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) {
 		serviceManager := deploy.NewServiceManager()
 
 		env := map[string]string{
-			"stackVersion": common.StackVersion,
+			"stackPlatform": "linux/" + utils.GetArchitecture(),
+			"stackVersion":  common.StackVersion,
 		}
 
 		err := serviceManager.RunCompose(
@@ -371,7 +372,8 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
 	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
 	if useCISnapshots || beatsLocalPath != "" {
-		artifactName := utils.BuildArtifactName("metricbeat", mts.Version, common.BeatVersionBase, "linux", "amd64", "tar.gz", true)
+		arch := utils.GetArchitecture()
+		artifactName := utils.BuildArtifactName("metricbeat", mts.Version, common.BeatVersionBase, "linux", arch, "tar.gz", true)
 
 		imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, common.BeatVersionBase, utils.TimeoutFactor, true)
 		if err != nil {
@@ -383,7 +385,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 			return err
 		}
 
-		mts.Version = mts.Version + "-amd64"
+		mts.Version = mts.Version + "-" + arch
 
 		err = deploy.TagImage(
 			"docker.elastic.co/beats/metricbeat:"+common.BeatVersionBase,
@@ -411,6 +413,8 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 		logLevel = log.DebugLevel.String()
 	}
 
+	arch := utils.GetArchitecture()
+
 	env := map[string]string{
 		"BEAT_STRICT_PERMS":     "false",
 		"indexName":             mts.getIndexName(),
@@ -423,7 +427,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 	}
 
 	env["metricbeatDockerNamespace"] = utils.GetDockerNamespaceEnvVar("beats")
-	env["metricbeatPlatform"] = "linux/amd64"
+	env["metricbeatPlatform"] = "linux/" + arch
 
 	err := serviceManager.AddServicesToCompose(
 		testSuite.currentContext,