feat: add support for running the tests in ARM

.ci/.e2e-tests.yaml

@@ -12,6 +12,7 @@ SUITES:
         tags: "metricbeat"
   - suite: "fleet"
     platforms:
+      - "arm"
       - "ubuntu-18.04"
     scenarios:
       - name: "Fleet"
@@ -22,6 +23,7 @@ SUITES:
         tags: "stand_alone_agent"
   - suite: "metricbeat"
     platforms:
+      - "arm"
       - "ubuntu-18.04"
     scenarios:
       - name: "ActiveMQ"

.ci/Jenkinsfile

@@ -322,8 +322,15 @@ def generateFunctionalTestStep(Map args = [:]){
   }
   tags += excludeNightlyTag
 
+  workerLabels = "${platform}"
+  if (platform != "arm") {
+    // ARM is not ready as immutable workers
+    workerLabels += " && immutable"
+  }
+  workerLabels += " && docker"
+
   return {
-    node("${platform} && immutable && docker") {
+    node("${workerLabels}") {
       try {
         deleteDir()
         unstash 'source'

cli/config/compose/profiles/fleet/docker-compose.yml

@@ -17,6 +17,7 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.0.0-SNAPSHOT}"
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
   kibana:

cli/config/compose/profiles/metricbeat/docker-compose.yml

@@ -10,5 +10,6 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.0.0-SNAPSHOT}"
+    platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"

cli/config/compose/services/elasticsearch/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
     image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag}"
+    platform: ${elasticsearchPlatform:-linux/amd64}
     ports:
       - "9200:9200"
       - "9300:9300"

e2e/_suites/fleet/ingest-manager_test.go

@@ -149,6 +149,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
 
 		workDir, _ := os.Getwd()
 		profileEnv = map[string]string{
+			"stackPlatform":    "linux/" + e2e.GetArchitecture(),
 			"stackVersion":     stackVersion,
 			"kibanaConfigPath": path.Join(workDir, "configurations", "kibana.config.yml"),
 		}

e2e/_suites/fleet/installers.go

@@ -154,11 +154,13 @@ func (i *DockerPackage) Preinstall() error {
 	// wait for tagging to ensure the loaded image is present
 	e2e.Sleep(3 * time.Second)
 
+	arch := e2e.GetArchitecture()
+
 	// we need to tag the loaded image because its tag relates to the target branch,
 	// and we want it to use the 'pr-12345' format.
 	return docker.TagImage(
 		"docker.elastic.co/beats/"+i.artifact+":"+agentVersionBase,
-		"docker.elastic.co/observability-ci/"+i.artifact+":"+i.originalVersion+"-amd64",
+		"docker.elastic.co/observability-ci/"+i.artifact+":"+i.originalVersion+"-"+arch,
 	)
 }
 

e2e/_suites/fleet/services.go

@@ -287,7 +287,7 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI
 	// extract the agent in the box, as it's mounted as a volume
 	artifact := "elastic-agent"
 	os := "linux"
-	arch := "amd64"
+	arch := e2e.GetArchitecture()
 	extension := "deb"
 
 	binaryName := e2e.BuildArtifactName(artifact, version, agentVersionBase, os, arch, extension, false)
@@ -359,7 +359,7 @@ func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error
 	}
 
 	os := "linux"
-	arch := "amd64"
+	arch := e2e.GetArchitecture()
 	extension := "tar.gz"
 
 	binaryName := e2e.BuildArtifactName(artifactName, version, agentVersionBase, os, arch, extension, true)

e2e/_suites/fleet/services_test.go

@@ -39,7 +39,7 @@ func TestDownloadAgentBinary(t *testing.T) {
 		assert.Equal(t, downloadedFilePath, expectedFilePath)
 	})
 
-	t.Run("Fetching DEB binary from local Beats dir", func(t *testing.T) {
+	t.Run("Fetching DEB binary (amd64) from local Beats dir", func(t *testing.T) {
 		defer os.Unsetenv("BEATS_LOCAL_PATH")
 		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
 
@@ -50,8 +50,19 @@ func TestDownloadAgentBinary(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, downloadedFilePath, expectedFilePath)
 	})
+	t.Run("Fetching DEB binary (arm64) from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		artifactName := "elastic-agent-8.0.0-SNAPSHOT-arm64.deb"
+		expectedFilePath := path.Join(distributionsDir, artifactName)
+
+		downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version)
+		assert.Nil(t, err)
+		assert.Equal(t, downloadedFilePath, expectedFilePath)
+	})
 
-	t.Run("Fetching TAR binary from local Beats dir", func(t *testing.T) {
+	t.Run("Fetching TAR binary (amd64) from local Beats dir", func(t *testing.T) {
 		defer os.Unsetenv("BEATS_LOCAL_PATH")
 		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
 
@@ -62,8 +73,19 @@ func TestDownloadAgentBinary(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, downloadedFilePath, expectedFilePath)
 	})
+	t.Run("Fetching TAR binary (arm64) from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		artifactName := "elastic-agent-8.0.0-SNAPSHOT-linux-arm64.tar.gz"
+		expectedFilePath := path.Join(distributionsDir, artifactName)
+
+		downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version)
+		assert.Nil(t, err)
+		assert.Equal(t, downloadedFilePath, expectedFilePath)
+	})
 
-	t.Run("Fetching Docker binary from local Beats dir", func(t *testing.T) {
+	t.Run("Fetching Docker binary (amd64) from local Beats dir", func(t *testing.T) {
 		defer os.Unsetenv("BEATS_LOCAL_PATH")
 		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
 
@@ -74,14 +96,36 @@ func TestDownloadAgentBinary(t *testing.T) {
 		assert.Nil(t, err)
 		assert.Equal(t, downloadedFilePath, expectedFilePath)
 	})
+	t.Run("Fetching Docker binary (arm64) from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		artifactName := "elastic-agent-8.0.0-SNAPSHOT-linux-arm64.docker.tar.gz"
+		expectedFilePath := path.Join(distributionsDir, artifactName)
+
+		downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version)
+		assert.Nil(t, err)
+		assert.Equal(t, downloadedFilePath, expectedFilePath)
+	})
 
-	t.Run("Fetching ubi8 Docker binary from local Beats dir", func(t *testing.T) {
+	t.Run("Fetching ubi8 Docker binary (amd64) from local Beats dir", func(t *testing.T) {
 		defer os.Unsetenv("BEATS_LOCAL_PATH")
 		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
 
 		artifactName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-amd64.docker.tar.gz"
 		expectedFilePath := path.Join(distributionsDir, artifactName)
 
+		downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version)
+		assert.Nil(t, err)
+		assert.Equal(t, downloadedFilePath, expectedFilePath)
+	})
+	t.Run("Fetching ubi8 Docker binary (arm64) from local Beats dir", func(t *testing.T) {
+		defer os.Unsetenv("BEATS_LOCAL_PATH")
+		os.Setenv("BEATS_LOCAL_PATH", beatsDir)
+
+		artifactName := "elastic-agent-ubi8-8.0.0-SNAPSHOT-linux-arm64.docker.tar.gz"
+		expectedFilePath := path.Join(distributionsDir, artifactName)
+
 		downloadedFilePath, err := downloadAgentBinary(artifactName, artifact, version)
 		assert.Nil(t, err)
 		assert.Equal(t, downloadedFilePath, expectedFilePath)

e2e/_suites/fleet/stand-alone.go

@@ -75,7 +75,8 @@ func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error
 
 		dockerInstaller.PreInstallFn()
 
-		dockerImageTag += "-amd64"
+		arch := e2e.GetArchitecture()
+		dockerImageTag += "-" + arch
 	}
 
 	serviceManager := services.NewServiceManager()
@@ -99,7 +100,7 @@ func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error
 
 	profileEnv["elasticAgentContainerName"] = containerName
 	profileEnv["elasticAgentConfigFile"] = sats.AgentConfigFilePath
-	profileEnv["elasticAgentPlatform"] = "linux/amd64"
+	profileEnv["elasticAgentPlatform"] = "linux/" + e2e.GetArchitecture()
 	profileEnv["elasticAgentTag"] = dockerImageTag
 
 	err = serviceManager.AddServicesToCompose(context.Background(), FleetProfileName, []string{ElasticAgentServiceName}, profileEnv)

e2e/_suites/metricbeat/metricbeat_test.go

@@ -239,7 +239,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) {
 		serviceManager := services.NewServiceManager()
 
 		env := map[string]string{
-			"stackVersion": stackVersion,
+			"stackPlatform": "linux/" + e2e.GetArchitecture(),
+			"stackVersion":  stackVersion,
 		}
 
 		err := serviceManager.RunCompose(true, []string{"metricbeat"}, env)
@@ -394,7 +395,8 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
 	beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "")
 	if useCISnapshots || beatsLocalPath != "" {
-		artifactName := e2e.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", "amd64", "tar.gz", true)
+		arch := e2e.GetArchitecture()
+		artifactName := e2e.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", arch, "tar.gz", true)
 
 		imagePath, err := e2e.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, metricbeatVersionBase, timeoutFactor, true)
 		if err != nil {
@@ -408,7 +410,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 
 		err = docker.TagImage(
 			"docker.elastic.co/beats/metricbeat:"+metricbeatVersionBase,
-			"docker.elastic.co/observability-ci/metricbeat:"+mts.Version+"-amd64",
+			"docker.elastic.co/observability-ci/metricbeat:"+mts.Version+"-"+arch,
 		)
 	}
 
@@ -429,19 +431,21 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error {
 		logLevel = log.DebugLevel.String()
 	}
 
+	arch := e2e.GetArchitecture()
+
 	env := map[string]string{
 		"BEAT_STRICT_PERMS":     "false",
 		"indexName":             mts.getIndexName(),
 		"logLevel":              logLevel,
 		"metricbeatConfigFile":  mts.configurationFile,
 		"metricbeatTag":         mts.Version,
 		"stackVersion":          stackVersion,
-		mts.ServiceName + "Tag": mts.ServiceVersion + "-amd64",
+		mts.ServiceName + "Tag": mts.ServiceVersion + "-" + arch,
 		"serviceName":           mts.ServiceName,
 	}
 
 	env["metricbeatDockerNamespace"] = e2e.GetDockerNamespaceEnvVar()
-	env["metricbeatPlatform"] = "linux/amd64"
+	env["metricbeatPlatform"] = "linux/" + arch
 
 	err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{"metricbeat"}, env)
 	if err != nil {

e2e/utils.go

@@ -59,8 +59,8 @@ func BuildArtifactName(artifact string, version string, fallbackVersion string,
 
 	useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS")
 	// we detected that the docker name on CI is using a different structure
-	// CI snapshots on GCP: elastic-agent-$VERSION-linux-amd64.docker.tar.gz
-	// Elastic's snapshots: elastic-agent-$VERSION-docker-image-linux-amd64.tar.gz
+	// CI snapshots on GCP: elastic-agent-$VERSION-linux-$ARCH.docker.tar.gz
+	// Elastic's snapshots: elastic-agent-$VERSION-docker-image-linux-$ARCH.tar.gz
 	if !useCISnapshots && isDocker {
 		dockerString = "docker-image"
 		artifactName = fmt.Sprintf("%s-%s-%s-%s-%s.%s", artifact, artifactVersion, dockerString, OS, arch, lowerCaseExtension)
@@ -148,6 +148,17 @@ func FetchBeatsBinary(artifactName string, artifact string, version string, fall
 	return handleDownload(downloadURL)
 }
 
+// GetArchitecture retrieves if the underlying system platform is arm64 or amd64
+func GetArchitecture() string {
+	arch := "amd64"
+	if os.Getenv("GOARCH") == "arm64" {
+		arch = "arm64"
+	}
+
+	log.Debugf("Golang's architecture is %s", arch)
+	return arch
+}
+
 // getGCPBucketCoordinates it calculates the bucket path in GCP
 func getGCPBucketCoordinates(fileName string, artifact string, version string, fallbackVersion string) (string, string, string) {
 	bucket := "beats-ci-artifacts"
@@ -163,7 +174,7 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, f
 
 	// we are setting a version from a pull request: the version of the artifact will be kept as the base one
 	// i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-x86_64.rpm
-	// i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-amd64.deb
+	// i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-$ARCH.deb
 	// i.e. /pull-requests/pr-21100/$THE_BEAT/$THE_BEAT-$VERSION-linux-x86_64.tar.gz
 	if strings.HasPrefix(strings.ToLower(version), "pr-") {
 		log.WithFields(log.Fields{
@@ -276,9 +287,9 @@ func GetElasticArtifactVersion(version string) string {
 
 // GetElasticArtifactURL returns the URL of a released artifact, which its full name is defined in the first argument,
 // from Elastic's artifact repository, building the JSON path query based on the full name
-// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-amd64.deb", "elastic-agent", "$VERSION")
+// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-$ARCH.deb", "elastic-agent", "$VERSION")
 // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION")
-// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-amd64.tar.gz", "elastic-agent","$VERSION")
+// i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-$ARCH.tar.gz", "elastic-agent","$VERSION")
 func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) {
 	exp := GetExponentialBackOff(time.Minute)