Support multiple deployment backends

cli/config/kubernetes/overlays/deployment.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elastic-agent
+  labels:
+    app: elastic-agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elastic-agent
+  template:
+    metadata:
+      labels:
+        app: elastic-agent
+    spec:
+      containers:
+      - name: elastic-agent
+        image: centos/systemd:latest
+        command: ["/usr/sbin/init"]
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 0
+          capabilities:
+            add: ["SYS_ADMIN"]

cli/config/kubernetes/overlays/kustomization.yaml

@@ -0,0 +1,5 @@
+bases:
+  - ../../base
+
+resources:
+- deployment.yaml

e2e/_suites/fleet/fleet.go

@@ -16,6 +16,7 @@ import (
 	"github.com/cucumber/godog"
 	"github.com/elastic/e2e-testing/internal/common"
 	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/docker"
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	"github.com/elastic/e2e-testing/internal/installer"
@@ -46,21 +47,18 @@ type FleetTestSuite struct {
 	FleetServerPolicy   kibana.Policy
 	Version             string // current elastic-agent version
 	kibanaClient        *kibana.Client
-
+	deployer            deploy.Deployment
 	// date controls for queries
 	AgentStoppedDate             time.Time
 	RuntimeDependenciesStartDate time.Time
 }
 
 // afterScenario destroys the state created by a scenario
 func (fts *FleetTestSuite) afterScenario() {
-
 	serviceName := common.ElasticAgentServiceName
-	serviceManager := compose.NewServiceManager()
 
 	if !fts.StandAlone {
 		agentInstaller := fts.getInstaller()
-		serviceName = fts.getServiceName(agentInstaller)
 
 		if log.IsLevelEnabled(log.DebugLevel) {
 			err := agentInstaller.PrintLogsFn(fts.Hostname)
@@ -92,7 +90,7 @@ func (fts *FleetTestSuite) afterScenario() {
 
 	developerMode := shell.GetEnvBool("DEVELOPER_MODE")
 	if !developerMode {
-		_ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv)
+		_ = fts.deployer.Remove([]string{common.FleetProfileName, serviceName}, common.ProfileEnv)
 	} else {
 		log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped")
 	}
@@ -335,7 +333,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 	fts.CurrentTokenID = enrollmentKey.ID
 
 	var fleetConfig *kibana.FleetConfig
-	fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken)
+	fleetConfig, err = deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken)
 
 	if err != nil {
 		return err
@@ -987,7 +985,8 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error {
 
 	containerName := fts.getContainerName(agentInstaller, 2) // name of the new container
 
-	fleetConfig, err := deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken)
+	fleetConfig, err := deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken)
+
 	// the installation process for TAR includes the enrollment
 	if agentInstaller.InstallerType != "tar" {
 		if err != nil {
@@ -1125,7 +1124,7 @@ func (fts *FleetTestSuite) checkDataStream() error {
 	return err
 }
 
-func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containerName string, token string) (*kibana.FleetConfig, error) {
+func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer deploy.Deployment, containerName string, token string) (*kibana.FleetConfig, error) {
 	profile := agentInstaller.Profile // name of the runtime dependencies compose file
 	service := agentInstaller.Service // name of the service
 	serviceTag := agentInstaller.Tag  // docker tag of the service
@@ -1140,9 +1139,8 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe
 	common.ProfileEnv[envVarsPrefix+"AgentBinarySrcPath"] = agentInstaller.BinaryPath
 	common.ProfileEnv[envVarsPrefix+"AgentBinaryTargetPath"] = "/" + agentInstaller.Name
 
-	serviceManager := compose.NewServiceManager()
-
-	err := serviceManager.AddServicesToCompose(context.Background(), profile, []string{service}, common.ProfileEnv)
+	services := []string{profile, service}
+	err := deployer.Add(services, common.ProfileEnv)
 	if err != nil {
 		log.WithFields(log.Fields{
 			"service": service,

e2e/_suites/fleet/ingest_manager_test.go

@@ -5,18 +5,15 @@
 package main
 
 import (
-	"context"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/cucumber/godog"
 	"github.com/cucumber/messages-go/v10"
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/internal/common"
-	"github.com/elastic/e2e-testing/internal/compose"
+	"github.com/elastic/e2e-testing/internal/deploy"
 	"github.com/elastic/e2e-testing/internal/docker"
-	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/kibana"
 	"github.com/elastic/e2e-testing/internal/shell"
@@ -34,6 +31,8 @@ func setUpSuite() {
 		log.Error(err)
 		os.Exit(1)
 	}
+
+	common.Provider = shell.GetEnv("PROVIDER", common.Provider)
 	developerMode := shell.GetEnvBool("DEVELOPER_MODE")
 	if developerMode {
 		log.Info("Running in Developer mode 💻: runtime dependencies between different test runs will be reused to speed up dev cycle")
@@ -87,6 +86,7 @@ func setUpSuite() {
 	imts = IngestManagerTestSuite{
 		Fleet: &FleetTestSuite{
 			kibanaClient: kibanaClient,
+			deployer:     deploy.New(common.Provider),
 			Installers:   map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map
 		},
 	}
@@ -109,80 +109,40 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) {
 }
 
 func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) {
-	serviceManager := compose.NewServiceManager()
+	developerMode := shell.GetEnvBool("DEVELOPER_MODE")
 
 	ctx.BeforeSuite(func() {
 		setUpSuite()
 
-		log.Trace("Installing Fleet runtime dependencies")
-
-		common.ProfileEnv = map[string]string{
-			"kibanaVersion": common.KibanaVersion,
-			"stackVersion":  common.StackVersion,
-		}
+		log.Trace("Bootstrapping Fleet Server")
 
 		if !shell.GetEnvBool("SKIP_PULL") {
 			log.Info("Pulling Docker images...")
 			docker.PullImages(common.StackVersion, common.AgentVersion, common.KibanaVersion)
 		}
 
-		common.ProfileEnv["kibanaDockerNamespace"] = "kibana"
-		if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) {
-			// because it comes from a PR
-			common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci"
-		}
-
-		profile := common.FleetProfileName
-		err := serviceManager.RunCompose(context.Background(), true, []string{profile}, common.ProfileEnv)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"profile": profile,
-				"error":   err.Error(),
-			}).Fatal("Could not run the runtime dependencies for the profile.")
-		}
-
-		minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute
-		healthy, err := elasticsearch.WaitForElasticsearch(context.Background(), minutesToBeHealthy)
-		if !healthy {
-			log.WithFields(log.Fields{
-				"error":   err,
-				"minutes": minutesToBeHealthy,
-			}).Fatal("The Elasticsearch cluster could not get the healthy status")
-		}
-
-		kibanaClient, err := kibana.NewClient()
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error": err,
-			}).Fatal("Unable to create kibana client")
-		}
-
-		healthyKibana, err := kibanaClient.WaitForReady(minutesToBeHealthy)
-		if !healthyKibana {
-			log.WithFields(log.Fields{
-				"error":   err,
-				"minutes": minutesToBeHealthy,
-			}).Fatal("The Kibana instance could not get the healthy status")
-		}
-
-		imts.Fleet.setup()
+		deployer := deploy.New(common.Provider)
+		deployer.Bootstrap(func() error {
+			kibanaClient, err := kibana.NewClient()
+			if err != nil {
+				log.WithField("error", err).Fatal("Unable to create kibana client")
+			}
+			err = kibanaClient.WaitForFleet()
+			if err != nil {
+				log.WithField("error", err).Fatal("Fleet could not be initialized")
+			}
+			return nil
+		})
 
+		imts.Fleet.Version = common.AgentVersionBase
 		imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC()
 	})
 
 	ctx.AfterSuite(func() {
-		developerMode := shell.GetEnvBool("DEVELOPER_MODE")
 		if !developerMode {
 			log.Debug("Destroying Fleet runtime dependencies")
-			profile := common.FleetProfileName
-
-			err := serviceManager.StopCompose(context.Background(), true, []string{profile})
-			if err != nil {
-				log.WithFields(log.Fields{
-					"error":   err,
-					"profile": profile,
-				}).Warn("Could not destroy the runtime dependencies for the profile.")
-			}
+			deployer := deploy.New(common.Provider)
+			deployer.Destroy()
 		}
 
 		installers := imts.Fleet.Installers

e2e/_suites/fleet/stand-alone.go

@@ -7,17 +7,17 @@ package main
 import (
 	"context"
 	"fmt"
+	"path"
+	"strings"
+	"time"
+
 	"github.com/cenkalti/backoff/v4"
 	"github.com/elastic/e2e-testing/cli/config"
 	"github.com/elastic/e2e-testing/internal/common"
-	"github.com/elastic/e2e-testing/internal/compose"
 	"github.com/elastic/e2e-testing/internal/docker"
 	"github.com/elastic/e2e-testing/internal/installer"
 	"github.com/elastic/e2e-testing/internal/shell"
 	"github.com/elastic/e2e-testing/internal/utils"
-	"path"
-	"strings"
-	"time"
 
 	"github.com/elastic/e2e-testing/internal/elasticsearch"
 	log "github.com/sirupsen/logrus"
@@ -62,9 +62,7 @@ func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error {
 }
 
 func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error {
-	serviceManager := compose.NewServiceManager()
-
-	err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv)
+	err := fts.deployer.Remove([]string{common.FleetProfileName, serviceName}, common.ProfileEnv)
 	if err != nil {
 		return err
 	}
@@ -111,8 +109,6 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st
 		dockerImageTag += "-amd64"
 	}
 
-	serviceManager := compose.NewServiceManager()
-
 	common.ProfileEnv["elasticAgentDockerImageSuffix"] = ""
 	if image != "default" {
 		common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image
@@ -130,8 +126,8 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st
 		common.ProfileEnv[k] = v
 	}
 
-	err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName,
-		[]string{common.ElasticAgentServiceName}, common.ProfileEnv, composeFilename)
+	services := []string{common.FleetProfileName, common.ElasticAgentServiceName}
+	err := fts.deployer.Add(services, common.ProfileEnv)
 	if err != nil {
 		log.Error("Could not deploy the elastic-agent")
 		return err

internal/common/defaults.go

@@ -44,3 +44,6 @@ var KibanaVersion = AgentVersionBase
 // ProfileEnv is the environment to be applied to any execution
 // affecting the runtime dependencies (or profile)
 var ProfileEnv map[string]string
+
+// Provider is the deployment provider used, currently docker is supported
+var Provider = "docker"

internal/deploy/base.go

@@ -0,0 +1,39 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package deploy
+
+import (
+	"strings"
+)
+
+// Deployment interface for operations dealing with deployments of the bits
+// required for testing
+type Deployment interface {
+	Add(services []string, env map[string]string) error    // adds a service to deployment
+	Bootstrap(waitCB func() error) error                   // will bootstrap or reuse existing cluster if kubernetes is selected
+	Destroy() error                                        // Teardown deployment
+	ExecIn(service string, cmd []string) (string, error)   // Execute arbitrary commands in service
+	Inspect(service string) (*ServiceManifest, error)      // inspects service
+	Remove(services []string, env map[string]string) error // Removes services from deployment
+}
+
+// ServiceManifest information about a service in a deployment
+type ServiceManifest struct {
+	ID         string
+	Name       string
+	Connection string // a string representing how to connect to service
+	Hostname   string
+}
+
+// New creates a new deployment
+func New(provider string) Deployment {
+	if strings.EqualFold(provider, "docker") {
+		return newDockerDeploy()
+	}
+	if strings.EqualFold(provider, "kubernetes") {
+		return newK8sDeploy()
+	}
+	return nil
+}