diff --git a/.ci/scripts/clean-docker.sh b/.ci/scripts/clean-docker.sh index 017132144d..bc3fc0f29c 100755 --- a/.ci/scripts/clean-docker.sh +++ b/.ci/scripts/clean-docker.sh @@ -9,7 +9,7 @@ set -euxo pipefail # Build and test the app using the install and test make goals. # -readonly VERSION="7.x-SNAPSHOT" +readonly VERSION="$(cat $(pwd)/.stack-version)" main() { # refresh docker images diff --git a/.ci/scripts/fleet-test.sh b/.ci/scripts/fleet-test.sh index 85c57b459f..edaa021877 100755 --- a/.ci/scripts/fleet-test.sh +++ b/.ci/scripts/fleet-test.sh @@ -9,10 +9,10 @@ set -euxo pipefail # Run the functional tests for fleets using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '8.0.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.x-SNAPSHOT'} +STACK_VERSION=${1:-"$(cat $(pwd)/.stack-version)"} SUITE='fleet' # Exclude the nightly tests in the CI. diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index 07a469c6b6..a0e2edd791 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -12,14 +12,16 @@ set -euxo pipefail # Parameters: # - SUITE - that's the suite to be tested. Default '' which means all of them. # - TAGS - that's the tags to be tested. Default '' which means all of them. -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.x-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.x-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # +BASE_VERSION="$(cat $(pwd)/.stack-version)" + SUITE=${1:-''} TAGS=${2:-''} -STACK_VERSION=${3:-'7.x-SNAPSHOT'} -BEAT_VERSION=${4:-'7.x-SNAPSHOT'} +STACK_VERSION=${3:-"${BASE_VERSION}"} +BEAT_VERSION=${4:-"${BASE_VERSION}"} ## Install the required dependencies for the given SUITE .ci/scripts/install-test-dependencies.sh "${SUITE}" diff --git a/.ci/scripts/metricbeat-test.sh b/.ci/scripts/metricbeat-test.sh index 2a24618ad4..460d649e63 100755 --- a/.ci/scripts/metricbeat-test.sh +++ b/.ci/scripts/metricbeat-test.sh @@ -9,12 +9,14 @@ set -euxo pipefail # Run the functional tests for metricbeat using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '8.0.0-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '8.0.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.x-SNAPSHOT'} -BEAT_VERSION=${2:-'7.x-SNAPSHOT'} +BASE_VERSION="$(cat $(pwd)/.stack-version)" + +STACK_VERSION=${1:-"${BASE_VERSION}"} +BEAT_VERSION=${2:-"${BASE_VERSION}"} SUITE='metricbeat' .ci/scripts/functional-test.sh "${SUITE}" "" "${STACK_VERSION}" "${BEAT_VERSION}" diff --git a/.stack-version b/.stack-version new file mode 100644 index 0000000000..f1358e30d8 --- /dev/null +++ b/.stack-version @@ -0,0 +1 @@ +8.0.0-SNAPSHOT diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index aef82542f8..ec3d728013 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -8,7 +8,7 @@ import ( "context" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -63,12 +63,16 @@ func buildDeployServiceCommand(srv string) *cobra.Command { Short: `Deploys a ` + srv + ` service`, Long: `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{} env = config.PutServiceEnvironment(env, srv, versionToRun) - err := serviceManager.AddServicesToCompose(context.Background(), deployToProfile, []string{srv}, env) + err := serviceManager.AddServicesToCompose( + context.Background(), + deploy.NewServiceRequest(deployToProfile), + []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, + env) if err != nil { log.WithFields(log.Fields{ "profile": deployToProfile, @@ -85,12 +89,16 @@ func buildUndeployServiceCommand(srv string) *cobra.Command { Short: `Undeploys a ` + srv + ` service`, Long: `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{} env = config.PutServiceEnvironment(env, srv, versionToRun) - err := serviceManager.RemoveServicesFromCompose(context.Background(), deployToProfile, []string{srv}, env) + err := serviceManager.RemoveServicesFromCompose( + context.Background(), + deploy.NewServiceRequest(deployToProfile), + []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, + env) if err != nil { log.WithFields(log.Fields{ "profile": deployToProfile, diff --git a/cli/cmd/run.go b/cli/cmd/run.go index 49cde2aeb1..d6c7e437f5 100644 --- a/cli/cmd/run.go +++ b/cli/cmd/run.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command { Short: `Runs a ` + srv + ` service`, Long: `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun) @@ -76,7 +76,8 @@ func buildRunServiceCommand(srv string) *cobra.Command { env[k] = v } - err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env) + err := serviceManager.RunCompose( + context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, env) if err != nil { log.WithFields(log.Fields{ "service": srv, @@ -96,7 +97,7 @@ Example: go run main.go run profile fleet -s elastic-agent:8.0.0-SNAPSHOT `, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{ "profileVersion": versionToRun, @@ -110,14 +111,15 @@ Example: env[k] = v } - err := serviceManager.RunCompose(context.Background(), true, []string{key}, env) + err := serviceManager.RunCompose( + context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}, env) if err != nil { log.WithFields(log.Fields{ "profile": key, }).Error("Could not run the profile.") } - composeNames := []string{} + composeNames := []deploy.ServiceRequest{} if len(servicesToRun) > 0 { for _, srv := range servicesToRun { arr := strings.Split(srv, ":") @@ -137,10 +139,10 @@ Example: }).Trace("Adding service") env = config.PutServiceEnvironment(env, image, tag) - composeNames = append(composeNames, image) + composeNames = append(composeNames, deploy.NewServiceRequest(image)) } - err = serviceManager.AddServicesToCompose(context.Background(), key, composeNames, env) + err = serviceManager.AddServicesToCompose(context.Background(), deploy.NewServiceRequest(key), composeNames, env) if err != nil { log.WithFields(log.Fields{ "profile": key, diff --git a/cli/cmd/stop.go b/cli/cmd/stop.go index 984802dc92..974f4f63a0 100644 --- a/cli/cmd/stop.go +++ b/cli/cmd/stop.go @@ -8,7 +8,7 @@ import ( "context" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -55,9 +55,10 @@ func buildStopServiceCommand(srv string) *cobra.Command { Short: `Stops a ` + srv + ` service`, Long: `Stops a ` + srv + ` service, stoppping its Docker container`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() - err := serviceManager.StopCompose(context.Background(), false, []string{srv}) + err := serviceManager.StopCompose( + context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}) if err != nil { log.WithFields(log.Fields{ "service": srv, @@ -73,9 +74,10 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command Short: `Stops the ` + profile.Name + ` profile`, Long: `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() - err := serviceManager.StopCompose(context.Background(), true, []string{key}) + err := serviceManager.StopCompose( + context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}) if err != nil { log.WithFields(log.Fields{ "profile": key, diff --git a/cli/config/compose/profiles/fleet/docker-compose.yml b/cli/config/compose/profiles/fleet/docker-compose.yml index 448252ed25..9c31f4f58a 100644 --- a/cli/config/compose/profiles/fleet/docker-compose.yml +++ b/cli/config/compose/profiles/fleet/docker-compose.yml @@ -29,7 +29,7 @@ services: test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null" retries: 600 interval: 1s - image: "docker.elastic.co/${kibanaDockerNamespace:-beats}/kibana:${kibanaVersion:-7.x-SNAPSHOT}" + image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-7.x-SNAPSHOT}" ports: - "5601:5601" volumes: diff --git a/cli/config/compose/services/centos/docker-compose.yml b/cli/config/compose/services/centos/docker-compose.yml deleted file mode 100644 index fa1c2f2162..0000000000 --- a/cli/config/compose/services/centos/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2.4' -services: - centos: - image: centos:${centosTag:-7} - container_name: ${centosContainerName} - entrypoint: tail -f /dev/null diff --git a/cli/config/compose/services/debian/docker-compose.yml b/cli/config/compose/services/debian/docker-compose.yml deleted file mode 100644 index cb4491e840..0000000000 --- a/cli/config/compose/services/debian/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2.4' -services: - debian: - image: debian:${debianTag:-9} - container_name: ${debianContainerName} - entrypoint: tail -f /dev/null diff --git a/cli/config/compose/services/centos-systemd/docker-compose.yml b/cli/config/compose/services/elastic-agent/centos/docker-compose.yml similarity index 94% rename from cli/config/compose/services/centos-systemd/docker-compose.yml rename to cli/config/compose/services/elastic-agent/centos/docker-compose.yml index 31ca6eb33d..dbe229fc10 100644 --- a/cli/config/compose/services/centos-systemd/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/centos/docker-compose.yml @@ -1,6 +1,6 @@ version: '2.4' services: - centos-systemd: + elastic-agent: image: centos/systemd:${centos_systemdTag:-latest} container_name: ${centos_systemdContainerName} entrypoint: "/usr/sbin/init" diff --git a/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml b/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml similarity index 100% rename from cli/config/compose/services/elastic-agent/docker-compose-cloud.yml rename to cli/config/compose/services/elastic-agent/cloud/docker-compose.yml diff --git a/cli/config/compose/services/debian-systemd/docker-compose.yml b/cli/config/compose/services/elastic-agent/debian/docker-compose.yml similarity index 94% rename from cli/config/compose/services/debian-systemd/docker-compose.yml rename to cli/config/compose/services/elastic-agent/debian/docker-compose.yml index 1333569887..2c39275ce1 100644 --- a/cli/config/compose/services/debian-systemd/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/debian/docker-compose.yml @@ -1,6 +1,6 @@ version: '2.4' services: - debian-systemd: + elastic-agent: image: alehaa/debian-systemd:${debian_systemdTag:-stretch} container_name: ${debian_systemdContainerName} entrypoint: "/sbin/init" diff --git a/cli/config/config.go b/cli/config/config.go index a77d574251..569ee254d5 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -73,16 +73,14 @@ func FileExists(configFile string) (bool, error) { // GetComposeFile returns the path of the compose file, looking up the // tool's workdir -func GetComposeFile(isProfile bool, composeName string, composeFileName ...string) (string, error) { - if isProfile || composeFileName == nil || composeFileName[0] == "" { - composeFileName = []string{"docker-compose.yml"} - } +func GetComposeFile(isProfile bool, composeName string) (string, error) { + composeFileName := "docker-compose.yml" serviceType := "services" if isProfile { serviceType = "profiles" } - composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName[0]) + composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName) found, err := io.Exists(composeFilePath) if found && err == nil { log.WithFields(log.Fields{ diff --git a/cli/config/kubernetes/overlays/deployment.yaml b/cli/config/kubernetes/overlays/deployment.yaml new file mode 100644 index 0000000000..9ae3ed7a11 --- /dev/null +++ b/cli/config/kubernetes/overlays/deployment.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastic-agent + labels: + app: elastic-agent +spec: + replicas: 1 + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + containers: + - name: elastic-agent + image: centos/systemd:latest + command: ["/usr/sbin/init"] + securityContext: + allowPrivilegeEscalation: true + runAsUser: 0 + capabilities: + add: ["SYS_ADMIN"] diff --git a/cli/config/kubernetes/overlays/kustomization.yaml b/cli/config/kubernetes/overlays/kustomization.yaml new file mode 100644 index 0000000000..37fdeffa9b --- /dev/null +++ b/cli/config/kubernetes/overlays/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../base + +resources: +- deployment.yaml diff --git a/e2e/_suites/fleet/features/backend_processes.feature b/e2e/_suites/fleet/features/backend_processes.feature index e4a2b1f710..4b52594aa1 100644 --- a/e2e/_suites/fleet/features/backend_processes.feature +++ b/e2e/_suites/fleet/features/backend_processes.feature @@ -6,8 +6,8 @@ Feature: Backend Processes Scenario Outline: Deploying the agent Given a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is in the "started" state on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -21,10 +21,10 @@ Examples: Debian @enroll Scenario Outline: Deploying the agent with enroll and then run on rpm and deb - Given a "" agent is deployed to Fleet with "systemd" installer + Given a "" agent is deployed to Fleet When the "elastic-agent" process is in the "started" state on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -57,8 +57,8 @@ Examples: Debian Scenario Outline: Restarting the installed agent Given a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is "restarted" on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -75,8 +75,8 @@ Scenario Outline: Restarting the host with persistent agent restarts backen Given a "" agent is deployed to Fleet with "tar" installer When the host is restarted Then the "elastic-agent" process is in the "started" state on the host - And there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + And there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 65c5962460..e5ccf2b2a5 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -21,7 +21,7 @@ Examples: Debian @enroll Scenario Outline: Deploying the agent with enroll and then run on rpm and deb - Given a "" agent is deployed to Fleet with "systemd" installer + Given a "" agent is deployed to Fleet When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" And system package dashboards are listed in Fleet diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 1113119e9c..6156f13a69 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -7,15 +7,15 @@ package main import ( "context" "fmt" - "github.com/google/uuid" "strings" "time" + "github.com/google/uuid" + "github.com/cenkalti/backoff/v4" "github.com/cucumber/godog" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" @@ -28,6 +28,8 @@ import ( const actionADDED = "added" const actionREMOVED = "removed" +var deployedAgentsCount = 0 + // FleetTestSuite represents the scenarios for Fleet-mode type FleetTestSuite struct { // integrations @@ -41,12 +43,11 @@ type FleetTestSuite struct { Installers map[string]installer.ElasticAgentInstaller Integration kibana.IntegrationPackage // the installed integration Policy kibana.Policy - FleetServerPolicy kibana.Policy PolicyUpdatedAt string // the moment the policy was updated + FleetServerPolicy kibana.Policy Version string // current elastic-agent version kibanaClient *kibana.Client - // fleet server - FleetServerHostname string // hostname of the fleet server. If empty, it means the agent is the first one, bootstrapping fleet server + deployer deploy.Deployment // date controls for queries AgentStoppedDate time.Time RuntimeDependenciesStartDate time.Time @@ -54,13 +55,12 @@ type FleetTestSuite struct { // afterScenario destroys the state created by a scenario func (fts *FleetTestSuite) afterScenario() { + defer func() { deployedAgentsCount = 0 }() serviceName := common.ElasticAgentServiceName - serviceManager := compose.NewServiceManager() if !fts.StandAlone { agentInstaller := fts.getInstaller() - serviceName = fts.getServiceName(agentInstaller) if log.IsLevelEnabled(log.DebugLevel) { err := agentInstaller.PrintLogsFn(fts.Hostname) @@ -92,7 +92,18 @@ func (fts *FleetTestSuite) afterScenario() { developerMode := shell.GetEnvBool("DEVELOPER_MODE") if !developerMode { - _ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) + image := "" + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + image = agentInstaller.Image + } + + _ = fts.deployer.Remove( + []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(serviceName).WithFlavour(image), + }, + common.ProfileEnv) } else { log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") } @@ -105,14 +116,13 @@ func (fts *FleetTestSuite) afterScenario() { }).Warn("The enrollment token could not be deleted") } - fts.kibanaClient.DeleteAllPolicies(fts.FleetServerPolicy) + fts.kibanaClient.DeleteAllPolicies() // clean up fields fts.CurrentTokenID = "" fts.CurrentToken = "" fts.Image = "" fts.Hostname = "" - fts.FleetServerHostname = "" fts.StandAlone = false } @@ -121,7 +131,7 @@ func (fts *FleetTestSuite) beforeScenario() { fts.StandAlone = false fts.ElasticAgentStopped = false - fts.Version = common.AgentVersion + fts.Version = common.BeatVersion policy, err := fts.kibanaClient.GetDefaultPolicy(false) if err != nil { @@ -131,18 +141,10 @@ func (fts *FleetTestSuite) beforeScenario() { } fts.Policy = policy - - fleetServerPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) - if err != nil { - log.WithFields(log.Fields{ - "err": err, - }).Warn("The default fleet server policy could not be obtained") - } - fts.FleetServerPolicy = fleetServerPolicy - } func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { + s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet) s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller) s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion) @@ -195,8 +197,8 @@ func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredSt return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForAgents, exp) if err != nil { @@ -230,7 +232,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion default: version = common.AgentStaleVersion } @@ -239,7 +241,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver // prepare installer for stale version if fts.Version != agentVersionBackup { - i := installer.GetElasticAgentInstaller(image, installerType, fts.Version, "") + i := installer.GetElasticAgentInstaller(image, installerType, fts.Version, deployedAgentsCount) fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i } @@ -252,7 +254,7 @@ func (fts *FleetTestSuite) installCerts() error { log.WithFields(log.Fields{ "installer": agentInstaller, "version": fts.Version, - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, }).Error("No installer found") return errors.New("no installer found") @@ -261,7 +263,7 @@ func (fts *FleetTestSuite) installCerts() error { err := agentInstaller.InstallCertsFn() if err != nil { log.WithFields(log.Fields{ - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, "error": err, "installer": agentInstaller, @@ -278,9 +280,9 @@ func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { case "stale": desiredVersion = common.AgentStaleVersion case "latest": - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion default: - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion } return fts.kibanaClient.UpgradeAgent(fts.Hostname, desiredVersion) @@ -291,7 +293,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion } agentInVersionFn := func() error { @@ -312,38 +314,45 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) return backoff.Retry(agentInVersionFn, exp) } -// supported installers: tar, systemd +// this step infers the installer type from the underlying OS image +// supported images: centos and debian +func (fts *FleetTestSuite) anAgentIsDeployedToFleet(image string) error { + installerType := "rpm" + if image == "debian" { + installerType = "deb" + } + + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType) +} + +// supported installers: tar, rpm, deb func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error { - return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, false) + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType) } -func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(image string, installerType string, bootstrapFleetServer bool) error { +func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(image string, installerType string) error { log.WithFields(log.Fields{ - "bootstrapFleetServer": bootstrapFleetServer, - "image": image, - "installer": installerType, - }).Trace("Deploying an agent to Fleet with base image and fleet server") + "image": image, + "installer": installerType, + }).Trace("Deploying an agent to Fleet with base image using an already bootstrapped Fleet Server") + + deployedAgentsCount++ fts.Image = image fts.InstallerType = installerType agentInstaller := fts.getInstaller() - containerName := fts.getContainerName(agentInstaller, 1) // name of the container + containerName := fts.getContainerName(agentInstaller) // name of the container // enroll the agent with a new token - policy := fts.Policy - if bootstrapFleetServer { - policy = fts.FleetServerPolicy - } - - enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(policy) + enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(fts.Policy) if err != nil { return err } @@ -351,7 +360,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i fts.CurrentTokenID = enrollmentKey.ID var fleetConfig *kibana.FleetConfig - fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, fts.FleetServerHostname) + fleetConfig, err = deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken) if err != nil { return err @@ -366,7 +375,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i } // get container hostname once - hostname, err := docker.GetContainerHostname(containerName) + hostname, err := deploy.GetContainerHostname(containerName) if err != nil { return err } @@ -378,8 +387,8 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i // getContainerName returns the current container name for the service: // we are using the Docker client instead of docker-compose because it does not support // returning the output of a command: it simply returns error level -func (fts *FleetTestSuite) getContainerName(i installer.ElasticAgentInstaller, index int) string { - return fmt.Sprintf("%s_%s_%s_%d", i.Profile, i.Image, common.ElasticAgentServiceName, index) +func (fts *FleetTestSuite) getContainerName(i installer.ElasticAgentInstaller) string { + return fmt.Sprintf("%s_%s_%d", i.Profile, common.ElasticAgentServiceName, deployedAgentsCount) } // getServiceName returns the current service name, the one defined at the docker compose @@ -388,15 +397,14 @@ func (fts *FleetTestSuite) getServiceName(i installer.ElasticAgentInstaller) str } func (fts *FleetTestSuite) getInstaller() installer.ElasticAgentInstaller { - bootstrappedAgent := fts.FleetServerHostname == "" - - key := fmt.Sprintf("%s-%s-%s-%t", fts.Image, fts.InstallerType, fts.Version, bootstrappedAgent) + key := fmt.Sprintf("%s-%s-%s-%d", fts.Image, fts.InstallerType, fts.Version, deployedAgentsCount) // check if the agent is already cached if i, exists := fts.Installers[key]; exists { return i } - agentInstaller := installer.GetElasticAgentInstaller(fts.Image, fts.InstallerType, fts.Version, fts.FleetServerHostname) + // setting current index for the installer + agentInstaller := installer.GetElasticAgentInstaller(fts.Image, fts.InstallerType, fts.Version, deployedAgentsCount) // cache the new installer fts.Installers[key] = agentInstaller @@ -411,17 +419,20 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st serviceName := agentInstaller.Service // name of the service + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image) + if state == "started" { - return installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start") + return installer.SystemctlRun(profileService, imageService, serviceName, "start") } else if state == "restarted" { - err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop") + err := installer.SystemctlRun(profileService, imageService, serviceName, "stop") if err != nil { return err } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) - err = installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start") + err = installer.SystemctlRun(profileService, imageService, serviceName, "start") if err != nil { return err } @@ -447,7 +458,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st "process": process, }).Trace("Stopping process on the service") - err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop") + err := installer.SystemctlRun(profileService, imageService, serviceName, "stop") if err != nil { log.WithFields(log.Fields{ "action": state, @@ -459,9 +470,9 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st return err } - containerName := fts.getContainerName(agentInstaller, 1) + containerName := fts.getContainerName(agentInstaller) - return docker.CheckProcessStateOnTheHost(containerName, process, "stopped", 1, common.TimeoutFactor) + return CheckProcessState(fts.deployer, containerName, process, "stopped", 1, utils.TimeoutFactor) } func (fts *FleetTestSuite) setup() error { @@ -486,10 +497,10 @@ func theAgentIsListedInFleetWithStatus(desiredStatus string, hostname string) er if err != nil { return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentOnlineFn := func() error { agentID, err := kibanaClient.GetAgentIDByHostname(hostname) @@ -556,7 +567,7 @@ func theAgentIsListedInFleetWithStatus(desiredStatus string, hostname string) er func (fts *FleetTestSuite) theFileSystemAgentFolderIsEmpty() error { agentInstaller := fts.getInstaller() - containerName := fts.getContainerName(agentInstaller, 1) + containerName := fts.getContainerName(agentInstaller) content, err := agentInstaller.ListElasticAgentWorkingDirContent(containerName) if err != nil { @@ -573,7 +584,7 @@ func (fts *FleetTestSuite) theFileSystemAgentFolderIsEmpty() error { func (fts *FleetTestSuite) theHostIsRestarted() error { agentInstaller := fts.getInstaller() - containerName := fts.getContainerName(agentInstaller, 1) + containerName := fts.getContainerName(agentInstaller) _, err := shell.Execute(context.Background(), ".", "docker", "stop", containerName) if err != nil { log.WithFields(log.Fields{ @@ -583,7 +594,7 @@ func (fts *FleetTestSuite) theHostIsRestarted() error { }).Error("Could not stop the service") } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) _, err = shell.Execute(context.Background(), ".", "docker", "start", containerName) if err != nil { @@ -606,10 +617,10 @@ func (fts *FleetTestSuite) systemPackageDashboardsAreListedInFleet() error { log.Trace("Checking system Package dashboards in Fleet") dataStreamsCount := 0 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) countDataStreamsFn := func() error { dataStreams, err := fts.kibanaClient.GetDataStreams() @@ -675,7 +686,7 @@ func (fts *FleetTestSuite) theAgentIsReenrolledOnTheHost() error { // during an unenroll the fleet server exits as there is no longer // and agent id associated with the enrollment. When fleet server // restarts it needs a new agent to associate with the boostrap - cfg, err := kibana.NewFleetConfig(fts.CurrentToken, fts.FleetServerHostname) + cfg, err := kibana.NewFleetConfig(fts.CurrentToken) if err != nil { return err } @@ -708,7 +719,7 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error { } func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { - return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.FleetServerPolicy, packageName, action) + return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.Policy, packageName, action) } func theIntegrationIsOperatedInThePolicy(client *kibana.Client, policy kibana.Policy, packageName string, action string) error { @@ -750,10 +761,10 @@ func theIntegrationIsOperatedInThePolicy(client *kibana.Client, policy kibana.Po func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() error { log.Trace("Checking if the hostname is not shown in the Administration view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { host, err := fts.kibanaClient.IsAgentListedInSecurityApp(fts.Hostname) @@ -790,10 +801,10 @@ func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() func (fts *FleetTestSuite) theHostNameIsShownInTheAdminViewInTheSecurityApp(status string) error { log.Trace("Checking if the hostname is shown in the Admin view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { matches, err := fts.kibanaClient.IsAgentListedInSecurityAppWithStatus(fts.Hostname, status) @@ -845,10 +856,10 @@ func (fts *FleetTestSuite) thePolicyResponseWillBeShownInTheSecurityApp() error return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { listed, err := fts.kibanaClient.IsPolicyResponseListedInSecurityApp(agentID) @@ -905,7 +916,7 @@ func (fts *FleetTestSuite) thePolicyIsUpdatedToHaveMode(name string, mode string return godog.ErrPending } - packageDS, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.FleetServerPolicy) + packageDS, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.Policy) if err != nil { return err @@ -939,15 +950,15 @@ func (fts *FleetTestSuite) thePolicyWillReflectTheChangeInTheSecurityApp() error return err } - pkgPolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.FleetServerPolicy) + pkgPolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.Policy) if err != nil { return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { err := fts.kibanaClient.GetAgentEvents("endpoint-security", agentID, pkgPolicy.ID, fts.PolicyUpdatedAt) @@ -1001,11 +1012,15 @@ func (fts *FleetTestSuite) theVersionOfThePackageIsInstalled(version string, pac func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error { log.Trace("Enrolling a new agent with an revoked token") + // increase the number of agents + deployedAgentsCount++ + agentInstaller := fts.getInstaller() - containerName := fts.getContainerName(agentInstaller, 2) // name of the new container + containerName := fts.getContainerName(agentInstaller) // name of the new container + + fleetConfig, err := deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken) - fleetConfig, err := deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, fts.FleetServerHostname) // the installation process for TAR includes the enrollment if agentInstaller.InstallerType != "tar" { if err != nil { @@ -1143,7 +1158,7 @@ func (fts *FleetTestSuite) checkDataStream() error { return err } -func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containerName string, token string, fleetServerHost string) (*kibana.FleetConfig, error) { +func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer deploy.Deployment, containerName string, token string) (*kibana.FleetConfig, error) { profile := agentInstaller.Profile // name of the runtime dependencies compose file service := agentInstaller.Service // name of the service serviceTag := agentInstaller.Tag // docker tag of the service @@ -1155,9 +1170,11 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe // we are setting the container name because Centos service could be reused by any other test suite common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName - serviceManager := compose.NewServiceManager() + agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image).WithScale(deployedAgentsCount) - err := serviceManager.AddServicesToCompose(context.Background(), profile, []string{service}, common.ProfileEnv) + services := []deploy.ServiceRequest{deploy.NewServiceRequest(profile), agentService} + + err := deployer.Add(services, common.ProfileEnv) if err != nil { log.WithFields(log.Fields{ "service": service, @@ -1170,7 +1187,7 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe targetFile := "/" // copy downloaded agent to the root dir of the container - err = docker.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar) + err = deploy.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar) if err != nil { return nil, err } @@ -1180,7 +1197,7 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return nil, err } - cfg, cfgError := kibana.NewFleetConfig(token, fleetServerHost) + cfg, cfgError := kibana.NewFleetConfig(token) if cfgError != nil { return nil, cfgError } @@ -1237,16 +1254,22 @@ func inputs(integration string) []kibana.Input { } func (fts *FleetTestSuite) getContainerLogs() error { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() - profile := common.FleetProfileName + image := "" + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + image = agentInstaller.Image + } + + profile := deploy.NewServiceRequest(common.FleetProfileName) serviceName := common.ElasticAgentServiceName - composes := []string{ - profile, // profile name - serviceName, // agent service + services := []deploy.ServiceRequest{ + profile, // profile name + deploy.NewServiceRequest(serviceName).WithFlavour(image), // agent service } - err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) + err := serviceManager.RunCommand(profile, services, []string{"logs", serviceName}, common.ProfileEnv) if err != nil { log.WithFields(log.Fields{ "error": err, diff --git a/e2e/_suites/fleet/fleet_server.go b/e2e/_suites/fleet/fleet_server.go deleted file mode 100644 index daecf45fd9..0000000000 --- a/e2e/_suites/fleet/fleet_server.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package main - -import log "github.com/sirupsen/logrus" - -func (fts *FleetTestSuite) bootstrapFleetServerWithInstaller(image string, installerType string) error { - fts.ElasticAgentStopped = true - - log.WithFields(log.Fields{ - "image": image, - "installer": installerType, - }).Trace("Bootstrapping fleet server for the agent") - - err := fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, true) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "image": image, - "installer": installerType, - }).Error("Fleet server could not be bootstrapped for the agent") - return err - } - - log.WithFields(log.Fields{ - "fleetServerHostname": fts.FleetServerHostname, - "image": image, - "installer": installerType, - }).Info("Fleet server was bootstrapped for the agent") - - err = fts.theAgentIsListedInFleetWithStatus("online") - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "fleetServerHostname": fts.FleetServerHostname, - "image": image, - "installer": installerType, - }).Error("Fleet server could not reach the online status") - return err - } - - // the new compose files for fleet-server (centos/debian) are setting the hostname - // we need it here, before getting the installer, to get the installer using fleet-server host - fts.FleetServerHostname = "fleet-server-" + image - - return nil -} diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index 629795b5d0..e9df20c969 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -5,18 +5,14 @@ package main import ( - "context" "os" - "strings" "time" "github.com/cucumber/godog" "github.com/cucumber/messages-go/v10" "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" - "github.com/elastic/e2e-testing/internal/elasticsearch" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" @@ -34,43 +30,14 @@ func setUpSuite() { log.Error(err) os.Exit(1) } + + common.Provider = shell.GetEnv("PROVIDER", common.Provider) developerMode := shell.GetEnvBool("DEVELOPER_MODE") if developerMode { log.Info("Running in Developer mode 💻: runtime dependencies between different test runs will be reused to speed up dev cycle") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(common.AgentVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersionBase, - }).Fatal("Failed to get agent base version, aborting") - } - common.AgentVersionBase = v - - common.TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", common.TimeoutFactor) - common.AgentVersion = shell.GetEnv("BEAT_VERSION", common.AgentVersionBase) - - // check if version is an alias - v, err = utils.GetElasticArtifactVersion(common.AgentVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersion, - }).Fatal("Failed to get agent version, aborting") - } - common.AgentVersion = v - - common.StackVersion = shell.GetEnv("STACK_VERSION", common.StackVersion) - v, err = utils.GetElasticArtifactVersion(common.StackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.StackVersion, - }).Fatal("Failed to get stack version, aborting") - } - common.StackVersion = v + common.InitVersions() common.KibanaVersion = shell.GetEnv("KIBANA_VERSION", "") if common.KibanaVersion == "" { @@ -88,6 +55,7 @@ func setUpSuite() { imts = IngestManagerTestSuite{ Fleet: &FleetTestSuite{ kibanaClient: kibanaClient, + deployer: deploy.New(common.Provider), Installers: map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map }, } @@ -111,91 +79,51 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { } func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { - serviceManager := compose.NewServiceManager() + developerMode := shell.GetEnvBool("DEVELOPER_MODE") ctx.BeforeSuite(func() { setUpSuite() - log.Trace("Installing Fleet runtime dependencies") - - common.ProfileEnv = map[string]string{ - "kibanaVersion": common.KibanaVersion, - "stackVersion": common.StackVersion, - } + log.Trace("Bootstrapping Fleet Server") if !shell.GetEnvBool("SKIP_PULL") { images := []string{ - "docker.elastic.co/beats/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/beats/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/beats/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/beats/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/elasticsearch/elasticsearch:" + common.StackVersion, "docker.elastic.co/kibana/kibana:" + common.KibanaVersion, - "docker.elastic.co/observability-ci/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/observability-ci/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/observability-ci/elasticsearch:" + common.StackVersion, "docker.elastic.co/observability-ci/elasticsearch-ubi8:" + common.StackVersion, "docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion, "docker.elastic.co/observability-ci/kibana-ubi8:" + common.KibanaVersion, } - docker.PullImages(images) - } - - common.ProfileEnv["kibanaDockerNamespace"] = "kibana" - if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) { - // because it comes from a PR - common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci" - } - - profile := common.FleetProfileName - err := serviceManager.RunCompose(context.Background(), true, []string{profile}, common.ProfileEnv) - if err != nil { - log.WithFields(log.Fields{ - "profile": profile, - "error": err.Error(), - }).Fatal("Could not run the runtime dependencies for the profile.") + deploy.PullImages(images) } - minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute - healthy, err := elasticsearch.WaitForElasticsearch(context.Background(), minutesToBeHealthy) - if !healthy { - log.WithFields(log.Fields{ - "error": err, - "minutes": minutesToBeHealthy, - }).Fatal("The Elasticsearch cluster could not get the healthy status") - } - - kibanaClient, err := kibana.NewClient() - if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Fatal("Unable to create kibana client") - } - - healthyKibana, err := kibanaClient.WaitForReady(minutesToBeHealthy) - if !healthyKibana { - log.WithFields(log.Fields{ - "error": err, - "minutes": minutesToBeHealthy, - }).Fatal("The Kibana instance could not get the healthy status") - } - - imts.Fleet.setup() + deployer := deploy.New(common.Provider) + deployer.Bootstrap(func() error { + kibanaClient, err := kibana.NewClient() + if err != nil { + log.WithField("error", err).Fatal("Unable to create kibana client") + } + err = kibanaClient.WaitForFleet() + if err != nil { + log.WithField("error", err).Fatal("Fleet could not be initialized") + } + return nil + }) + imts.Fleet.Version = common.BeatVersionBase imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC() }) ctx.AfterSuite(func() { - developerMode := shell.GetEnvBool("DEVELOPER_MODE") if !developerMode { log.Debug("Destroying Fleet runtime dependencies") - profile := common.FleetProfileName - - err := serviceManager.StopCompose(context.Background(), true, []string{profile}) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "profile": profile, - }).Warn("Could not destroy the runtime dependencies for the profile.") - } + deployer := deploy.New(common.Provider) + deployer.Destroy() } installers := imts.Fleet.Installers diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 701bebda9f..b60d0ff10d 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -7,17 +7,17 @@ package main import ( "context" "fmt" + "path" + "strings" + "time" + "github.com/cenkalti/backoff/v4" "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "path" - "strings" - "time" "github.com/elastic/e2e-testing/internal/elasticsearch" log "github.com/sirupsen/logrus" @@ -32,6 +32,7 @@ func (fts *FleetTestSuite) bootstrapFleetServerFromAStandaloneAgent(image string if err != nil { return err } + fts.FleetServerPolicy = fleetPolicy return fts.startStandAloneAgent(image, "", map[string]string{"fleetServerMode": "1"}) } @@ -43,11 +44,11 @@ func (fts *FleetTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud( } fts.FleetServerPolicy = fleetPolicy volume := path.Join(config.OpDir(), "compose", "services", "elastic-agent", "apm-legacy") - return fts.startStandAloneAgent(image, "docker-compose-cloud.yml", map[string]string{"apmVolume": volume}) + return fts.startStandAloneAgent(image, "cloud", map[string]string{"apmVolume": volume}) } func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 minimumHitsCount := 50 result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) @@ -61,9 +62,11 @@ func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { } func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error { - serviceManager := compose.NewServiceManager() - - err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(serviceName), + } + err := fts.deployer.Remove(services, common.ProfileEnv) if err != nil { return err } @@ -91,11 +94,11 @@ func (fts *FleetTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error return elasticsearch.AssertHitsAreNotPresent(result) } -func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename string, env map[string]string) error { +func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, env map[string]string) error { fts.StandAlone = true log.Trace("Deploying an agent to Fleet") - dockerImageTag := common.AgentVersion + dockerImageTag := common.BeatVersion useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") @@ -103,15 +106,13 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st // load the docker images that were already: // a. downloaded from the GCP bucket // b. fetched from the local beats binaries - dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.AgentVersion, "") + dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.BeatVersion, deployedAgentsCount) dockerInstaller.PreInstallFn() dockerImageTag += "-amd64" } - serviceManager := compose.NewServiceManager() - common.ProfileEnv["elasticAgentDockerImageSuffix"] = "" if image != "default" { common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image @@ -129,15 +130,18 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st common.ProfileEnv[k] = v } - err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName, - []string{common.ElasticAgentServiceName}, common.ProfileEnv, composeFilename) + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(flavour), + } + err := fts.deployer.Add(services, common.ProfileEnv) if err != nil { log.Error("Could not deploy the elastic-agent") return err } // get container hostname once - hostname, err := docker.GetContainerHostname(containerName) + hostname, err := deploy.GetContainerHostname(containerName) if err != nil { return err } @@ -155,21 +159,21 @@ func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename st func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { log.WithFields(log.Fields{ - "policyID": fts.FleetServerPolicy.ID, + "policyID": fts.Policy.ID, "package": packageName, }).Trace("Checking if the policy shows the package added") maxTimeout := time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) configurationIsPresentFn := func() error { - packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.FleetServerPolicy) + packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.Policy) if err != nil { log.WithFields(log.Fields{ "packagePolicy": packagePolicy, - "policy": fts.FleetServerPolicy, + "policy": fts.Policy, "retry": retryCount, "error": err, }).Warn("The integration was not found in the policy") @@ -204,7 +208,7 @@ func (fts *FleetTestSuite) installTestTools(containerName string) error { "containerName": containerName, }).Trace("Installing test tools ") - _, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + _, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { log.WithFields(log.Fields{ "command": cmd, diff --git a/e2e/_suites/fleet/world.go b/e2e/_suites/fleet/world.go index 132f56f575..4e02804303 100644 --- a/e2e/_suites/fleet/world.go +++ b/e2e/_suites/fleet/world.go @@ -7,9 +7,14 @@ package main import ( "fmt" "strconv" + "strings" + "time" + "github.com/cenkalti/backoff/v4" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" ) // IngestManagerTestSuite represents a test suite, holding references to the pieces needed to run the tests @@ -30,7 +35,7 @@ func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocur containerName = fmt.Sprintf("%s_%s_%d", profile, common.ElasticAgentServiceName, 1) } else { agentInstaller := imts.Fleet.getInstaller() - containerName = imts.Fleet.getContainerName(agentInstaller, 1) + containerName = imts.Fleet.getContainerName(agentInstaller) } count, err := strconv.Atoi(ocurrences) @@ -38,5 +43,201 @@ func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocur return err } - return docker.CheckProcessStateOnTheHost(containerName, process, state, count, common.TimeoutFactor) + return CheckProcessState(imts.Fleet.deployer, containerName, process, state, count, utils.TimeoutFactor) +} + +// CheckProcessState checks if a process is in the desired state in a container +// name of the container for the service: +// we are using the underlying deployer to run the commands in the container/service +func CheckProcessState(deployer deploy.Deployment, service string, process string, state string, occurrences int, timeoutFactor int) error { + timeout := time.Duration(utils.TimeoutFactor) * time.Minute + + err := waitForProcess(deployer, service, process, state, occurrences, timeout) + if err != nil { + if state == "started" { + log.WithFields(log.Fields{ + "service ": service, + "error": err, + "timeout": timeout, + }).Error("The process was not found but should be present") + } else { + log.WithFields(log.Fields{ + "service": service, + "error": err, + "timeout": timeout, + }).Error("The process was found but shouldn't be present") + } + + return err + } + + return nil +} + +// waitForProcess polls a container executing "ps" command until the process is in the desired state (present or not), +// or a timeout happens +func waitForProcess(deployer deploy.Deployment, service string, process string, desiredState string, ocurrences int, maxTimeout time.Duration) error { + exp := utils.GetExponentialBackOff(maxTimeout) + + mustBePresent := false + if desiredState == "started" { + mustBePresent = true + } + retryCount := 1 + + // wrap service into a request for the deployer + serviceRequest := deploy.NewServiceRequest(service) + + processStatus := func() error { + log.WithFields(log.Fields{ + "desiredState": desiredState, + "ocurrences": ocurrences, + "process": process, + }).Trace("Checking process desired state on the container") + + // pgrep -d: -d, --delimiter specify output delimiter + //i.e. "pgrep -d , metricbeat": 483,519 + cmds := []string{"pgrep", "-d", ",", process} + output, err := deployer.ExecIn(serviceRequest, cmds) + if err != nil { + log.WithFields(log.Fields{ + "cmds": cmds, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": ocurrences, + "process": process, + "retry": retryCount, + }).Warn("Could not get number of processes in the container") + + retryCount++ + + return err + } + + // tokenize the pids to get each pid's state, adding them to an array if they match the desired state + // From Split docs: + // If output does not contain sep and sep is not empty, Split returns a + // slice of length 1 whose only element is s, that's why we first initialise to the empty array + pids := strings.Split(output, ",") + if len(pids) == 1 && pids[0] == "" { + pids = []string{} + } + + log.WithFields(log.Fields{ + "count": len(pids), + "desiredState": desiredState, + "mustBePresent": mustBePresent, + "pids": pids, + "process": process, + }).Tracef("Pids for process found") + + desiredStatePids := []string{} + + for _, pid := range pids { + pidStateCmds := []string{"ps", "-q", pid, "-o", "state", "--no-headers"} + pidState, err := deployer.ExecIn(serviceRequest, pidStateCmds) + if err != nil { + log.WithFields(log.Fields{ + "cmds": cmds, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": ocurrences, + "pid": pid, + "process": process, + "retry": retryCount, + }).Warn("Could not check pid status in the container") + + retryCount++ + + return err + } + + log.WithFields(log.Fields{ + "desiredState": desiredState, + "mustBePresent": mustBePresent, + "pid": pid, + "pidState": pidState, + "process": process, + }).Tracef("Checking if process is in the S state") + + // if the process must be present, then check for the S state + // From 'man ps': + // D uninterruptible sleep (usually IO) + // R running or runnable (on run queue) + // S interruptible sleep (waiting for an event to complete) + // T stopped by job control signal + // t stopped by debugger during the tracing + // W paging (not valid since the 2.6.xx kernel) + // X dead (should never be seen) + // Z defunct ("zombie") process, terminated but not reaped by its parent + if mustBePresent && pidState == "S" { + desiredStatePids = append(desiredStatePids, pid) + } else if !mustBePresent { + desiredStatePids = append(desiredStatePids, pid) + } + } + + occurrencesMatched := (len(desiredStatePids) == ocurrences) + + // both true or both false + if mustBePresent == occurrencesMatched { + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "desiredState": desiredState, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": len(desiredStatePids), + "process": process, + }).Infof("Process desired state checked") + + return nil + } + + if mustBePresent { + err = fmt.Errorf("%s process is not running in the container with the desired number of occurrences (%d) yet", process, ocurrences) + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "ocurrences": len(desiredStatePids), + "process": process, + "retry": retryCount, + }).Warn(err.Error()) + + retryCount++ + + return err + } + + err = fmt.Errorf("%s process is still running in the container", process) + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "ocurrences": len(desiredStatePids), + "process": process, + "state": desiredState, + "retry": retryCount, + }).Warn(err.Error()) + + retryCount++ + + return err + } + + err := backoff.Retry(processStatus, exp) + if err != nil { + return err + } + + return nil } diff --git a/e2e/_suites/helm/helm_charts_test.go b/e2e/_suites/helm/helm_charts_test.go index b3ae9974e3..8ab65ce860 100644 --- a/e2e/_suites/helm/helm_charts_test.go +++ b/e2e/_suites/helm/helm_charts_test.go @@ -15,7 +15,7 @@ import ( "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/e2e/steps" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/helm" "github.com/elastic/e2e-testing/internal/kubectl" "github.com/elastic/e2e-testing/internal/shell" @@ -37,10 +37,6 @@ var elasticAPMActive = false var helmManager helm.Manager -// timeoutFactor a multiplier for the max timeout when doing backoff retries. -// It can be overriden by TIMEOUT_FACTOR env var -var timeoutFactor = 2 - //nolint:unused var kubectlClient kubectl.Kubectl @@ -53,10 +49,6 @@ var helmChartVersion = "7.11.2" // kubernetesVersion represents the default version used for Kubernetes var kubernetesVersion = "1.18.2" -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = "7.x-SNAPSHOT" - var testSuite HelmChartTestSuite var tx *apm.Transaction @@ -80,17 +72,8 @@ func setupSuite() { helmVersion = shell.GetEnv("HELM_VERSION", helmVersion) helmChartVersion = shell.GetEnv("HELM_CHART_VERSION", helmChartVersion) kubernetesVersion = shell.GetEnv("KUBERNETES_VERSION", kubernetesVersion) - timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor) - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err := utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v + common.InitVersions() h, err := helm.Factory(helmVersion) if err != nil { @@ -187,9 +170,9 @@ func (ts *HelmChartTestSuite) aResourceWillExposePods(resourceType string) error return err } - maxTimeout := time.Duration(timeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 checkEndpointsFn := func() error { @@ -412,7 +395,7 @@ func (ts *HelmChartTestSuite) install(ctx context.Context, chart string) error { "chart": ts.Name, }).Info("Rancher Local Path Provisioner and local-path storage class for Elasticsearch volumes installed") - maxTimeout := common.TimeoutFactor * 100 + maxTimeout := utils.TimeoutFactor * 100 log.Debug("Applying workaround to use Rancher's local-path storage class for Elasticsearch volumes") flags = []string{"--wait", fmt.Sprintf("--timeout=%ds", maxTimeout), "--values", "https://raw.githubusercontent.com/elastic/helm-charts/master/elasticsearch/examples/kubernetes-kind/values.yaml"} @@ -677,19 +660,20 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") if elasticAPMActive && elasticAPMEnvironment == "local" { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - err := serviceManager.RunCompose(suiteContext, true, []string{"helm"}, env) + err := serviceManager.RunCompose( + suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("helm")}, env) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", }).Warn("Could not run the profile.") } - steps.AddAPMServicesForInstrumentation(suiteContext, "helm", stackVersion, true, env) + steps.AddAPMServicesForInstrumentation(suiteContext, "helm", common.StackVersion, true, env) } err := testSuite.createCluster(suiteContext, testSuite.KubernetesVersion) @@ -731,8 +715,8 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { } if elasticAPMActive { - serviceManager := compose.NewServiceManager() - err := serviceManager.StopCompose(suiteContext, true, []string{"helm"}) + serviceManager := deploy.NewServiceManager() + err := serviceManager.StopCompose(suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("helm")}) if err != nil { log.WithFields(log.Fields{ "profile": "helm", diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index 2717b0bc2f..4e1d33c838 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -23,8 +23,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kubernetes" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" @@ -37,14 +36,6 @@ const defaultBeatVersion = "7.x-SNAPSHOT" var defaultEventsWaitTimeout = 60 * time.Second var defaultDeployWaitTimeout = 60 * time.Second -func init() { - // initialise timeout factor - common.TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", common.TimeoutFactor) - - defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(common.TimeoutFactor) - defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(common.TimeoutFactor) -} - type podsManager struct { kubectl kubernetes.Control ctx context.Context @@ -131,13 +122,13 @@ func (m *podsManager) configureDockerImage(podName string) error { // this method will detect if the GITHUB_CHECK_SHA1 variable is set artifactName := utils.BuildArtifactName(podName, beatVersion, defaultBeatVersion, "linux", "amd64", "tar.gz", true) - imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, podName, beatVersion, defaultBeatVersion, utils.TimeoutFactor, true) if err != nil { return err } // load the TAR file into the docker host as a Docker image - err = docker.LoadImage(imagePath) + err = deploy.LoadImage(imagePath) if err != nil { return err } @@ -145,7 +136,7 @@ func (m *podsManager) configureDockerImage(podName string) error { beatVersion = beatVersion + "-amd64" // tag the image with the proper docker tag, including platform - err = docker.TagImage( + err = deploy.TagImage( "docker.elastic.co/beats/"+podName+":"+defaultBeatVersion, "docker.elastic.co/observability-ci/"+podName+":"+beatVersion, ) @@ -472,6 +463,9 @@ func InitializeTestSuite(ctx *godog.TestSuiteContext) { // init logger config.Init() + defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(utils.TimeoutFactor) + defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(utils.TimeoutFactor) + err := cluster.Initialize(suiteContext, "testdata/kind.yml") if err != nil { log.WithError(err).Fatal("Failed to initialize cluster") diff --git a/e2e/_suites/metricbeat/metricbeat_test.go b/e2e/_suites/metricbeat/metricbeat_test.go index de3ef0c8f5..3900248c11 100644 --- a/e2e/_suites/metricbeat/metricbeat_test.go +++ b/e2e/_suites/metricbeat/metricbeat_test.go @@ -17,8 +17,7 @@ import ( "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/e2e/steps" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" @@ -34,17 +33,7 @@ var developerMode = false var elasticAPMActive = false -var metricbeatVersionBase = "7.x-SNAPSHOT" - -// metricbeatVersion is the version of the metricbeat to use -// It can be overriden by BEAT_VERSION env var -var metricbeatVersion = metricbeatVersionBase - -var serviceManager compose.ServiceManager - -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = metricbeatVersionBase +var serviceManager deploy.ServiceManager var testSuite MetricbeatTestSuite @@ -66,29 +55,9 @@ func setupSuite() { }).Info("Current execution will be instrumented 🛠") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(metricbeatVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": metricbeatVersionBase, - }).Fatal("Failed to get metricbeat base version, aborting") - } - metricbeatVersionBase = v + common.InitVersions() - metricbeatVersion = shell.GetEnv("BEAT_VERSION", metricbeatVersionBase) - - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err = utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v - - serviceManager = compose.NewServiceManager() + serviceManager = deploy.NewServiceManager() testSuite = MetricbeatTestSuite{ Query: elasticsearch.Query{}, @@ -153,7 +122,7 @@ func (mts *MetricbeatTestSuite) CleanUp() error { testSuite.currentContext = apm.ContextWithSpan(context.Background(), span) defer span.End() - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() fn := func(ctx context.Context) { err := elasticsearch.DeleteIndex(ctx, mts.getIndexName()) @@ -167,15 +136,15 @@ func (mts *MetricbeatTestSuite) CleanUp() error { defer fn(context.Background()) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - services := []string{"metricbeat"} + services := []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")} if mts.ServiceName != "" { - services = append(services, mts.ServiceName) + services = append(services, deploy.NewServiceRequest(mts.ServiceName)) } - err := serviceManager.RemoveServicesFromCompose(mts.currentContext, "metricbeat", services, env) + err := serviceManager.RemoveServicesFromCompose(mts.currentContext, deploy.NewServiceRequest("metricbeat"), services, env) if mts.cleanUpTmpFiles { if _, err := os.Stat(mts.configurationFile); err == nil { @@ -259,20 +228,21 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { suiteContext = apm.ContextWithSpan(suiteContext, suiteParentSpan) defer suiteParentSpan.End() - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - err := serviceManager.RunCompose(suiteContext, true, []string{"metricbeat"}, env) + err := serviceManager.RunCompose( + suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}, env) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", }).Fatal("Could not run the profile.") } - minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute + minutesToBeHealthy := time.Duration(utils.TimeoutFactor) * time.Minute healthy, err := elasticsearch.WaitForElasticsearch(suiteContext, minutesToBeHealthy) if !healthy { log.WithFields(log.Fields{ @@ -283,7 +253,7 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") if elasticAPMActive && elasticAPMEnvironment == "local" { - steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", stackVersion, true, env) + steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", common.StackVersion, true, env) } }) @@ -305,8 +275,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { defer suiteParentSpan.End() if !developerMode { - serviceManager := compose.NewServiceManager() - err := serviceManager.StopCompose(suiteContext, true, []string{"metricbeat"}) + serviceManager := deploy.NewServiceManager() + err := serviceManager.StopCompose(suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", @@ -320,7 +290,7 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForModule(serviceType stri serviceType = strings.ToLower(serviceType) // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() mts.ServiceType = serviceType @@ -373,16 +343,10 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForVariantModule(serviceVa } func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string) error { - // restore initial state - metricbeatVersionBackup := metricbeatVersion - defer func() { metricbeatVersion = metricbeatVersionBackup }() - // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() - metricbeatVersion = utils.CheckPRVersion(metricbeatVersion, metricbeatVersionBase) - configurationFilePath, err := steps.FetchBeatConfiguration(false, "metricbeat", configuration+".yml") if err != nil { return err @@ -407,22 +371,22 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") if useCISnapshots || beatsLocalPath != "" { - artifactName := utils.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", "amd64", "tar.gz", true) + artifactName := utils.BuildArtifactName("metricbeat", mts.Version, common.BeatVersionBase, "linux", "amd64", "tar.gz", true) - imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, metricbeatVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return err } - err = docker.LoadImage(imagePath) + err = deploy.LoadImage(imagePath) if err != nil { return err } mts.Version = mts.Version + "-amd64" - err = docker.TagImage( - "docker.elastic.co/beats/metricbeat:"+metricbeatVersionBase, + err = deploy.TagImage( + "docker.elastic.co/beats/metricbeat:"+common.BeatVersionBase, "docker.elastic.co/observability-ci/metricbeat:"+mts.Version, ) if err != nil { @@ -431,7 +395,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { } // this is needed because, in general, the target service (apache, mysql, redis) does not have a healthcheck - waitForService := time.Duration(common.TimeoutFactor) * 10 * time.Second + waitForService := time.Duration(utils.TimeoutFactor) * 10 * time.Second if mts.ServiceName == "ceph" { // see https://github.com/elastic/beats/blob/ef6274d0d1e36308a333cbed69846a1bd63528ae/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go#L35 // Ceph service needs more time to start up @@ -440,7 +404,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { utils.Sleep(waitForService) - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() logLevel := log.GetLevel().String() if log.GetLevel() == log.TraceLevel { @@ -453,7 +417,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { "logLevel": logLevel, "metricbeatConfigFile": mts.configurationFile, "metricbeatTag": mts.Version, - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, mts.ServiceName + "Tag": mts.ServiceVersion, "serviceName": mts.ServiceName, } @@ -461,7 +425,11 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { env["metricbeatDockerNamespace"] = utils.GetDockerNamespaceEnvVar("beats") env["metricbeatPlatform"] = "linux/amd64" - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{"metricbeat"}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, + deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}, + env) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -492,13 +460,13 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { } if log.IsLevelEnabled(log.DebugLevel) { - composes := []string{ - "metricbeat", // profile name - "metricbeat", // metricbeat service + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest("metricbeat"), // profile name + deploy.NewServiceRequest("metricbeat"), // metricbeat service } if developerMode { - err = serviceManager.RunCommand("metricbeat", composes, []string{"logs", "metricbeat"}, env) + err = serviceManager.RunCommand(deploy.NewServiceRequest("metricbeat"), services, []string{"logs", "metricbeat"}, env) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -517,11 +485,14 @@ func (mts *MetricbeatTestSuite) serviceIsRunningForMetricbeat(serviceType string serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceEnvironment(env, serviceType, serviceVersion) - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{serviceType}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest(serviceType)}, + env) if err != nil { log.WithFields(log.Fields{ "service": serviceType, @@ -542,11 +513,14 @@ func (mts *MetricbeatTestSuite) serviceVariantIsRunningForMetricbeat( serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceVariantEnvironment(env, serviceType, serviceVariant, serviceVersion) - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{serviceType}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest(serviceType)}, + env) if err != nil { log.WithFields(log.Fields{ "service": serviceType, @@ -577,7 +551,7 @@ func (mts *MetricbeatTestSuite) thereAreEventsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { @@ -613,7 +587,7 @@ func (mts *MetricbeatTestSuite) thereAreNoErrorsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { diff --git a/e2e/steps/befores.go b/e2e/steps/befores.go index d4cb3a2e3b..200c45095b 100644 --- a/e2e/steps/befores.go +++ b/e2e/steps/befores.go @@ -8,24 +8,24 @@ import ( "context" "strings" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/shell" log "github.com/sirupsen/logrus" ) // AddAPMServicesForInstrumentation adds a Kibana and APM Server instances to the running project func AddAPMServicesForInstrumentation(ctx context.Context, profile string, stackVersion string, needsKibana bool, env map[string]string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() apmServerURL := shell.GetEnv("APM_SERVER_URL", "") if strings.HasPrefix(apmServerURL, "http://localhost") { - apmServices := []string{ - "apm-server", + apmServices := []deploy.ServiceRequest{ + deploy.NewServiceRequest("apm-server"), } if needsKibana { env["kibanaTag"] = stackVersion - apmServices = append(apmServices, "kibana") + apmServices = append(apmServices, deploy.NewServiceRequest("kibana")) } log.WithFields(log.Fields{ @@ -34,7 +34,7 @@ func AddAPMServicesForInstrumentation(ctx context.Context, profile string, stack }).Info("Starting local APM services for instrumentation") env["apmServerTag"] = stackVersion - err := serviceManager.AddServicesToCompose(ctx, profile, apmServices, env) + err := serviceManager.AddServicesToCompose(ctx, deploy.NewServiceRequest(profile), apmServices, env) if err != nil { log.WithFields(log.Fields{ "error": err, diff --git a/internal/common/defaults.go b/internal/common/defaults.go index fda9208c78..a5de943f2b 100644 --- a/internal/common/defaults.go +++ b/internal/common/defaults.go @@ -4,6 +4,12 @@ package common +import ( + "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" +) + // ElasticAgentProcessName the name of the process for the Elastic Agent const ElasticAgentProcessName = "elastic-agent" @@ -22,12 +28,12 @@ const FleetProfileName = "fleet" // FleetServerAgentServiceName the name of the service for the Elastic Agent const FleetServerAgentServiceName = "fleet-server" -// AgentVersionBase is the base version of the agent to use -var AgentVersionBase = "7.x-SNAPSHOT" +// BeatVersionBase is the base version of the Beat to use +var BeatVersionBase = "7.x-SNAPSHOT" -// AgentVersion is the version of the agent to use +// BeatVersion is the version of the Beat to use // It can be overriden by BEAT_VERSION env var -var AgentVersion = AgentVersionBase +var BeatVersion = BeatVersionBase // AgentStaleVersion is the version of the agent to use as a base during upgrade // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. @@ -35,12 +41,54 @@ var AgentStaleVersion = "7.13-SNAPSHOT" // StackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var -var StackVersion = AgentVersionBase +var StackVersion = BeatVersionBase // KibanaVersion is the version of kibana to use // It can be override by KIBANA_VERSION -var KibanaVersion = AgentVersionBase +var KibanaVersion = BeatVersionBase // ProfileEnv is the environment to be applied to any execution // affecting the runtime dependencies (or profile) var ProfileEnv map[string]string + +// Provider is the deployment provider used, currently docker is supported +var Provider = "docker" + +// InitVersions initialise default versions. We do not want to do it in the init phase +// supporting lazy-loading the versions when needed. Basically, the CLI part does not +// need to load them +func InitVersions() { + v, err := utils.GetElasticArtifactVersion(BeatVersionBase) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersionBase, + }).Fatal("Failed to get Beat base version, aborting") + } + BeatVersionBase = v + + BeatVersion = shell.GetEnv("BEAT_VERSION", BeatVersionBase) + + // check if version is an alias + v, err = utils.GetElasticArtifactVersion(BeatVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersion, + }).Fatal("Failed to get Beat version, aborting") + } + BeatVersion = v + + // detects if the BeatVersion is set by the GITHUB_CHECK_SHA1 variable + BeatVersion = utils.CheckPRVersion(BeatVersion, BeatVersionBase) + + StackVersion = shell.GetEnv("STACK_VERSION", BeatVersionBase) + v, err = utils.GetElasticArtifactVersion(StackVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": StackVersion, + }).Fatal("Failed to get stack version, aborting") + } + StackVersion = v +} diff --git a/internal/deploy/base.go b/internal/deploy/base.go new file mode 100644 index 0000000000..4500c6ee2c --- /dev/null +++ b/internal/deploy/base.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "strings" +) + +// Deployment interface for operations dealing with deployments of the bits +// required for testing +type Deployment interface { + Add(services []ServiceRequest, env map[string]string) error // adds a service to deployment + Bootstrap(waitCB func() error) error // will bootstrap or reuse existing cluster if kubernetes is selected + Destroy() error // Teardown deployment + ExecIn(service ServiceRequest, cmd []string) (string, error) // Execute arbitrary commands in service + Inspect(service ServiceRequest) (*ServiceManifest, error) // inspects service + Remove(services []ServiceRequest, env map[string]string) error // Removes services from deployment +} + +// ServiceManifest information about a service in a deployment +type ServiceManifest struct { + ID string + Name string + Connection string // a string representing how to connect to service + Hostname string +} + +// ServiceRequest represents the service to be created using the provider +type ServiceRequest struct { + Name string + Flavour string // optional, configured using builder method + Scale int // default: 1 +} + +// NewServiceRequest creates a request for a service +func NewServiceRequest(n string) ServiceRequest { + return ServiceRequest{ + Name: n, + Scale: 1, + } +} + +// WithFlavour adds a flavour for the service, resulting in a look-up of the service in the config directory, +// using flavour as a subdir of the service +func (sr ServiceRequest) WithFlavour(f string) ServiceRequest { + sr.Flavour = f + return sr +} + +// WithScale adds the scale index to the service +func (sr ServiceRequest) WithScale(s int) ServiceRequest { + if s < 1 { + s = 1 + } + + sr.Scale = s + return sr +} + +// New creates a new deployment +func New(provider string) Deployment { + if strings.EqualFold(provider, "docker") { + return newDockerDeploy() + } + if strings.EqualFold(provider, "kubernetes") { + return newK8sDeploy() + } + return nil +} diff --git a/internal/compose/compose.go b/internal/deploy/compose.go similarity index 55% rename from internal/compose/compose.go rename to internal/deploy/compose.go index a2f3e2cff5..2a52128c18 100644 --- a/internal/compose/compose.go +++ b/internal/deploy/compose.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package compose +package deploy import ( "context" @@ -19,12 +19,12 @@ import ( // ServiceManager manages lifecycle of a service type ServiceManager interface { - AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string, composeFilename ...string) error - ExecCommandInService(profile string, image string, serviceName string, cmds []string, env map[string]string, detach bool) error - RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error - RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error - RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error - StopCompose(ctx context.Context, isProfile bool, composeNames []string) error + AddServicesToCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error + ExecCommandInService(profile ServiceRequest, image ServiceRequest, serviceName string, cmds []string, env map[string]string, detach bool) error + RemoveServicesFromCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error + RunCommand(profile ServiceRequest, services []ServiceRequest, composeArgs []string, env map[string]string) error + RunCompose(ctx context.Context, isProfile bool, services []ServiceRequest, env map[string]string) error + StopCompose(ctx context.Context, isProfile bool, services []ServiceRequest) error } // DockerServiceManager implementation of the service manager interface @@ -37,7 +37,7 @@ func NewServiceManager() ServiceManager { } // AddServicesToCompose adds services to a running docker compose -func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string, composeFilename ...string) error { +func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Add services to Docker Compose", "docker-compose.services.add", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -45,18 +45,30 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil log.WithFields(log.Fields{ "profile": profile, - "services": composeNames, + "services": services, }).Trace("Adding services to compose") - newComposeNames := []string{profile} - newComposeNames = append(newComposeNames, composeNames...) + scaleCmds := []string{} + newServices := []ServiceRequest{profile} + for _, srv := range services { + newServices = append(newServices, srv) + if srv.Scale > 1 { + scaleCmds = append(scaleCmds, fmt.Sprintf("%s=%d", srv.Name, srv.Scale)) + } + } - persistedEnv := state.Recover(profile+"-profile", config.Op.Workspace) + persistedEnv := state.Recover(profile.Name+"-profile", config.Op.Workspace) for k, v := range env { persistedEnv[k] = v } - err := executeCompose(true, newComposeNames, []string{"up", "-d"}, persistedEnv, composeFilename...) + cmds := []string{"up", "-d"} + if len(scaleCmds) > 0 { + cmds = append(cmds, "--scale") + cmds = append(cmds, scaleCmds...) + } + + err := executeCompose(true, newServices, cmds, persistedEnv) if err != nil { return err } @@ -65,8 +77,8 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil } // ExecCommandInService executes a command in a service from a profile -func (sm *DockerServiceManager) ExecCommandInService(profile string, image string, serviceName string, cmds []string, env map[string]string, detach bool) error { - composes := []string{ +func (sm *DockerServiceManager) ExecCommandInService(profile ServiceRequest, image ServiceRequest, serviceName string, cmds []string, env map[string]string, detach bool) error { + services := []ServiceRequest{ profile, // profile name image, // image for the service } @@ -74,10 +86,11 @@ func (sm *DockerServiceManager) ExecCommandInService(profile string, image strin if detach { composeArgs = append(composeArgs, "-d") } + composeArgs = append(composeArgs, "--index", fmt.Sprintf("%d", image.Scale)) composeArgs = append(composeArgs, serviceName) composeArgs = append(composeArgs, cmds...) - err := sm.RunCommand(profile, composes, composeArgs, env) + err := sm.RunCommand(profile, services, composeArgs, env) if err != nil { log.WithFields(log.Fields{ "command": cmds, @@ -92,7 +105,7 @@ func (sm *DockerServiceManager) ExecCommandInService(profile string, image strin } // RemoveServicesFromCompose removes services from a running docker compose -func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Remove services from Docker Compose", "docker-compose.services.remove", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -100,33 +113,33 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p log.WithFields(log.Fields{ "profile": profile, - "services": composeNames, + "services": services, }).Trace("Removing services from compose") - newComposeNames := []string{profile} - newComposeNames = append(newComposeNames, composeNames...) + newServices := []ServiceRequest{profile} + newServices = append(newServices, services...) - persistedEnv := state.Recover(profile+"-profile", config.Op.Workspace) + persistedEnv := state.Recover(profile.Name+"-profile", config.Op.Workspace) for k, v := range env { persistedEnv[k] = v } - for _, composeName := range composeNames { + for _, srv := range services { command := []string{"rm", "-fvs"} - command = append(command, composeName) + command = append(command, srv.Name) - err := executeCompose(true, newComposeNames, command, persistedEnv) + err := executeCompose(true, newServices, command, persistedEnv) if err != nil { log.WithFields(log.Fields{ "command": command, - "service": composeName, + "service": srv, "profile": profile, }).Error("Could not remove service from compose") return err } log.WithFields(log.Fields{ "profile": profile, - "service": composeName, + "service": srv, }).Debug("Service removed from compose") } @@ -134,48 +147,54 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p } // RunCommand executes a docker-compose command in a running a docker compose -func (sm *DockerServiceManager) RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error { - return executeCompose(true, composeNames, composeArgs, env) +func (sm *DockerServiceManager) RunCommand(profile ServiceRequest, services []ServiceRequest, composeArgs []string, env map[string]string) error { + return executeCompose(true, services, composeArgs, env) } // RunCompose runs a docker compose by its name -func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Starting Docker Compose files", "docker-compose.services.up", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() - return executeCompose(isProfile, composeNames, []string{"up", "-d"}, env) + return executeCompose(isProfile, services, []string{"up", "-d"}, env) } // StopCompose stops a docker compose by its name -func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, composeNames []string) error { +func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, services []ServiceRequest) error { span, _ := apm.StartSpanOptions(ctx, "Stopping Docker Compose files", "docker-compose.services.down", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() - composeFilePaths := make([]string, len(composeNames)) - for i, composeName := range composeNames { + composeFilePaths := make([]string, len(services)) + for i, srv := range services { b := isProfile - if i == 0 && !isProfile && (len(composeName) == 1) { + if i == 0 && !isProfile && (len(services) == 1) { b = true } - composeFilePath, err := config.GetComposeFile(b, composeName) + serviceIncludingFlavour := srv.Name + if srv.Flavour != "" { + // discover the flavour in the subdir + serviceIncludingFlavour = filepath.Join(srv.Name, srv.Flavour) + } + + composeFilePath, err := config.GetComposeFile(b, serviceIncludingFlavour) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } composeFilePaths[i] = composeFilePath } - ID := composeNames[0] + "-service" + ID := services[0].Name + "-service" if isProfile { - ID = composeNames[0] + "-profile" + ID = services[0].Name + "-profile" } persistedEnv := state.Recover(ID, config.Op.Workspace) - err := executeCompose(isProfile, composeNames, []string{"down", "--remove-orphans"}, persistedEnv) + err := executeCompose(isProfile, services, []string{"down", "--remove-orphans"}, persistedEnv) if err != nil { return fmt.Errorf("Could not stop compose file: %v - %v", composeFilePaths, err) } @@ -183,28 +202,34 @@ func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, log.WithFields(log.Fields{ "composeFilePath": composeFilePaths, - "profile": composeNames[0], + "profile": services[0].Name, }).Trace("Docker compose down.") return nil } -func executeCompose(isProfile bool, composeNames []string, command []string, env map[string]string, composeFilename ...string) error { - composeFilePaths := make([]string, len(composeNames)) - for i, composeName := range composeNames { +func executeCompose(isProfile bool, services []ServiceRequest, command []string, env map[string]string) error { + composeFilePaths := make([]string, len(services)) + for i, srv := range services { b := false if i == 0 && isProfile { b = true } - composeFilePath, err := config.GetComposeFile(b, composeName, composeFilename...) + serviceIncludingFlavour := srv.Name + if srv.Flavour != "" { + // discover the flavour in the subdir + serviceIncludingFlavour = filepath.Join(srv.Name, srv.Flavour) + } + + composeFilePath, err := config.GetComposeFile(b, serviceIncludingFlavour) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } composeFilePaths[i] = composeFilePath } - compose := tc.NewLocalDockerCompose(composeFilePaths, composeNames[0]) + compose := tc.NewLocalDockerCompose(composeFilePaths, services[0].Name) execError := compose. WithCommand(command). WithEnv(env). @@ -225,7 +250,7 @@ func executeCompose(isProfile bool, composeNames []string, command []string, env "cmd": command, "composeFilePaths": composeFilePaths, "env": env, - "profile": composeNames[0], + "profile": services[0].Name, }).Debug("Docker compose executed.") return nil diff --git a/internal/deploy/docker.go b/internal/deploy/docker.go new file mode 100644 index 0000000000..27faf4860b --- /dev/null +++ b/internal/deploy/docker.go @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "context" + "strings" + + "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" +) + +// DockerDeploymentManifest deploy manifest for docker +type dockerDeploymentManifest struct { + Context context.Context +} + +func newDockerDeploy() Deployment { + return &dockerDeploymentManifest{Context: context.Background()} +} + +// Add adds services deployment +func (c *dockerDeploymentManifest) Add(services []ServiceRequest, env map[string]string) error { + serviceManager := NewServiceManager() + + return serviceManager.AddServicesToCompose(c.Context, services[0], services[1:], env) +} + +// Bootstrap sets up environment with docker compose +func (c *dockerDeploymentManifest) Bootstrap(waitCB func() error) error { + serviceManager := NewServiceManager() + common.ProfileEnv = map[string]string{ + "kibanaVersion": common.KibanaVersion, + "stackVersion": common.StackVersion, + } + + common.ProfileEnv["kibanaDockerNamespace"] = "kibana" + if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) { + // because it comes from a PR + common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci" + } + + profile := NewServiceRequest(common.FleetProfileName) + err := serviceManager.RunCompose(c.Context, true, []ServiceRequest{profile}, common.ProfileEnv) + if err != nil { + log.WithFields(log.Fields{ + "profile": profile, + "error": err.Error(), + }).Fatal("Could not run the runtime dependencies for the profile.") + } + err = waitCB() + if err != nil { + return err + } + return nil +} + +// Destroy teardown docker environment +func (c *dockerDeploymentManifest) Destroy() error { + serviceManager := NewServiceManager() + err := serviceManager.StopCompose(c.Context, true, []ServiceRequest{NewServiceRequest(common.FleetProfileName)}) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "profile": common.FleetProfileName, + }).Fatal("Could not destroy the runtime dependencies for the profile.") + } + return nil +} + +// ExecIn execute command in service +func (c *dockerDeploymentManifest) ExecIn(service ServiceRequest, cmd []string) (string, error) { + output, err := ExecCommandIntoContainer(c.Context, service, "root", cmd) + if err != nil { + return "", err + } + return output, nil +} + +// Inspect inspects a service +func (c *dockerDeploymentManifest) Inspect(service ServiceRequest) (*ServiceManifest, error) { + inspect, err := InspectContainer(service) + if err != nil { + return &ServiceManifest{}, err + } + return &ServiceManifest{ + ID: inspect.ID, + Name: strings.TrimPrefix(inspect.Name, "/"), + Connection: service.Name, + Hostname: inspect.NetworkSettings.Networks["fleet_default"].Aliases[0], + }, nil +} + +// Remove remove services from deployment +func (c *dockerDeploymentManifest) Remove(services []ServiceRequest, env map[string]string) error { + serviceManager := NewServiceManager() + + return serviceManager.RemoveServicesFromCompose(c.Context, services[0], services[1:], env) +} diff --git a/internal/docker/docker.go b/internal/deploy/docker_client.go similarity index 61% rename from internal/docker/docker.go rename to internal/deploy/docker_client.go index fc63fd30a8..88704adc2d 100644 --- a/internal/docker/docker.go +++ b/internal/deploy/docker_client.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package docker +package deploy import ( "archive/tar" @@ -23,7 +23,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" ) @@ -68,36 +68,6 @@ func buildTarForDeployment(file *os.File) (bytes.Buffer, error) { return buffer, nil } -// CheckProcessStateOnTheHost checks if a process is in the desired state in a container -// name of the container for the service: -// we are using the Docker client instead of docker-compose -// because it does not support returning the output of a -// command: it simply returns error level -func CheckProcessStateOnTheHost(containerName string, process string, state string, occurrences int, timeoutFactor int) error { - timeout := time.Duration(common.TimeoutFactor) * time.Minute - - err := WaitForProcess(containerName, process, state, occurrences, timeout) - if err != nil { - if state == "started" { - log.WithFields(log.Fields{ - "container ": containerName, - "error": err, - "timeout": timeout, - }).Error("The process was not found but should be present") - } else { - log.WithFields(log.Fields{ - "container": containerName, - "error": err, - "timeout": timeout, - }).Error("The process was found but shouldn't be present") - } - - return err - } - - return nil -} - // CopyFileToContainer copies a file to the running container func CopyFileToContainer(ctx context.Context, containerName string, srcPath string, parentDir string, isTar bool) error { dockerClient := getDockerClient() @@ -165,17 +135,19 @@ func CopyFileToContainer(ctx context.Context, containerName string, srcPath stri } // ExecCommandIntoContainer executes a command, as a user, into a container -func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) { - return ExecCommandIntoContainerWithEnv(ctx, containerName, user, cmd, []string{}) +func ExecCommandIntoContainer(ctx context.Context, container ServiceRequest, user string, cmd []string) (string, error) { + return ExecCommandIntoContainerWithEnv(ctx, container, user, cmd, []string{}) } // ExecCommandIntoContainerWithEnv executes a command, as a user, with env, into a container -func ExecCommandIntoContainerWithEnv(ctx context.Context, containerName string, user string, cmd []string, env []string) (string, error) { +func ExecCommandIntoContainerWithEnv(ctx context.Context, container ServiceRequest, user string, cmd []string, env []string) (string, error) { dockerClient := getDockerClient() detach := false tty := false + containerName := container.Name + log.WithFields(log.Fields{ "container": containerName, "command": cmd, @@ -296,7 +268,7 @@ func GetContainerHostname(containerName string) (string, error) { "containerName": containerName, }).Trace("Retrieving container name from the Docker client") - hostname, err := ExecCommandIntoContainer(context.Background(), containerName, "root", []string{"cat", "/etc/hostname"}) + hostname, err := ExecCommandIntoContainer(context.Background(), NewServiceRequest(containerName), "root", []string{"cat", "/etc/hostname"}) if err != nil { log.WithFields(log.Fields{ "containerName": containerName, @@ -315,14 +287,13 @@ func GetContainerHostname(containerName string) (string, error) { // InspectContainer returns the JSON representation of the inspection of a // Docker container, identified by its name -func InspectContainer(name string) (*types.ContainerJSON, error) { +func InspectContainer(service ServiceRequest) (*types.ContainerJSON, error) { dockerClient := getDockerClient() ctx := context.Background() labelFilters := filters.NewArgs() - labelFilters.Add("label", "service.owner=co.elastic.observability") - labelFilters.Add("label", "service.container.name="+name) + labelFilters.Add("name", service.Name) containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true, Filters: labelFilters}) if err != nil { @@ -340,6 +311,18 @@ func InspectContainer(name string) (*types.ContainerJSON, error) { return &inspect, nil } +// ListContainers returns a list of running containers +func ListContainers() ([]types.Container, error) { + dockerClient := getDockerClient() + ctx := context.Background() + + containers, err := dockerClient.ContainerList(ctx, types.ContainerListOptions{}) + if err != nil { + return []types.Container{}, err + } + return containers, nil +} + // RemoveContainer removes a container identified by its container name func RemoveContainer(containerName string) error { dockerClient := getDockerClient() @@ -404,7 +387,7 @@ func TagImage(src string, target string) error { dockerClient := getDockerClient() maxTimeout := 15 * time.Second - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 0 tagImageFn := func() error { @@ -455,171 +438,6 @@ func RemoveDevNetwork() error { return nil } -// WaitForProcess polls a container executing "ps" command until the process is in the desired state (present or not), -// or a timeout happens -func WaitForProcess(containerName string, process string, desiredState string, ocurrences int, maxTimeout time.Duration) error { - exp := common.GetExponentialBackOff(maxTimeout) - - mustBePresent := false - if desiredState == "started" { - mustBePresent = true - } - retryCount := 1 - - processStatus := func() error { - log.WithFields(log.Fields{ - "desiredState": desiredState, - "ocurrences": ocurrences, - "process": process, - }).Trace("Checking process desired state on the container") - - // pgrep -d: -d, --delimiter specify output delimiter - //i.e. "pgrep -d , metricbeat": 483,519 - cmds := []string{"pgrep", "-d", ",", process} - output, err := ExecCommandIntoContainer(context.Background(), containerName, "root", cmds) - if err != nil { - log.WithFields(log.Fields{ - "cmds": cmds, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": ocurrences, - "process": process, - "retry": retryCount, - }).Warn("Could not get number of processes in the container") - - retryCount++ - - return err - } - - // tokenize the pids to get each pid's state, adding them to an array if they match the desired state - // From Split docs: - // If output does not contain sep and sep is not empty, Split returns a - // slice of length 1 whose only element is s, that's why we first initialise to the empty array - pids := strings.Split(output, ",") - if len(pids) == 1 && pids[0] == "" { - pids = []string{} - } - - log.WithFields(log.Fields{ - "count": len(pids), - "desiredState": desiredState, - "mustBePresent": mustBePresent, - "pids": pids, - "process": process, - }).Tracef("Pids for process found") - - desiredStatePids := []string{} - - for _, pid := range pids { - pidStateCmds := []string{"ps", "-q", pid, "-o", "state", "--no-headers"} - pidState, err := ExecCommandIntoContainer(context.Background(), containerName, "root", pidStateCmds) - if err != nil { - log.WithFields(log.Fields{ - "cmds": cmds, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": ocurrences, - "pid": pid, - "process": process, - "retry": retryCount, - }).Warn("Could not check pid status in the container") - - retryCount++ - - return err - } - - log.WithFields(log.Fields{ - "desiredState": desiredState, - "mustBePresent": mustBePresent, - "pid": pid, - "pidState": pidState, - "process": process, - }).Tracef("Checking if process is in the S state") - - // if the process must be present, then check for the S state - // From 'man ps': - // D uninterruptible sleep (usually IO) - // R running or runnable (on run queue) - // S interruptible sleep (waiting for an event to complete) - // T stopped by job control signal - // t stopped by debugger during the tracing - // W paging (not valid since the 2.6.xx kernel) - // X dead (should never be seen) - // Z defunct ("zombie") process, terminated but not reaped by its parent - if mustBePresent && pidState == "S" { - desiredStatePids = append(desiredStatePids, pid) - } else if !mustBePresent { - desiredStatePids = append(desiredStatePids, pid) - } - } - - occurrencesMatched := (len(desiredStatePids) == ocurrences) - - // both true or both false - if mustBePresent == occurrencesMatched { - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "desiredState": desiredState, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": len(desiredStatePids), - "process": process, - }).Infof("Process desired state checked") - - return nil - } - - if mustBePresent { - err = fmt.Errorf("%s process is not running in the container with the desired number of occurrences (%d) yet", process, ocurrences) - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "ocurrences": len(desiredStatePids), - "process": process, - "retry": retryCount, - }).Warn(err.Error()) - - retryCount++ - - return err - } - - err = fmt.Errorf("%s process is still running in the container", process) - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "ocurrences": len(desiredStatePids), - "process": process, - "state": desiredState, - "retry": retryCount, - }).Warn(err.Error()) - - retryCount++ - - return err - } - - err := backoff.Retry(processStatus, exp) - if err != nil { - return err - } - - return nil -} - func getDockerClient() *client.Client { if instance != nil { return instance diff --git a/internal/docker/docker_test.go b/internal/deploy/docker_client_test.go similarity index 89% rename from internal/docker/docker_test.go rename to internal/deploy/docker_client_test.go index 2451a8309e..f44d420244 100644 --- a/internal/docker/docker_test.go +++ b/internal/deploy/docker_client_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package docker +package deploy import ( "context" @@ -40,7 +40,7 @@ func Test_CopyFile(t *testing.T) { err = CopyFileToContainer(ctx, containerName, src, target, false) assert.Nil(t, err) - output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"cat", "/tmp/dockerCopy.txt"}) + output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"cat", "/tmp/dockerCopy.txt"}) assert.Nil(t, err) assert.True(t, strings.HasSuffix(output, "OK!"), "File contains the 'OK!' string") }) @@ -76,7 +76,7 @@ func Test_CopyFile(t *testing.T) { err = CopyFileToContainer(ctx, containerName, src, target, true) assert.Nil(t, err) - output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"ls", "/project/txtr/kermit.jpg"}) + output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"ls", "/project/txtr/kermit.jpg"}) assert.Nil(t, err) assert.True(t, strings.Contains(output, "/project/txtr/kermit.jpg"), "File '/project/txtr/kermit.jpg' should be present") }) diff --git a/internal/deploy/kubernetes.go b/internal/deploy/kubernetes.go new file mode 100644 index 0000000000..2b13196712 --- /dev/null +++ b/internal/deploy/kubernetes.go @@ -0,0 +1,119 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/elastic/e2e-testing/internal/kubernetes" + "github.com/pkg/errors" +) + +var cluster kubernetes.Cluster +var kubectl kubernetes.Control + +// KubernetesDeploymentManifest deploy manifest for kubernetes +type kubernetesDeploymentManifest struct { + Context context.Context +} + +func newK8sDeploy() Deployment { + return &kubernetesDeploymentManifest{Context: context.Background()} +} + +// Add adds services deployment +func (c *kubernetesDeploymentManifest) Add(services []ServiceRequest, env map[string]string) error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + + for _, service := range services { + _, err := kubectl.Run(c.Context, "apply", "-k", fmt.Sprintf("../../../cli/config/kubernetes/overlays/%s", service.Name)) + if err != nil { + return err + } + } + return nil +} + +// Bootstrap sets up environment with kind +func (c *kubernetesDeploymentManifest) Bootstrap(waitCB func() error) error { + err := cluster.Initialize(c.Context, "../../../cli/config/kubernetes/kind.yaml") + if err != nil { + return err + } + + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + _, err = kubectl.Run(c.Context, "apply", "-k", "../../../cli/config/kubernetes/base") + if err != nil { + return err + } + err = waitCB() + if err != nil { + return err + } + return nil +} + +// Destroy teardown kubernetes environment +func (c *kubernetesDeploymentManifest) Destroy() error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + cluster.Cleanup(c.Context) + return nil +} + +// ExecIn execute command in service +func (c *kubernetesDeploymentManifest) ExecIn(service ServiceRequest, cmd []string) (string, error) { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + args := []string{"exec", "deployment/" + service.Name, "--"} + for _, arg := range cmd { + args = append(cmd, arg) + } + output, err := kubectl.Run(c.Context, args...) + if err != nil { + return "", err + } + return output, nil +} + +type kubernetesServiceManifest struct { + Metadata struct { + Name string `json:"name"` + ID string `json:"uid"` + } `json:"metadata"` +} + +// Inspect inspects a service +func (c *kubernetesDeploymentManifest) Inspect(service ServiceRequest) (*ServiceManifest, error) { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + out, err := kubectl.Run(c.Context, "get", "deployment/"+service.Name, "-o", "json") + if err != nil { + return &ServiceManifest{}, err + } + var inspect kubernetesServiceManifest + if err = json.Unmarshal([]byte(out), &inspect); err != nil { + return &ServiceManifest{}, errors.Wrap(err, "Could not convert metadata to JSON") + } + return &ServiceManifest{ + ID: inspect.Metadata.ID, + Name: strings.TrimPrefix(inspect.Metadata.Name, "/"), + Connection: service.Name, + Hostname: service.Name, + }, nil +} + +// Remove remove services from deployment +func (c *kubernetesDeploymentManifest) Remove(services []ServiceRequest, env map[string]string) error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + + for _, service := range services { + _, err := kubectl.Run(c.Context, "delete", "-k", fmt.Sprintf("../../../cli/config/kubernetes/overlays/%s", service.Name)) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go index c8b32e48f8..4de42d8a8e 100644 --- a/internal/elasticsearch/client.go +++ b/internal/elasticsearch/client.go @@ -13,9 +13,9 @@ import ( "time" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" es "github.com/elastic/go-elasticsearch/v8" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -202,7 +202,7 @@ func WaitForElasticsearch(ctx context.Context, maxTimeoutMinutes time.Duration) // WaitForElasticsearchFromHostPort waits for an elasticsearch running in a host:port to be healthy, returning false // if elasticsearch does not get healthy status in a defined number of minutes. func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int, maxTimeoutMinutes time.Duration) (bool, error) { - exp := common.GetExponentialBackOff(maxTimeoutMinutes) + exp := utils.GetExponentialBackOff(maxTimeoutMinutes) retryCount := 1 @@ -251,7 +251,7 @@ func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int // WaitForIndices waits for the elasticsearch indices to return the list of indices. func WaitForIndices() (string, error) { - exp := common.GetExponentialBackOff(60 * time.Second) + exp := utils.GetExponentialBackOff(60 * time.Second) retryCount := 1 body := "" @@ -294,7 +294,7 @@ func WaitForIndices() (string, error) { // WaitForNumberOfHits waits for an elasticsearch query to return more than a number of hits, // returning false if the query does not reach that number in a defined number of time. func WaitForNumberOfHits(ctx context.Context, indexName string, query map[string]interface{}, desiredHits int, maxTimeout time.Duration) (SearchResult, error) { - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 result := SearchResult{} diff --git a/internal/installer/base.go b/internal/installer/base.go index 1178c46909..f65a11ae77 100644 --- a/internal/installer/base.go +++ b/internal/installer/base.go @@ -10,8 +10,7 @@ import ( "strings" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" log "github.com/sirupsen/logrus" ) @@ -38,8 +37,11 @@ type BasePackage struct { // extractPackage depends on the underlying OS, so 'cmds' must contain the specific instructions for the OS func (i *BasePackage) extractPackage(cmds []string) error { - sm := compose.NewServiceManager() - err := sm.ExecCommandInService(i.profile, i.image, i.service, cmds, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err := sm.ExecCommandInService( + deploy.NewServiceRequest(i.profile), imageService, i.service, cmds, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ "command": cmds, @@ -56,11 +58,14 @@ func (i *BasePackage) extractPackage(cmds []string) error { // Postinstall executes operations after installing a DEB package func (i *BasePackage) Postinstall() error { - err := SystemctlRun(i.profile, i.image, i.service, "enable") + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err := SystemctlRun(profileService, imageService, i.service, "enable") if err != nil { return err } - return SystemctlRun(i.profile, i.image, i.service, "start") + return SystemctlRun(profileService, imageService, i.service, "start") } // PrintLogs prints logs for the agent @@ -74,8 +79,11 @@ func (i *BasePackage) PrintLogs(containerName string) error { "cat", i.logFile, } - sm := compose.NewServiceManager() - err = sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err = sm.ExecCommandInService( + deploy.NewServiceRequest(i.profile), imageService, i.service, cmd, common.ProfileEnv, false) if err != nil { return err } @@ -110,7 +118,7 @@ func getElasticAgentHash(containerName string, commitFile string) (string, error "cat", commitFile, } - fullHash, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + fullHash, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { return "", err } @@ -129,9 +137,9 @@ func getElasticAgentHash(containerName string, commitFile string) (string, error } // SystemctlRun runs systemctl in profile or service -func SystemctlRun(profile string, image string, service string, command string) error { +func SystemctlRun(profile deploy.ServiceRequest, image deploy.ServiceRequest, service string, command string) error { cmd := []string{"systemctl", command, common.ElasticAgentProcessName} - sm := compose.NewServiceManager() + sm := deploy.NewServiceManager() err := sm.ExecCommandInService(profile, image, service, cmd, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/deb.go b/internal/installer/deb.go index 5ec4c07c2b..08c15b9f8f 100644 --- a/internal/installer/deb.go +++ b/internal/installer/deb.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -40,14 +40,17 @@ func (i *DEBPackage) InstallCerts() error { return installCertsForDebian(i.profile, i.image, i.service) } func installCertsForDebian(profile string, image string, service string) error { - sm := compose.NewServiceManager() - if err := sm.ExecCommandInService(profile, image, service, []string{"apt-get", "update"}, common.ProfileEnv, false); err != nil { + sm := deploy.NewServiceManager() + serviceProfile := deploy.NewServiceRequest(profile) + serviceImage := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(image) + + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"apt-get", "update"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"apt", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-certificates", "-f"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-certificates", "-f"}, common.ProfileEnv, false); err != nil { return err } return nil @@ -67,8 +70,7 @@ func (i *DEBPackage) Uninstall() error { // newDebianInstaller returns an instance of the Debian installer for a specific version func newDebianInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { - image = image + "-systemd" // we want to consume systemd boxes - service := image + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -77,7 +79,7 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI arch := "amd64" extension := "deb" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -91,8 +93,11 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour("debian") + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, image, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } workingDir := "/var/lib/elastic-agent" diff --git a/internal/installer/docker.go b/internal/installer/docker.go index 1877dde9e6..bd6cd1162e 100644 --- a/internal/installer/docker.go +++ b/internal/installer/docker.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -55,14 +55,14 @@ func (i *DockerPackage) InstallCerts() error { // Preinstall executes operations before installing a Docker package func (i *DockerPackage) Preinstall() error { - err := docker.LoadImage(i.installerPath) + err := deploy.LoadImage(i.installerPath) if err != nil { return err } // we need to tag the loaded image because its tag relates to the target branch - return docker.TagImage( - "docker.elastic.co/beats/"+i.artifact+":"+common.AgentVersionBase, + return deploy.TagImage( + "docker.elastic.co/beats/"+i.artifact+":"+common.BeatVersionBase, "docker.elastic.co/observability-ci/"+i.artifact+":"+i.originalVersion+"-amd64", ) } @@ -99,14 +99,14 @@ func (i *DockerPackage) WithOS(OS string) *DockerPackage { // WithVersion sets the version func (i *DockerPackage) WithVersion(version string) *DockerPackage { - i.version = utils.CheckPRVersion(version, common.AgentVersionBase) // sanitize version + i.version = utils.CheckPRVersion(version, common.BeatVersionBase) // sanitize version i.originalVersion = version return i } // newDockerInstaller returns an instance of the Docker installer func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error) { - image := "elastic-agent" + image := common.ElasticAgentServiceName service := image profile := common.FleetProfileName @@ -123,7 +123,7 @@ func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error arch := "amd64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifactName, version, common.AgentVersionBase, os, arch, extension, true) + binaryName := utils.BuildArtifactName(artifactName, version, common.BeatVersionBase, os, arch, extension, true) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/elasticagent.go b/internal/installer/elasticagent.go index ec32ea4f6b..258e9fb778 100644 --- a/internal/installer/elasticagent.go +++ b/internal/installer/elasticagent.go @@ -9,8 +9,7 @@ import ( "fmt" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -47,7 +46,7 @@ func (i *ElasticAgentInstaller) ListElasticAgentWorkingDirContent(containerName "ls", "-l", i.workingDir, } - content, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + content, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { return "", err } @@ -62,9 +61,9 @@ func (i *ElasticAgentInstaller) ListElasticAgentWorkingDirContent(containerName } // runElasticAgentCommandEnv runs a command for the elastic-agent -func runElasticAgentCommandEnv(profile string, image string, service string, process string, command string, arguments []string, env map[string]string) error { +func runElasticAgentCommandEnv(profile deploy.ServiceRequest, image deploy.ServiceRequest, service string, process string, command string, arguments []string, env map[string]string) error { cmds := []string{ - "timeout", fmt.Sprintf("%dm", common.TimeoutFactor), process, command, + "timeout", fmt.Sprintf("%dm", utils.TimeoutFactor), process, command, } cmds = append(cmds, arguments...) @@ -72,7 +71,7 @@ func runElasticAgentCommandEnv(profile string, image string, service string, pro common.ProfileEnv[k] = v } - sm := compose.NewServiceManager() + sm := deploy.NewServiceManager() err := sm.ExecCommandInService(profile, image, service, cmds, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ @@ -97,7 +96,7 @@ func runElasticAgentCommandEnv(profile string, image string, service string, pro // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact // to be downloaded will be defined by the latest snapshot produced by the Beats CI. func downloadAgentBinary(artifactName string, artifact string, version string) (string, error) { - imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.AgentVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return "", err } @@ -106,23 +105,22 @@ func downloadAgentBinary(artifactName string, artifact string, version string) ( } // GetElasticAgentInstaller returns an installer from a docker image -func GetElasticAgentInstaller(image string, installerType string, version string, fleetServerHost string) ElasticAgentInstaller { +func GetElasticAgentInstaller(image string, installerType string, version string, index int) ElasticAgentInstaller { log.WithFields(log.Fields{ - "fleetServerHost": fleetServerHost, - "image": image, - "installer": installerType, - "version": version, + "image": image, + "installer": installerType, + "version": version, }).Debug("Configuring installer for the agent") var installer ElasticAgentInstaller var err error if "centos" == image && "tar" == installerType { - installer, err = newTarInstaller("centos", "latest", version, fleetServerHost) - } else if "centos" == image && "systemd" == installerType { + installer, err = newTarInstaller("centos", "latest", version, index) + } else if "centos" == image && "rpm" == installerType { installer, err = newCentosInstaller("centos", "latest", version) } else if "debian" == image && "tar" == installerType { - installer, err = newTarInstaller("debian", "stretch", version, fleetServerHost) - } else if "debian" == image && "systemd" == installerType { + installer, err = newTarInstaller("debian", "stretch", version, index) + } else if "debian" == image && "deb" == installerType { installer, err = newDebianInstaller("debian", "stretch", version) } else if "docker" == image && "default" == installerType { installer, err = newDockerInstaller(false, version) diff --git a/internal/installer/rpm.go b/internal/installer/rpm.go index acc7c554fb..dc320db349 100644 --- a/internal/installer/rpm.go +++ b/internal/installer/rpm.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -41,17 +41,20 @@ func (i *RPMPackage) InstallCerts() error { return installCertsForCentos(i.profile, i.image, i.service) } func installCertsForCentos(profile string, image string, service string) error { - sm := compose.NewServiceManager() - if err := sm.ExecCommandInService(profile, image, service, []string{"yum", "check-update"}, common.ProfileEnv, false); err != nil { + sm := deploy.NewServiceManager() + serviceProfile := deploy.NewServiceRequest(profile) + serviceImage := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(image) + + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"yum", "check-update"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"yum", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"yum", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-trust", "force-enable"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-trust", "force-enable"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-trust", "extract"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-trust", "extract"}, common.ProfileEnv, false); err != nil { return err } return nil @@ -71,8 +74,7 @@ func (i *RPMPackage) Uninstall() error { // newCentosInstaller returns an instance of the Centos installer for a specific version func newCentosInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { - image = image + "-systemd" // we want to consume systemd boxes - service := image + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -81,7 +83,7 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI arch := "x86_64" extension := "rpm" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -95,8 +97,11 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour("centos") + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, image, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } workingDir := "/var/lib/elastic-agent" diff --git a/internal/installer/tar.go b/internal/installer/tar.go index 759679120f..b853832b4e 100644 --- a/internal/installer/tar.go +++ b/internal/installer/tar.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -20,6 +20,7 @@ type TARPackage struct { // optional fields arch string artifact string + index int OS string OSFlavour string // at this moment, centos or debian version string @@ -45,7 +46,10 @@ func (i *TARPackage) Install(cfg *kibana.FleetConfig) error { binary := fmt.Sprintf("/elastic-agent/%s", i.artifact) args := cfg.Flags() - err := runElasticAgentCommandEnv(i.profile, i.image, i.service, binary, "install", args, map[string]string{}) + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + + err := runElasticAgentCommandEnv(profileService, imageService, i.service, binary, "install", args, map[string]string{}) if err != nil { return fmt.Errorf("Failed to install the agent with subcommand: %v", err) } @@ -83,9 +87,13 @@ func (i *TARPackage) Preinstall() error { {"rm", "-fr", "/elastic-agent"}, {"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"}, } + + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + for _, cmd := range cmds { - sm := compose.NewServiceManager() - err := sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + err := sm.ExecCommandInService(profileService, imageService, i.service, cmd, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ "command": cmd, @@ -106,7 +114,10 @@ func (i *TARPackage) Preinstall() error { func (i *TARPackage) Uninstall() error { args := []string{"-f"} - return runElasticAgentCommandEnv(i.profile, i.image, i.service, common.ElasticAgentProcessName, "uninstall", args, map[string]string{}) + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + + return runElasticAgentCommandEnv(profileService, imageService, i.service, common.ElasticAgentProcessName, "uninstall", args, map[string]string{}) } // WithArch sets the architecture @@ -121,6 +132,12 @@ func (i *TARPackage) WithArtifact(artifact string) *TARPackage { return i } +// WithIndex sets the index of the agent +func (i *TARPackage) WithIndex(index int) *TARPackage { + i.index = index + return i +} + // WithOS sets the OS func (i *TARPackage) WithOS(OS string) *TARPackage { i.OS = OS @@ -140,13 +157,8 @@ func (i *TARPackage) WithVersion(version string) *TARPackage { } // newTarInstaller returns an instance of the Debian installer for a specific version -func newTarInstaller(image string, tag string, version string, fleetServerHost string) (ElasticAgentInstaller, error) { - dockerImage := image + "-systemd" // we want to consume systemd boxes - if fleetServerHost == "" { - dockerImage = "fleet-server-" + image - } - - service := dockerImage +func newTarInstaller(image string, tag string, version string, index int) (ElasticAgentInstaller, error) { + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -155,7 +167,7 @@ func newTarInstaller(image string, tag string, version string, fleetServerHost s arch := "x86_64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -177,17 +189,21 @@ func newTarInstaller(image string, tag string, version string, fleetServerHost s logFileName := "elastic-agent-json.log" logFile := logsDir + "/" + logFileName + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour(image) + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, dockerImage, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } // - installerPackage := NewTARPackage(binaryName, profile, dockerImage, service, commitFile, logFile). + installerPackage := NewTARPackage(binaryName, profile, image, service, commitFile, logFile). WithArch(arch). WithArtifact(artifact). + WithIndex(index). WithOS(os). WithOSFlavour(image). - WithVersion(utils.CheckPRVersion(version, common.AgentVersionBase)) // sanitize version + WithVersion(utils.CheckPRVersion(version, common.BeatVersionBase)) // sanitize version return ElasticAgentInstaller{ artifactArch: arch, @@ -197,7 +213,7 @@ func newTarInstaller(image string, tag string, version string, fleetServerHost s artifactVersion: version, BinaryPath: binaryPath, EnrollFn: enrollFn, - Image: dockerImage, + Image: image, InstallFn: installerPackage.Install, InstallCertsFn: installerPackage.InstallCerts, InstallerType: "tar", diff --git a/internal/kibana/fleet.go b/internal/kibana/fleet.go index 01b8d7d13a..64ec8dbbea 100644 --- a/internal/kibana/fleet.go +++ b/internal/kibana/fleet.go @@ -7,7 +7,6 @@ package kibana import ( "fmt" - "github.com/elastic/e2e-testing/internal/common" log "github.com/sirupsen/logrus" ) @@ -21,21 +20,11 @@ type FleetConfig struct { KibanaURI string FleetServerPort int FleetServerURI string - // server - BootstrapFleetServer bool - ServerPolicyID string } -// NewFleetConfig builds a new configuration for the fleet agent, defaulting ES credentials, URI and port. -// If the 'fleetServerHost' flag is empty, then it will create the config for the initial fleet server -// used to bootstrap Fleet Server -// If the 'fleetServerHost' flag is not empty, the it will create the config for an agent using an existing Fleet -// Server to connect to Fleet. It will also retrieve the default policy ID for fleet server -func NewFleetConfig(token string, fleetServerHost string) (*FleetConfig, error) { - bootstrapFleetServer := (fleetServerHost == "") - +// NewFleetConfig builds a new configuration for the fleet agent, defaulting fleet-server host, ES credentials, URI and port. +func NewFleetConfig(token string) (*FleetConfig, error) { cfg := &FleetConfig{ - BootstrapFleetServer: bootstrapFleetServer, EnrollmentToken: token, ElasticsearchCredentials: "elastic:changeme", ElasticsearchPort: 9200, @@ -43,43 +32,20 @@ func NewFleetConfig(token string, fleetServerHost string) (*FleetConfig, error) KibanaPort: 5601, KibanaURI: "kibana", FleetServerPort: 8220, - FleetServerURI: fleetServerHost, - } - - client, err := NewClient() - if err != nil { - return cfg, err + FleetServerURI: "fleet-server", } - if !bootstrapFleetServer { - defaultFleetServerPolicy, err := client.GetDefaultPolicy(true) - if err != nil { - return nil, err - } - - cfg.ServerPolicyID = defaultFleetServerPolicy.ID - - log.WithFields(log.Fields{ - "elasticsearch": cfg.ElasticsearchURI, - "elasticsearchPort": cfg.ElasticsearchPort, - "policyID": cfg.ServerPolicyID, - "token": cfg.EnrollmentToken, - }).Debug("Fleet Server config created") - } + log.WithFields(log.Fields{ + "elasticsearch": cfg.ElasticsearchURI, + "elasticsearchPort": cfg.ElasticsearchPort, + "token": cfg.EnrollmentToken, + }).Debug("Fleet Server config created") return cfg, nil } // Flags bootstrap flags for fleet server func (cfg FleetConfig) Flags() []string { - if cfg.BootstrapFleetServer { - // TO-DO: remove all code to calculate the fleet-server policy, because it's inferred by the fleet-server - return []string{ - "--force", - "--fleet-server-es", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), - } - } - /* // agent using an already bootstrapped fleet-server fleetServerHost := "https://hostname_of_the_bootstrapped_fleet_server:8220" @@ -91,14 +57,10 @@ func (cfg FleetConfig) Flags() []string { } */ - baseFlags := []string{"-e", "-v", "--force", "--insecure", "--enrollment-token=" + cfg.EnrollmentToken} - if common.AgentVersionBase == "8.0.0-SNAPSHOT" { - return append(baseFlags, "--url", fmt.Sprintf("https://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.FleetServerURI, cfg.FleetServerPort)) - } - - if cfg.ServerPolicyID != "" { - baseFlags = append(baseFlags, "--fleet-server-insecure-http", "--fleet-server", fmt.Sprintf("https://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), "--fleet-server-host=http://0.0.0.0", "--fleet-server-policy", cfg.ServerPolicyID) + flags := []string{ + "-e", "-v", "--force", "--insecure", "--enrollment-token=" + cfg.EnrollmentToken, + "--url", fmt.Sprintf("http://%s:%d", cfg.FleetServerURI, cfg.FleetServerPort), } - return append(baseFlags, "--kibana-url", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.KibanaURI, cfg.KibanaPort)) + return flags } diff --git a/internal/kibana/policies.go b/internal/kibana/policies.go index 5667a499ae..d96baaf533 100644 --- a/internal/kibana/policies.go +++ b/internal/kibana/policies.go @@ -76,8 +76,8 @@ func (c *Client) ListPolicies() ([]Policy, error) { return resp.Items, nil } -// DeleteAllPolicies deletes all policies -func (c *Client) DeleteAllPolicies(except Policy) { +// DeleteAllPolicies deletes all policies except fleet_server and system +func (c *Client) DeleteAllPolicies() { // Cleanup all package policies packagePolicies, err := c.ListPackagePolicies() if err != nil { @@ -87,7 +87,8 @@ func (c *Client) DeleteAllPolicies(except Policy) { } for _, pkgPolicy := range packagePolicies { // Do not remove the fleet server package integration otherwise fleet server fails to bootstrap - if !strings.Contains(pkgPolicy.Name, "fleet_server") && pkgPolicy.PolicyID == except.ID { + if !strings.Contains(pkgPolicy.Name, "fleet_server") && !strings.Contains(pkgPolicy.Name, "system") { + log.WithField("pkgPolicy", pkgPolicy.Name).Trace("Removing package policy") err = c.DeleteIntegrationFromPolicy(pkgPolicy) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/kibana/server.go b/internal/kibana/server.go index cd37d7ae92..0ff3ae8f91 100644 --- a/internal/kibana/server.go +++ b/internal/kibana/server.go @@ -12,7 +12,7 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -193,8 +193,8 @@ func (c *Client) RecreateFleet() error { }).Info("Fleet setup done") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -238,14 +238,14 @@ func (c *Client) WaitForFleet() error { "body": jsonResponse, "error": err, "statusCode": statusCode, - }).Error("Kibana has not been initialized") - return errors.New("Kibana has not been initialized") + }).Warn("Fleet is not ready") + return errors.New("Fleet is not ready") } - log.Info("Kibana setup initialized") + log.Info("Fleet setup complete") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -257,8 +257,8 @@ func (c *Client) WaitForFleet() error { // WaitForReady waits for Kibana to be healthy and accept connections func (c *Client) WaitForReady(maxTimeoutMinutes time.Duration) (bool, error) { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) ctx := context.Background() diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index a050d6af05..dbbf5fa65c 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -17,8 +17,8 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" - "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" ) // Control struct for k8s cluster @@ -67,7 +67,7 @@ func (c Control) createNamespace(ctx context.Context, namespace string) error { // Wait for default account to be available, if not it is not possible to // deploy pods in this namespace. timeout := 60 * time.Second - exp := backoff.WithContext(common.GetExponentialBackOff(timeout), ctx) + exp := backoff.WithContext(utils.GetExponentialBackOff(timeout), ctx) return backoff.Retry(func() error { _, err := c.Run(ctx, "get", "serviceaccount", "default") if err != nil { diff --git a/internal/common/retry.go b/internal/utils/retry.go similarity index 87% rename from internal/common/retry.go rename to internal/utils/retry.go index fa38cb95a1..c1ce828388 100644 --- a/internal/common/retry.go +++ b/internal/utils/retry.go @@ -2,18 +2,23 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package common +package utils import ( "time" backoff "github.com/cenkalti/backoff/v4" + "github.com/elastic/e2e-testing/internal/shell" ) // TimeoutFactor a multiplier for the max timeout when doing backoff retries. // It can be overriden by TIMEOUT_FACTOR env var var TimeoutFactor = 3 +func init() { + TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", TimeoutFactor) +} + // GetExponentialBackOff returns a preconfigured exponential backoff instance func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOff { var ( diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 3059381535..ce942d2ac7 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -18,7 +18,6 @@ import ( "github.com/Jeffail/gabs/v2" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" internalio "github.com/elastic/e2e-testing/internal/io" "github.com/elastic/e2e-testing/internal/shell" @@ -26,6 +25,11 @@ import ( log "github.com/sirupsen/logrus" ) +// to avoid fetching the same Elastic artifacts version, we are adding this map to cache the version of the Elastic artifacts, +// using as key the URL of the version. If another request is trying to fetch the same URL, it will return the string version +// of the already requested one. +var elasticVersionsCache = map[string]string{} + // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. @@ -187,7 +191,17 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, f // If the version is a PR, then it will return the version without checking the artifacts API // i.e. GetElasticArtifactVersion("$VERSION") func GetElasticArtifactVersion(version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + cacheKey := fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version) + + if val, ok := elasticVersionsCache[cacheKey]; ok { + log.WithFields(log.Fields{ + "URL": cacheKey, + "version": val, + }).Debug("Retrieving version from local cache") + return val, nil + } + + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -195,7 +209,7 @@ func GetElasticArtifactVersion(version string) (string, error) { apiStatus := func() error { r := curl.HTTPRequest{ - URL: fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version), + URL: cacheKey, } response, err := curl.Get(r) @@ -247,6 +261,8 @@ func GetElasticArtifactVersion(version string) (string, error) { "version": latestVersion, }).Debug("Latest version for current version obtained") + elasticVersionsCache[cacheKey] = latestVersion + return latestVersion, nil } @@ -256,7 +272,7 @@ func GetElasticArtifactVersion(version string) (string, error) { // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-amd64.tar.gz", "elastic-agent","$VERSION") func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -320,7 +336,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) // GetObjectURLFromBucket extracts the media URL for the desired artifact from the // Google Cloud Storage bucket used by the CI to push snapshots func GetObjectURLFromBucket(bucket string, prefix string, object string, maxtimeout time.Duration) (string, error) { - exp := common.GetExponentialBackOff(maxtimeout) + exp := GetExponentialBackOff(maxtimeout) retryCount := 1 @@ -447,7 +463,7 @@ func DownloadFile(url string) (string, error) { filepath := tempFile.Name() - exp := common.GetExponentialBackOff(3) + exp := GetExponentialBackOff(3) retryCount := 1 var fileReader io.ReadCloser