diff --git a/e2e/_suites/fleet/features/apm_integration.feature b/e2e/_suites/fleet/features/apm_integration.feature index 76b93879b0..cff83f53d7 100644 --- a/e2e/_suites/fleet/features/apm_integration.feature +++ b/e2e/_suites/fleet/features/apm_integration.feature @@ -6,8 +6,8 @@ Scenarios for APM Scenario Outline: Deploying a stand-alone agent with fleet server mode Given a "" stand-alone agent is deployed with fleet server mode And the stand-alone agent is listed in Fleet as "online" - When the "Elastic APM" integration is added to the policy - Then the "Elastic APM" datasource is shown in the policy + When the "Elastic APM" integration is added in the policy + Then the "Elastic APM" datasource is shown in the policy as added And the "apm-server" process is in the "started" state on the host diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index c69b936ef8..014f87f5d5 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -31,7 +31,7 @@ const actionREMOVED = "removed" // FleetTestSuite represents the scenarios for Fleet-mode type FleetTestSuite struct { // integrations - Cleanup bool + StandAlone bool CurrentToken string // current enrollment token CurrentTokenID string // current enrollment tokenID ElasticAgentStopped bool // will be used to signal when the agent process can be called again in the tear-down stage @@ -47,32 +47,39 @@ type FleetTestSuite struct { kibanaClient *kibana.Client // fleet server FleetServerHostname string // hostname of the fleet server. If empty, it means the agent is the first one, bootstrapping fleet server + // date controls for queries + AgentStoppedDate time.Time + RuntimeDependenciesStartDate time.Time } // afterScenario destroys the state created by a scenario func (fts *FleetTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() - agentInstaller := fts.getInstaller() + serviceName := common.ElasticAgentServiceName + serviceManager := compose.NewServiceManager() - serviceName := fts.getServiceName(agentInstaller) + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + serviceName = fts.getServiceName(agentInstaller) - if log.IsLevelEnabled(log.DebugLevel) { - err := agentInstaller.PrintLogsFn(fts.Hostname) - if err != nil { - log.WithFields(log.Fields{ - "containerName": fts.Hostname, - "error": err, - }).Warn("Could not get agent logs in the container") + if log.IsLevelEnabled(log.DebugLevel) { + err := agentInstaller.PrintLogsFn(fts.Hostname) + if err != nil { + log.WithFields(log.Fields{ + "containerName": fts.Hostname, + "error": err, + }).Warn("Could not get agent logs in the container") + } } - } - - // only call it when the elastic-agent is present - if !fts.ElasticAgentStopped { - err := agentInstaller.UninstallFn() - if err != nil { - log.Warnf("Could not uninstall the agent after the scenario: %v", err) + // only call it when the elastic-agent is present + if !fts.ElasticAgentStopped { + err := agentInstaller.UninstallFn() + if err != nil { + log.Warnf("Could not uninstall the agent after the scenario: %v", err) + } } + } else if log.IsLevelEnabled(log.DebugLevel) { + _ = fts.getContainerLogs() } err := fts.unenrollHostname() @@ -106,11 +113,12 @@ func (fts *FleetTestSuite) afterScenario() { fts.Image = "" fts.Hostname = "" fts.FleetServerHostname = "" + fts.StandAlone = false } // beforeScenario creates the state needed by a scenario func (fts *FleetTestSuite) beforeScenario() { - fts.Cleanup = false + fts.StandAlone = false fts.ElasticAgentStopped = false fts.Version = common.AgentVersion @@ -160,6 +168,41 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the policy response will be shown in the Security App$`, fts.thePolicyResponseWillBeShownInTheSecurityApp) s.Step(`^the policy is updated to have "([^"]*)" in "([^"]*)" mode$`, fts.thePolicyIsUpdatedToHaveMode) s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) + + // stand-alone only steps + s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, fts.aStandaloneAgentIsDeployed) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, fts.bootstrapFleetServerFromAStandaloneAgent) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, fts.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud) + s.Step(`^there is new data in the index from agent$`, fts.thereIsNewDataInTheIndexFromAgent) + s.Step(`^the "([^"]*)" docker container is stopped$`, fts.theDockerContainerIsStopped) + s.Step(`^there is no new data in the index after agent shuts down$`, fts.thereIsNoNewDataInTheIndexAfterAgentShutsDown) + s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, fts.theStandaloneAgentIsListedInFleetWithStatus) +} + +func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { + waitForAgents := func() error { + agents, err := fts.kibanaClient.ListAgents() + if err != nil { + return err + } + + if len(agents) == 0 { + return errors.New("No agents found") + } + + agentZero := agents[0] + hostname := agentZero.LocalMetadata.Host.HostName + + return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) + } + maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + exp := common.GetExponentialBackOff(maxTimeout) + + err := backoff.Retry(waitForAgents, exp) + if err != nil { + return err + } + return nil } func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { @@ -310,7 +353,6 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i var fleetConfig *kibana.FleetConfig fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, fts.FleetServerHostname) - fts.Cleanup = true if err != nil { return err } @@ -665,46 +707,6 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error { return nil } -func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { - return thePolicyShowsTheDatasourceAdded(fts.kibanaClient, fts.FleetServerPolicy, packageName) -} - -func thePolicyShowsTheDatasourceAdded(client *kibana.Client, policy kibana.Policy, packageName string) error { - log.WithFields(log.Fields{ - "policyID": policy.ID, - "package": packageName, - }).Trace("Checking if the policy shows the package added") - - maxTimeout := time.Minute - retryCount := 1 - - exp := common.GetExponentialBackOff(maxTimeout) - - configurationIsPresentFn := func() error { - packagePolicy, err := client.GetIntegrationFromAgentPolicy(packageName, policy) - if err != nil { - log.WithFields(log.Fields{ - "packagePolicy": packagePolicy, - "policy": policy, - "retry": retryCount, - "error": err, - }).Warn("The integration was not found in the policy") - retryCount++ - return err - } - - retryCount++ - return err - } - - err := backoff.Retry(configurationIsPresentFn, exp) - if err != nil { - return err - } - - return nil -} - func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.FleetServerPolicy, packageName, action) } @@ -1227,3 +1229,26 @@ func inputs(integration string) []kibana.Input { } return []kibana.Input{} } + +func (fts *FleetTestSuite) getContainerLogs() error { + serviceManager := compose.NewServiceManager() + + profile := common.FleetProfileName + serviceName := common.ElasticAgentServiceName + + composes := []string{ + profile, // profile name + serviceName, // agent service + } + err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "service": serviceName, + }).Error("Could not retrieve Elastic Agent logs") + + return err + } + + return nil +} diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index c2366e2785..7ed54d2977 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -89,37 +89,23 @@ func setUpSuite() { kibanaClient: kibanaClient, Installers: map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map }, - StandAlone: &StandAloneTestSuite{ - kibanaClient: kibanaClient, - }, } } func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { ctx.BeforeScenario(func(*messages.Pickle) { log.Trace("Before Fleet scenario") - - imts.StandAlone.Cleanup = false - imts.Fleet.beforeScenario() }) ctx.AfterScenario(func(*messages.Pickle, error) { log.Trace("After Fleet scenario") - - if imts.StandAlone.Cleanup { - imts.StandAlone.afterScenario() - } - - if imts.Fleet.Cleanup { - imts.Fleet.afterScenario() - } + imts.Fleet.afterScenario() }) ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost) imts.Fleet.contributeSteps(ctx) - imts.StandAlone.contributeSteps(ctx) } func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { @@ -176,7 +162,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { imts.Fleet.setup() - imts.StandAlone.RuntimeDependenciesStartDate = time.Now().UTC() + imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC() }) ctx.AfterSuite(func() { diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index e52db64915..701bebda9f 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -7,127 +7,92 @@ package main import ( "context" "fmt" - "github.com/elastic/e2e-testing/cli/config" - "path" - "strings" - "time" - "github.com/cenkalti/backoff/v4" - "github.com/cucumber/godog" + "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/compose" "github.com/elastic/e2e-testing/internal/docker" - "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/installer" - "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "github.com/pkg/errors" + "path" + "strings" + "time" + + "github.com/elastic/e2e-testing/internal/elasticsearch" log "github.com/sirupsen/logrus" ) -// StandAloneTestSuite represents the scenarios for Stand-alone-mode -type StandAloneTestSuite struct { - Cleanup bool - Hostname string - Image string - FleetPolicy kibana.Policy - // date controls for queries - AgentStoppedDate time.Time - RuntimeDependenciesStartDate time.Time - kibanaClient *kibana.Client +func (fts *FleetTestSuite) aStandaloneAgentIsDeployed(image string) error { + return fts.startStandAloneAgent(image, "", nil) } -// afterScenario destroys the state created by a scenario -func (sats *StandAloneTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() - serviceName := common.ElasticAgentServiceName - - if log.IsLevelEnabled(log.DebugLevel) { - _ = sats.getContainerLogs() - } - - developerMode := shell.GetEnvBool("DEVELOPER_MODE") - if !developerMode { - _ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - } else { - log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") +func (fts *FleetTestSuite) bootstrapFleetServerFromAStandaloneAgent(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err } - - sats.kibanaClient.DeleteAllPolicies(sats.FleetPolicy) -} - -func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { - s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, sats.aStandaloneAgentIsDeployed) - s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, sats.bootstrapFleetServerFromAStandaloneAgent) - s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, sats.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud) - s.Step(`^there is new data in the index from agent$`, sats.thereIsNewDataInTheIndexFromAgent) - s.Step(`^the "([^"]*)" docker container is stopped$`, sats.theDockerContainerIsStopped) - s.Step(`^there is no new data in the index after agent shuts down$`, sats.thereIsNoNewDataInTheIndexAfterAgentShutsDown) - s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, sats.theStandaloneAgentIsListedInFleetWithStatus) - s.Step(`^the "([^"]*)" integration is added to the policy$`, sats.theIntegrationIsAddedToThePolicy) - s.Step(`^the "([^"]*)" datasource is shown in the policy$`, sats.thePolicyShowsTheDatasourceAdded) + fts.FleetServerPolicy = fleetPolicy + return fts.startStandAloneAgent(image, "", map[string]string{"fleetServerMode": "1"}) } -func (sats *StandAloneTestSuite) theIntegrationIsAddedToThePolicy(packageName string) error { - return theIntegrationIsOperatedInThePolicy(sats.kibanaClient, sats.FleetPolicy, packageName, "added") -} - -func (sats *StandAloneTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { - return thePolicyShowsTheDatasourceAdded(sats.kibanaClient, sats.FleetPolicy, packageName) -} - -func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud(image string) error { - fleetPolicy, err := sats.kibanaClient.GetDefaultPolicy(true) +func (fts *FleetTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) if err != nil { return err } - sats.FleetPolicy = fleetPolicy + fts.FleetServerPolicy = fleetPolicy volume := path.Join(config.OpDir(), "compose", "services", "elastic-agent", "apm-legacy") - return sats.startAgent(image, "docker-compose-cloud.yml", map[string]string{"apmVolume": volume}) + return fts.startStandAloneAgent(image, "docker-compose-cloud.yml", map[string]string{"apmVolume": volume}) } -func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { - waitForAgents := func() error { - agents, err := sats.kibanaClient.ListAgents() - if err != nil { - return err - } +func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { + maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + minimumHitsCount := 50 - if len(agents) == 0 { - return errors.New("No agents found") - } + result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) + if err != nil { + return err + } + + log.Tracef("Search result: %v", result) - agentZero := agents[0] - hostname := agentZero.LocalMetadata.Host.HostName + return elasticsearch.AssertHitsArePresent(result) +} - return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) - } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) +func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error { + serviceManager := compose.NewServiceManager() - err := backoff.Retry(waitForAgents, exp) + err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) if err != nil { return err } + fts.AgentStoppedDate = time.Now().UTC() + return nil } -func (sats *StandAloneTestSuite) bootstrapFleetServerFromAStandaloneAgent(image string) error { - fleetPolicy, err := sats.kibanaClient.GetDefaultPolicy(true) +func (fts *FleetTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { + maxTimeout := time.Duration(30) * time.Second + minimumHitsCount := 1 + + result, err := searchAgentData(fts.Hostname, fts.AgentStoppedDate, minimumHitsCount, maxTimeout) if err != nil { - return err + if strings.Contains(err.Error(), "type:index_not_found_exception") { + return err + } + + log.WithFields(log.Fields{ + "error": err, + }).Info("No documents were found for the Agent in the index after it stopped") + return nil } - sats.FleetPolicy = fleetPolicy - return sats.startAgent(image, "", map[string]string{"fleetServerMode": "1"}) -} -func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error { - return sats.startAgent(image, "", nil) + return elasticsearch.AssertHitsAreNotPresent(result) } -func (sats *StandAloneTestSuite) startAgent(image string, composeFilename string, env map[string]string) error { - +func (fts *FleetTestSuite) startStandAloneAgent(image string, composeFilename string, env map[string]string) error { + fts.StandAlone = true log.Trace("Deploying an agent to Fleet") dockerImageTag := common.AgentVersion @@ -177,11 +142,10 @@ func (sats *StandAloneTestSuite) startAgent(image string, composeFilename string return err } - sats.Image = image - sats.Hostname = hostname - sats.Cleanup = true + fts.Image = image + fts.Hostname = hostname - err = sats.installTestTools(containerName) + err = fts.installTestTools(containerName) if err != nil { return err } @@ -189,23 +153,36 @@ func (sats *StandAloneTestSuite) startAgent(image string, composeFilename string return nil } -func (sats *StandAloneTestSuite) getContainerLogs() error { - serviceManager := compose.NewServiceManager() +func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { + log.WithFields(log.Fields{ + "policyID": fts.FleetServerPolicy.ID, + "package": packageName, + }).Trace("Checking if the policy shows the package added") + + maxTimeout := time.Minute + retryCount := 1 - profile := common.FleetProfileName - serviceName := common.ElasticAgentServiceName + exp := common.GetExponentialBackOff(maxTimeout) - composes := []string{ - profile, // profile name - serviceName, // agent service + configurationIsPresentFn := func() error { + packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.FleetServerPolicy) + if err != nil { + log.WithFields(log.Fields{ + "packagePolicy": packagePolicy, + "policy": fts.FleetServerPolicy, + "retry": retryCount, + "error": err, + }).Warn("The integration was not found in the policy") + retryCount++ + return err + } + + retryCount++ + return err } - err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "service": serviceName, - }).Error("Could not retrieve Elastic Agent logs") + err := backoff.Retry(configurationIsPresentFn, exp) + if err != nil { return err } @@ -215,8 +192,8 @@ func (sats *StandAloneTestSuite) getContainerLogs() error { // installTestTools we need the container name because we use the Docker Client instead of Docker Compose // we are going to install those tools we use in the test framework for checking // and verifications -func (sats *StandAloneTestSuite) installTestTools(containerName string) error { - if sats.Image != "ubi8" { +func (fts *FleetTestSuite) installTestTools(containerName string) error { + if fts.Image != "ubi8" { return nil } @@ -245,51 +222,6 @@ func (sats *StandAloneTestSuite) installTestTools(containerName string) error { return nil } -func (sats *StandAloneTestSuite) thereIsNewDataInTheIndexFromAgent() error { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - minimumHitsCount := 50 - - result, err := searchAgentData(sats.Hostname, sats.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) - if err != nil { - return err - } - - log.Tracef("Search result: %v", result) - - return elasticsearch.AssertHitsArePresent(result) -} - -func (sats *StandAloneTestSuite) theDockerContainerIsStopped(serviceName string) error { - serviceManager := compose.NewServiceManager() - - err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - if err != nil { - return err - } - sats.AgentStoppedDate = time.Now().UTC() - - return nil -} - -func (sats *StandAloneTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { - maxTimeout := time.Duration(30) * time.Second - minimumHitsCount := 1 - - result, err := searchAgentData(sats.Hostname, sats.AgentStoppedDate, minimumHitsCount, maxTimeout) - if err != nil { - if strings.Contains(err.Error(), "type:index_not_found_exception") { - return err - } - - log.WithFields(log.Fields{ - "error": err, - }).Info("No documents were found for the Agent in the index after it stopped") - return nil - } - - return elasticsearch.AssertHitsAreNotPresent(result) -} - func searchAgentData(hostname string, startDate time.Time, minimumHitsCount int, maxTimeout time.Duration) (elasticsearch.SearchResult, error) { timezone := "America/New_York" diff --git a/e2e/_suites/fleet/world.go b/e2e/_suites/fleet/world.go index 0560e1d2d8..27cc74345f 100644 --- a/e2e/_suites/fleet/world.go +++ b/e2e/_suites/fleet/world.go @@ -13,8 +13,7 @@ import ( // IngestManagerTestSuite represents a test suite, holding references to the pieces needed to run the tests type IngestManagerTestSuite struct { - Fleet *FleetTestSuite - StandAlone *StandAloneTestSuite + Fleet *FleetTestSuite } func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state string) error { @@ -22,7 +21,7 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state var containerName string - if imts.StandAlone.Hostname != "" { + if imts.Fleet.StandAlone { containerName = fmt.Sprintf("%s_%s_%d", profile, common.ElasticAgentServiceName, 1) } else { agentInstaller := imts.Fleet.getInstaller()