From 81d77659e041d2be1219f1d5c0e5051520046c08 Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 23 Apr 2021 12:44:00 +0200 Subject: [PATCH 1/6] some tests for apm-server --- .../apm-legacy/config/apm-server.yml | 21 ++++ .../apm-legacy/config/capabilities.yml | 5 + .../apm-legacy/config/credentials.yml | 10 ++ .../apm-legacy/config/fleet-setup.yml | 9 ++ .../elastic-agent/docker-compose-cloud.yml | 21 ++++ .../services/elastic-agent/docker-compose.yml | 4 - cli/config/config.go | 16 ++- .../fleet/features/apm_integration.feature | 30 +++++ e2e/_suites/fleet/fleet.go | 115 ++++++++++-------- e2e/_suites/fleet/ingest_manager_test.go | 4 + e2e/_suites/fleet/stand-alone.go | 43 +++++-- internal/compose/compose.go | 18 +-- internal/kibana/integrations.go | 2 +- internal/kibana/policies.go | 24 ++++ 14 files changed, 241 insertions(+), 81 deletions(-) create mode 100644 cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml create mode 100644 cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml create mode 100644 cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml create mode 100644 cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml create mode 100644 cli/config/compose/services/elastic-agent/docker-compose-cloud.yml create mode 100644 e2e/_suites/fleet/features/apm_integration.feature diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml new file mode 100644 index 0000000000..ab3e77f654 --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml @@ -0,0 +1,21 @@ +monitoring.enabled: true +http.enabled: true +http.port: 5067 +http.host: "0.0.0.0" +apm-server: + host: "0.0.0:8200" + secret_token: "1234" + # Enable APM Server Golang expvar support (https://golang.org/pkg/expvar/). + expvar: + enabled: true + url: "/debug/vars" + kibana: + # For APM Agent configuration in Kibana, enabled must be true. + enabled: true + host: "" + username: "elastic" + password: "changeme" +output.elasticsearch: + hosts: [""] + username: "elastic" + password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml new file mode 100644 index 0000000000..e2ad548a4c --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml @@ -0,0 +1,5 @@ +capabilities: +- rule: allow + input: fleet-server +- rule: deny + input: "*" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml new file mode 100644 index 0000000000..ffe9bc6f27 --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml @@ -0,0 +1,10 @@ +fleet_server: + elasticsearch: + host: "" + username: "elastic" + password: "changeme" +kibana: + fleet: + host: "" + username: "admin" + password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml new file mode 100644 index 0000000000..30feaf7e85 --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml @@ -0,0 +1,9 @@ +fleet: + enroll: true + force: false + insecure: true +fleet_server: + enable: true +kibana: + fleet: + setup: true diff --git a/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml b/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml new file mode 100644 index 0000000000..024a047ead --- /dev/null +++ b/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml @@ -0,0 +1,21 @@ +version: '2.4' +services: + elastic-agent: + image: docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.0.0-SNAPSHOT} + container_name: ${elasticAgentContainerName} + environment: + - "FLEET_SERVER_ENABLE=1" + - "FLEET_SERVER_INSECURE_HTTP=1" + - "ELASTIC_AGENT_CLOUD=1" + - "APM_SERVER_PATH=/apm-legacy/apm-server/" + - "STATE_PATH=/apm-legacy/elastic-agent/" + - "CONFIG_PATH=/apm-legacy/config/" + - "DATA_PATH=/apm-legacy/data/" + - "LOGS_PATH=/apm-legacy/logs/" + - "HOME_PATH=/apm-legacy/" + volumes: + - "./apm-legacy:/apm-legacy" + ports: + - "127.0.0.1:8220:8220" + - "127.0.0.1:8200:8200" + - "127.0.0.1:5066:5066" diff --git a/cli/config/compose/services/elastic-agent/docker-compose.yml b/cli/config/compose/services/elastic-agent/docker-compose.yml index 1f04538ef3..3d493cd0ab 100644 --- a/cli/config/compose/services/elastic-agent/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/docker-compose.yml @@ -9,12 +9,8 @@ services: kibana: condition: service_healthy environment: - - "FLEET_SERVER_ELASTICSEARCH_HOST=http://${elasticsearchHost:-elasticsearch}:${elasticsearchPort:-9200}" - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}" - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}" - - "FLEET_SERVER_HOST=0.0.0.0" - - "FLEET_SERVER_ELASTICSEARCH_USERNAME=elastic" - - "FLEET_SERVER_ELASTICSEARCH_PASSWORD=changeme" platform: ${elasticAgentPlatform:-linux/amd64} ports: - "127.0.0.1:8220:8220" diff --git a/cli/config/config.go b/cli/config/config.go index 4eae0ccc93..eea18f70ad 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -73,14 +73,16 @@ func FileExists(configFile string) (bool, error) { // GetComposeFile returns the path of the compose file, looking up the // tool's workdir -func GetComposeFile(isProfile bool, composeName string) (string, error) { - composeFileName := "docker-compose.yml" +func GetComposeFile(isProfile bool, composeName string, composeFileName ...string) (string, error) { + if isProfile || composeFileName == nil || composeFileName[0] == "" { + composeFileName = []string{"docker-compose.yml"} + } serviceType := "services" if isProfile { serviceType = "profiles" } - composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName) + composeFilePath := path.Join(Op.Workspace, "compose", serviceType, composeName, composeFileName[0]) found, err := io.Exists(composeFilePath) if found && err == nil { log.WithFields(log.Fields{ @@ -138,7 +140,11 @@ func InitConfig() { if Op != nil { return } + newConfig(OpDir()) +} +// OpDir returns the directory to copy to +func OpDir() string { home, err := homedir.Dir() if err != nil { log.WithFields(log.Fields{ @@ -146,9 +152,7 @@ func InitConfig() { }).Fatal("Could not get current user's HOME dir") } - w := filepath.Join(home, ".op") - - newConfig(w) + return filepath.Join(home, ".op") } // PutServiceEnvironment puts the environment variables for the service, replacing "SERVICE_" diff --git a/e2e/_suites/fleet/features/apm_integration.feature b/e2e/_suites/fleet/features/apm_integration.feature new file mode 100644 index 0000000000..76b93879b0 --- /dev/null +++ b/e2e/_suites/fleet/features/apm_integration.feature @@ -0,0 +1,30 @@ +@apm_server +Feature: APM Integration +Scenarios for APM + +@install +Scenario Outline: Deploying a stand-alone agent with fleet server mode + Given a "" stand-alone agent is deployed with fleet server mode + And the stand-alone agent is listed in Fleet as "online" + When the "Elastic APM" integration is added to the policy + Then the "Elastic APM" datasource is shown in the policy + And the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | + + + +@cloud +Scenario Outline: Deploying a stand-alone agent with fleet server mode on cloud + When a "" stand-alone agent is deployed with fleet server mode on cloud + Then the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index f632d9a21f..402ca7e390 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -96,26 +96,7 @@ func (fts *FleetTestSuite) afterScenario() { }).Warn("The enrollment token could not be deleted") } - // Cleanup all package policies - packagePolicies, err := fts.kibanaClient.ListPackagePolicies() - if err != nil { - log.WithFields(log.Fields{ - "err": err, - "policy": fts.FleetPolicy, - }).Error("The package policies could not be found") - } - for _, pkgPolicy := range packagePolicies { - // Do not remove the fleet server package integration otherwise fleet server fails to bootstrap - if !strings.Contains(pkgPolicy.Name, "fleet_server") && pkgPolicy.PolicyID == fts.FleetPolicy.ID { - err = fts.kibanaClient.DeleteIntegrationFromPolicy(pkgPolicy) - if err != nil { - log.WithFields(log.Fields{ - "err": err, - "packagePolicy": pkgPolicy, - }).Error("The integration could not be deleted from the configuration") - } - } - } + fts.kibanaClient.DeleteAllPolicies(fts.FleetPolicy) // clean up fields fts.CurrentTokenID = "" @@ -679,8 +660,12 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error { } func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { + return thePolicyShowsTheDatasourceAdded(fts.kibanaClient, fts.Policy, packageName) +} + +func thePolicyShowsTheDatasourceAdded(client *kibana.Client, policy kibana.Policy, packageName string) error { log.WithFields(log.Fields{ - "policyID": fts.FleetPolicy.ID, + "policyID": policy.ID, "package": packageName, }).Trace("Checking if the policy shows the package added") @@ -690,11 +675,11 @@ func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) exp := common.GetExponentialBackOff(maxTimeout) configurationIsPresentFn := func() error { - packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.FleetPolicy) + packagePolicy, err := client.GetIntegrationFromAgentPolicy(packageName, policy) if err != nil { log.WithFields(log.Fields{ "packagePolicy": packagePolicy, - "policy": fts.FleetPolicy, + "policy": policy, "retry": retryCount, "error": err, }).Warn("The integration was not found in the policy") @@ -715,13 +700,17 @@ func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) } func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { + return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.FleetPolicy, packageName, action) +} + +func theIntegrationIsOperatedInThePolicy(client *kibana.Client, policy kibana.Policy, packageName string, action string) error { log.WithFields(log.Fields{ "action": action, - "policy": fts.FleetPolicy, + "policy": policy, "package": packageName, }).Trace("Doing an operation for a package on a policy") - integration, err := fts.kibanaClient.GetIntegrationByPackageName(packageName) + integration, err := client.GetIntegrationByPackageName(packageName) if err != nil { return err } @@ -731,44 +720,20 @@ func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName strin Name: integration.Name, Description: integration.Title, Namespace: "default", - PolicyID: fts.FleetPolicy.ID, + PolicyID: policy.ID, Enabled: true, Package: integration, Inputs: []kibana.Input{}, } + packageDataStream.Inputs = inputs(integration.Name) - if strings.EqualFold(integration.Name, "linux") { - packageDataStream.Inputs = []kibana.Input{ - { - Type: "linux/metrics", - Enabled: true, - Streams: []interface{}{ - map[string]interface{}{ - "id": "linux/metrics-linux.memory-" + uuid.New().String(), - "enabled": true, - "data_stream": map[string]interface{}{ - "dataset": "linux.memory", - "type": "metrics", - }, - }, - }, - Vars: map[string]kibana.Var{ - "period": { - Value: "1s", - Type: "string", - }, - }, - }, - } - } - - return fts.kibanaClient.AddIntegrationToPolicy(packageDataStream) + return client.AddIntegrationToPolicy(packageDataStream) } else if strings.ToLower(action) == actionREMOVED { - packageDataStream, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(integration.Name, fts.FleetPolicy) + packageDataStream, err := client.GetIntegrationFromAgentPolicy(integration.Name, policy) if err != nil { return err } - return fts.kibanaClient.DeleteIntegrationFromPolicy(packageDataStream) + return client.DeleteIntegrationFromPolicy(packageDataStream) } return nil @@ -1220,3 +1185,45 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return cfg, agentInstaller.PostInstallFn() } + +func inputs(integration string) []kibana.Input { + switch integration { + case "linux": + return []kibana.Input{ + { + Type: "linux/metrics", + Enabled: true, + Streams: []interface{}{ + map[string]interface{}{ + "id": "linux/metrics-linux.memory-" + uuid.New().String(), + "enabled": true, + "data_stream": map[string]interface{}{ + "dataset": "linux.memory", + "type": "metrics", + }, + }, + }, + Vars: map[string]kibana.Var{ + "period": { + Value: "1s", + Type: "string", + }, + }, + }} + case "apm": + return []kibana.Input{ + { + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]kibana.Var{ + "apm-server": { + Value: "host", + Type: "localhost:8200", + }, + }, + }, + } + } + return nil +} diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index ab09ecb7b2..d8e60f6972 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -7,6 +7,7 @@ package main import ( "context" "os" + "path/filepath" "strings" "time" @@ -130,6 +131,8 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { if imts.Fleet.Cleanup { imts.Fleet.afterScenario() } + + os.RemoveAll(filepath.Join(config.OpDir(), "compose")) }) ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost) @@ -162,6 +165,7 @@ func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { if err != nil { log.WithFields(log.Fields{ "profile": profile, + "error": err.Error(), }).Fatal("Could not run the runtime dependencies for the profile.") } diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index fd4d723cfe..10c7195a57 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -26,9 +26,10 @@ import ( // StandAloneTestSuite represents the scenarios for Stand-alone-mode type StandAloneTestSuite struct { - Cleanup bool - Hostname string - Image string + Cleanup bool + Hostname string + Image string + FleetPolicy kibana.Policy // date controls for queries AgentStoppedDate time.Time RuntimeDependenciesStartDate time.Time @@ -50,15 +51,37 @@ func (sats *StandAloneTestSuite) afterScenario() { } else { log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") } + + sats.kibanaClient.DeleteAllPolicies(sats.FleetPolicy) } func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, sats.aStandaloneAgentIsDeployed) s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, sats.aStandaloneAgentIsDeployedWithFleetServerMode) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, sats.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud) s.Step(`^there is new data in the index from agent$`, sats.thereIsNewDataInTheIndexFromAgent) s.Step(`^the "([^"]*)" docker container is stopped$`, sats.theDockerContainerIsStopped) s.Step(`^there is no new data in the index after agent shuts down$`, sats.thereIsNoNewDataInTheIndexAfterAgentShutsDown) s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, sats.theStandaloneAgentIsListedInFleetWithStatus) + s.Step(`^the "([^"]*)" integration is added to the policy$`, sats.theIntegrationIsAddedToThePolicy) + s.Step(`^the "([^"]*)" datasource is shown in the policy$`, sats.thePolicyShowsTheDatasourceAdded) +} + +func (sats *StandAloneTestSuite) theIntegrationIsAddedToThePolicy(packageName string) error { + return theIntegrationIsOperatedInThePolicy(sats.kibanaClient, sats.FleetPolicy, packageName, "added") +} + +func (sats *StandAloneTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { + return thePolicyShowsTheDatasourceAdded(sats.kibanaClient, sats.FleetPolicy, packageName) +} + +func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud(image string) error { + fleetPolicy, err := sats.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err + } + sats.FleetPolicy = fleetPolicy + return sats.startAgent(image, "docker-compose-cloud.yml", nil) } func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { @@ -88,14 +111,19 @@ func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(des } func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerMode(image string) error { - return sats.startAgent(image, map[string]string{"fleetServerMode": "1"}) + fleetPolicy, err := sats.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err + } + sats.FleetPolicy = fleetPolicy + return sats.startAgent(image, "", map[string]string{"fleetServerMode": "1"}) } func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error { - return sats.startAgent(image, nil) + return sats.startAgent(image, "", nil) } -func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) error { +func (sats *StandAloneTestSuite) startAgent(image string, composeFilename string, env map[string]string) error { log.Trace("Deploying an agent to Fleet") @@ -133,7 +161,8 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) common.ProfileEnv[k] = v } - err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName, []string{common.ElasticAgentServiceName}, common.ProfileEnv) + err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName, + []string{common.ElasticAgentServiceName}, common.ProfileEnv, composeFilename) if err != nil { log.Error("Could not deploy the elastic-agent") return err diff --git a/internal/compose/compose.go b/internal/compose/compose.go index cc6374282a..a2f3e2cff5 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -19,7 +19,7 @@ import ( // ServiceManager manages lifecycle of a service type ServiceManager interface { - AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error + AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string, composeFilename ...string) error ExecCommandInService(profile string, image string, serviceName string, cmds []string, env map[string]string, detach bool) error RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error @@ -37,7 +37,7 @@ func NewServiceManager() ServiceManager { } // AddServicesToCompose adds services to a running docker compose -func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string, composeFilename ...string) error { span, _ := apm.StartSpanOptions(ctx, "Add services to Docker Compose", "docker-compose.services.add", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -56,7 +56,7 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil persistedEnv[k] = v } - err := executeCompose(sm, true, newComposeNames, []string{"up", "-d"}, persistedEnv) + err := executeCompose(true, newComposeNames, []string{"up", "-d"}, persistedEnv, composeFilename...) if err != nil { return err } @@ -115,7 +115,7 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p command := []string{"rm", "-fvs"} command = append(command, composeName) - err := executeCompose(sm, true, newComposeNames, command, persistedEnv) + err := executeCompose(true, newComposeNames, command, persistedEnv) if err != nil { log.WithFields(log.Fields{ "command": command, @@ -135,7 +135,7 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p // RunCommand executes a docker-compose command in a running a docker compose func (sm *DockerServiceManager) RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error { - return executeCompose(sm, true, composeNames, composeArgs, env) + return executeCompose(true, composeNames, composeArgs, env) } // RunCompose runs a docker compose by its name @@ -145,7 +145,7 @@ func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, }) defer span.End() - return executeCompose(sm, isProfile, composeNames, []string{"up", "-d"}, env) + return executeCompose(isProfile, composeNames, []string{"up", "-d"}, env) } // StopCompose stops a docker compose by its name @@ -175,7 +175,7 @@ func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, } persistedEnv := state.Recover(ID, config.Op.Workspace) - err := executeCompose(sm, isProfile, composeNames, []string{"down", "--remove-orphans"}, persistedEnv) + err := executeCompose(isProfile, composeNames, []string{"down", "--remove-orphans"}, persistedEnv) if err != nil { return fmt.Errorf("Could not stop compose file: %v - %v", composeFilePaths, err) } @@ -189,7 +189,7 @@ func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, return nil } -func executeCompose(sm *DockerServiceManager, isProfile bool, composeNames []string, command []string, env map[string]string) error { +func executeCompose(isProfile bool, composeNames []string, command []string, env map[string]string, composeFilename ...string) error { composeFilePaths := make([]string, len(composeNames)) for i, composeName := range composeNames { b := false @@ -197,7 +197,7 @@ func executeCompose(sm *DockerServiceManager, isProfile bool, composeNames []str b = true } - composeFilePath, err := config.GetComposeFile(b, composeName) + composeFilePath, err := config.GetComposeFile(b, composeName, composeFilename...) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } diff --git a/internal/kibana/integrations.go b/internal/kibana/integrations.go index 2c63dea9a4..8807cc287a 100644 --- a/internal/kibana/integrations.go +++ b/internal/kibana/integrations.go @@ -52,7 +52,7 @@ func (c *Client) DeleteIntegrationFromPolicy(packageDS PackageDataStream) error // GetIntegrations returns all available integrations func (c *Client) GetIntegrations() ([]IntegrationPackage, error) { - statusCode, respBody, err := c.get(fmt.Sprintf("%s/epm/packages", FleetAPI)) + statusCode, respBody, err := c.get(fmt.Sprintf("%s/epm/packages?experimental=true", FleetAPI)) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/kibana/policies.go b/internal/kibana/policies.go index 7e7402b416..5667a499ae 100644 --- a/internal/kibana/policies.go +++ b/internal/kibana/policies.go @@ -7,6 +7,7 @@ package kibana import ( "encoding/json" "fmt" + "strings" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -75,6 +76,29 @@ func (c *Client) ListPolicies() ([]Policy, error) { return resp.Items, nil } +// DeleteAllPolicies deletes all policies +func (c *Client) DeleteAllPolicies(except Policy) { + // Cleanup all package policies + packagePolicies, err := c.ListPackagePolicies() + if err != nil { + log.WithFields(log.Fields{ + "err": err, + }).Error("The package policies could not be found") + } + for _, pkgPolicy := range packagePolicies { + // Do not remove the fleet server package integration otherwise fleet server fails to bootstrap + if !strings.Contains(pkgPolicy.Name, "fleet_server") && pkgPolicy.PolicyID == except.ID { + err = c.DeleteIntegrationFromPolicy(pkgPolicy) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "packagePolicy": pkgPolicy, + }).Error("The integration could not be deleted from the configuration") + } + } + } +} + // Var represents a single variable at the package or // data stream level, encapsulating the data type of the // variable and it's value. From 6129f555368099e537b1179a3df6cedfc7a67af1 Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 23 Apr 2021 14:47:32 +0200 Subject: [PATCH 2/6] clean op dir on init instead of after --- cli/config/config.go | 8 ++++---- e2e/_suites/fleet/ingest_manager_test.go | 3 --- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cli/config/config.go b/cli/config/config.go index eea18f70ad..13171d0a18 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -140,11 +140,11 @@ func InitConfig() { if Op != nil { return } - newConfig(OpDir()) + newConfig(opDir()) } -// OpDir returns the directory to copy to -func OpDir() string { +// opDir returns the directory to copy to +func opDir() string { home, err := homedir.Dir() if err != nil { log.WithFields(log.Fields{ @@ -228,7 +228,7 @@ func PutServiceVariantEnvironment(env map[string]string, service string, service func checkConfigDirectory(dir string) { found, err := io.Exists(dir) if found && err == nil { - return + os.RemoveAll(dir) } _ = io.MkdirAll(dir) } diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index d8e60f6972..c92394a5ee 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -7,7 +7,6 @@ package main import ( "context" "os" - "path/filepath" "strings" "time" @@ -131,8 +130,6 @@ func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { if imts.Fleet.Cleanup { imts.Fleet.afterScenario() } - - os.RemoveAll(filepath.Join(config.OpDir(), "compose")) }) ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost) From 7ad417b6e1d90b1ba87df349c13693ddb952e553 Mon Sep 17 00:00:00 2001 From: Juan Date: Tue, 27 Apr 2021 13:34:06 +0200 Subject: [PATCH 3/6] fix things --- .../elastic-agent/apm-legacy/config/apm-server.yml | 4 ++-- .../elastic-agent/apm-legacy/config/credentials.yml | 6 +++--- .../services/elastic-agent/docker-compose-cloud.yml | 7 ++++++- cli/config/config.go | 6 +++--- e2e/_suites/fleet/stand-alone.go | 5 ++++- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml index ab3e77f654..ac1a563a91 100644 --- a/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml @@ -12,10 +12,10 @@ apm-server: kibana: # For APM Agent configuration in Kibana, enabled must be true. enabled: true - host: "" + host: "kibana" username: "elastic" password: "changeme" output.elasticsearch: - hosts: [""] + hosts: ["http://elasticsearch:9200"] username: "elastic" password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml index ffe9bc6f27..90e67b0a2a 100644 --- a/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml @@ -1,10 +1,10 @@ fleet_server: elasticsearch: - host: "" + host: "elasticsearch" username: "elastic" password: "changeme" kibana: fleet: - host: "" - username: "admin" + host: "kibana" + username: "elastic" password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml b/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml index 024a047ead..e4613f7f0c 100644 --- a/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml +++ b/cli/config/compose/services/elastic-agent/docker-compose-cloud.yml @@ -3,6 +3,11 @@ services: elastic-agent: image: docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.0.0-SNAPSHOT} container_name: ${elasticAgentContainerName} + depends_on: + elasticsearch: + condition: service_healthy + kibana: + condition: service_healthy environment: - "FLEET_SERVER_ENABLE=1" - "FLEET_SERVER_INSECURE_HTTP=1" @@ -14,7 +19,7 @@ services: - "LOGS_PATH=/apm-legacy/logs/" - "HOME_PATH=/apm-legacy/" volumes: - - "./apm-legacy:/apm-legacy" + - "${apmVolume}:/apm-legacy" ports: - "127.0.0.1:8220:8220" - "127.0.0.1:8200:8200" diff --git a/cli/config/config.go b/cli/config/config.go index 5763d14442..e41f5ca7a3 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -140,11 +140,11 @@ func InitConfig() { if Op != nil { return } - newConfig(opDir()) + newConfig(OpDir()) } -// opDir returns the directory to copy to -func opDir() string { +// OpDir returns the directory to copy to +func OpDir() string { home, err := homedir.Dir() if err != nil { log.WithFields(log.Fields{ diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index 10c7195a57..59c0aafca3 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -7,6 +7,8 @@ package main import ( "context" "fmt" + "github.com/elastic/e2e-testing/cli/config" + "path" "strings" "time" @@ -81,7 +83,8 @@ func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOn return err } sats.FleetPolicy = fleetPolicy - return sats.startAgent(image, "docker-compose-cloud.yml", nil) + volume := path.Join(config.OpDir(), "compose", "services", "elastic-agent", "apm-legacy") + return sats.startAgent(image, "docker-compose-cloud.yml", map[string]string{"apmVolume": volume}) } func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { From 903794feb6e01c4186f23961f37bd9cd4a318ca7 Mon Sep 17 00:00:00 2001 From: Juan Date: Tue, 27 Apr 2021 15:39:24 +0200 Subject: [PATCH 4/6] fix more things --- cli/config/config.go | 6 +++--- e2e/_suites/fleet/fleet.go | 29 +++++++++++++++-------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/cli/config/config.go b/cli/config/config.go index e41f5ca7a3..445d260677 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -132,11 +132,11 @@ func Init() { } shell.CheckInstalledSoftware(binaries...) - InitConfig() + initConfig() } -// InitConfig initialises configuration -func InitConfig() { +// initConfig initialises configuration +func initConfig() { if Op != nil { return } diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 402ca7e390..dd1edc7653 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -1188,6 +1188,20 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe func inputs(integration string) []kibana.Input { switch integration { + case "apm": + return []kibana.Input{ + { + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]kibana.Var{ + "apm-server": { + Value: "host", + Type: "localhost:8200", + }, + }, + }, + } case "linux": return []kibana.Input{ { @@ -1209,21 +1223,8 @@ func inputs(integration string) []kibana.Input { Type: "string", }, }, - }} - case "apm": - return []kibana.Input{ - { - Type: "apm", - Enabled: true, - Streams: []interface{}{}, - Vars: map[string]kibana.Var{ - "apm-server": { - Value: "host", - Type: "localhost:8200", - }, - }, }, } } - return nil + return []kibana.Input{} } From ef431bbe7c03cbcd7410ef3726db33c51aa37476 Mon Sep 17 00:00:00 2001 From: Juan Date: Wed, 28 Apr 2021 13:38:53 +0200 Subject: [PATCH 5/6] fiiiix --- cli/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/config/config.go b/cli/config/config.go index 445d260677..a77d574251 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -228,7 +228,7 @@ func PutServiceVariantEnvironment(env map[string]string, service string, service func checkConfigDirectory(dir string) { found, err := io.Exists(dir) if found && err == nil { - os.RemoveAll(dir) + return } _ = io.MkdirAll(dir) } From d40400a851791219f6dfe97abf49fda8890459f6 Mon Sep 17 00:00:00 2001 From: Juan Date: Wed, 28 Apr 2021 13:57:25 +0200 Subject: [PATCH 6/6] fix merge --- e2e/_suites/fleet/fleet.go | 45 +++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index 2a81a288a2..5132467758 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -7,6 +7,7 @@ package main import ( "context" "fmt" + "github.com/google/uuid" "strings" "time" @@ -20,7 +21,6 @@ import ( "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "github.com/google/uuid" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -1184,3 +1184,46 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return cfg, agentInstaller.PostInstallFn() } + +func inputs(integration string) []kibana.Input { + switch integration { + case "apm": + return []kibana.Input{ + { + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]kibana.Var{ + "apm-server": { + Value: "host", + Type: "localhost:8200", + }, + }, + }, + } + case "linux": + return []kibana.Input{ + { + Type: "linux/metrics", + Enabled: true, + Streams: []interface{}{ + map[string]interface{}{ + "id": "linux/metrics-linux.memory-" + uuid.New().String(), + "enabled": true, + "data_stream": map[string]interface{}{ + "dataset": "linux.memory", + "type": "metrics", + }, + }, + }, + Vars: map[string]kibana.Var{ + "period": { + Value: "1s", + Type: "string", + }, + }, + }, + } + } + return []kibana.Input{} +}