diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml index c9613c115b..02a4399f08 100644 --- a/.ci/.e2e-tests.yaml +++ b/.ci/.e2e-tests.yaml @@ -17,9 +17,6 @@ SUITES: - name: "Fleet" pullRequestFilter: " && ~debian" tags: "fleet_mode_agent" - - name: "Fleet Server" - pullRequestFilter: " && ~debian" - tags: "fleet_server" - name: "Endpoint Integration" pullRequestFilter: " && ~debian" tags: "agent_endpoint_integration" diff --git a/.ci/scripts/clean-docker.sh b/.ci/scripts/clean-docker.sh index ef2a271355..bc3fc0f29c 100755 --- a/.ci/scripts/clean-docker.sh +++ b/.ci/scripts/clean-docker.sh @@ -9,7 +9,7 @@ set -euxo pipefail # Build and test the app using the install and test make goals. # -readonly VERSION="7.13.0-SNAPSHOT" +readonly VERSION="$(cat $(pwd)/.stack-version)" main() { # refresh docker images diff --git a/.ci/scripts/fleet-test.sh b/.ci/scripts/fleet-test.sh index 30ec8f59ed..edaa021877 100755 --- a/.ci/scripts/fleet-test.sh +++ b/.ci/scripts/fleet-test.sh @@ -9,10 +9,10 @@ set -euxo pipefail # Run the functional tests for fleets using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.13.0-SNAPSHOT'} +STACK_VERSION=${1:-"$(cat $(pwd)/.stack-version)"} SUITE='fleet' # Exclude the nightly tests in the CI. diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index e9117fa6ce..a0e2edd791 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -12,14 +12,16 @@ set -euxo pipefail # Parameters: # - SUITE - that's the suite to be tested. Default '' which means all of them. # - TAGS - that's the tags to be tested. Default '' which means all of them. -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # +BASE_VERSION="$(cat $(pwd)/.stack-version)" + SUITE=${1:-''} TAGS=${2:-''} -STACK_VERSION=${3:-'7.13.0-SNAPSHOT'} -BEAT_VERSION=${4:-'7.13.0-SNAPSHOT'} +STACK_VERSION=${3:-"${BASE_VERSION}"} +BEAT_VERSION=${4:-"${BASE_VERSION}"} ## Install the required dependencies for the given SUITE .ci/scripts/install-test-dependencies.sh "${SUITE}" diff --git a/.ci/scripts/metricbeat-test.sh b/.ci/scripts/metricbeat-test.sh index 7e78fa2b97..460d649e63 100755 --- a/.ci/scripts/metricbeat-test.sh +++ b/.ci/scripts/metricbeat-test.sh @@ -9,12 +9,14 @@ set -euxo pipefail # Run the functional tests for metricbeat using the functional-test wrapper # # Parameters: -# - STACK_VERSION - that's the version of the stack to be tested. Default '7.13.0-SNAPSHOT'. -# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.13.0-SNAPSHOT'. +# - STACK_VERSION - that's the version of the stack to be tested. Default is stored in '.stack-version'. +# - BEAT_VERSION - that's the version of the metricbeat to be tested. Default is stored in '.stack-version'. # -STACK_VERSION=${1:-'7.13.0-SNAPSHOT'} -BEAT_VERSION=${2:-'7.13.0-SNAPSHOT'} +BASE_VERSION="$(cat $(pwd)/.stack-version)" + +STACK_VERSION=${1:-"${BASE_VERSION}"} +BEAT_VERSION=${2:-"${BASE_VERSION}"} SUITE='metricbeat' .ci/scripts/functional-test.sh "${SUITE}" "" "${STACK_VERSION}" "${BEAT_VERSION}" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7514981ed2..9687ae0d33 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,8 @@ repos: exclude: ^notice/overrides.json - id: check-merge-conflict - id: check-yaml + exclude: > + (?x)^(cli/config/kubernetes.*)$ - id: check-xml - id: end-of-file-fixer exclude: > diff --git a/.stack-version b/.stack-version new file mode 100644 index 0000000000..96cfbb19ae --- /dev/null +++ b/.stack-version @@ -0,0 +1 @@ +7.13.0-SNAPSHOT diff --git a/cli/cmd/deploy.go b/cli/cmd/deploy.go index aef82542f8..ec3d728013 100644 --- a/cli/cmd/deploy.go +++ b/cli/cmd/deploy.go @@ -8,7 +8,7 @@ import ( "context" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -63,12 +63,16 @@ func buildDeployServiceCommand(srv string) *cobra.Command { Short: `Deploys a ` + srv + ` service`, Long: `Deploys a ` + srv + ` service, adding it to a running profile, identified by its name`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{} env = config.PutServiceEnvironment(env, srv, versionToRun) - err := serviceManager.AddServicesToCompose(context.Background(), deployToProfile, []string{srv}, env) + err := serviceManager.AddServicesToCompose( + context.Background(), + deploy.NewServiceRequest(deployToProfile), + []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, + env) if err != nil { log.WithFields(log.Fields{ "profile": deployToProfile, @@ -85,12 +89,16 @@ func buildUndeployServiceCommand(srv string) *cobra.Command { Short: `Undeploys a ` + srv + ` service`, Long: `Undeploys a ` + srv + ` service, removing it from a running profile, identified by its name`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{} env = config.PutServiceEnvironment(env, srv, versionToRun) - err := serviceManager.RemoveServicesFromCompose(context.Background(), deployToProfile, []string{srv}, env) + err := serviceManager.RemoveServicesFromCompose( + context.Background(), + deploy.NewServiceRequest(deployToProfile), + []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, + env) if err != nil { log.WithFields(log.Fields{ "profile": deployToProfile, diff --git a/cli/cmd/run.go b/cli/cmd/run.go index c2dbb3a8c4..6006090112 100644 --- a/cli/cmd/run.go +++ b/cli/cmd/run.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -64,7 +64,7 @@ func buildRunServiceCommand(srv string) *cobra.Command { Short: `Runs a ` + srv + ` service`, Long: `Runs a ` + srv + ` service, spinning up a Docker container for it and exposing its internal configuration so that you are able to connect to it in an easy manner`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := config.PutServiceEnvironment(map[string]string{}, srv, versionToRun) @@ -76,7 +76,8 @@ func buildRunServiceCommand(srv string) *cobra.Command { env[k] = v } - err := serviceManager.RunCompose(context.Background(), false, []string{srv}, env) + err := serviceManager.RunCompose( + context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}, env) if err != nil { log.WithFields(log.Fields{ "service": srv, @@ -96,7 +97,7 @@ Example: go run main.go run profile fleet -s elastic-agent:7.13.0-SNAPSHOT `, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{ "profileVersion": versionToRun, @@ -110,14 +111,15 @@ Example: env[k] = v } - err := serviceManager.RunCompose(context.Background(), true, []string{key}, env) + err := serviceManager.RunCompose( + context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}, env) if err != nil { log.WithFields(log.Fields{ "profile": key, }).Error("Could not run the profile.") } - composeNames := []string{} + composeNames := []deploy.ServiceRequest{} if len(servicesToRun) > 0 { for _, srv := range servicesToRun { arr := strings.Split(srv, ":") @@ -137,10 +139,10 @@ Example: }).Trace("Adding service") env = config.PutServiceEnvironment(env, image, tag) - composeNames = append(composeNames, image) + composeNames = append(composeNames, deploy.NewServiceRequest(image)) } - err = serviceManager.AddServicesToCompose(context.Background(), key, composeNames, env) + err = serviceManager.AddServicesToCompose(context.Background(), deploy.NewServiceRequest(key), composeNames, env) if err != nil { log.WithFields(log.Fields{ "profile": key, diff --git a/cli/cmd/stop.go b/cli/cmd/stop.go index 984802dc92..974f4f63a0 100644 --- a/cli/cmd/stop.go +++ b/cli/cmd/stop.go @@ -8,7 +8,7 @@ import ( "context" "github.com/elastic/e2e-testing/cli/config" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -55,9 +55,10 @@ func buildStopServiceCommand(srv string) *cobra.Command { Short: `Stops a ` + srv + ` service`, Long: `Stops a ` + srv + ` service, stoppping its Docker container`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() - err := serviceManager.StopCompose(context.Background(), false, []string{srv}) + err := serviceManager.StopCompose( + context.Background(), false, []deploy.ServiceRequest{deploy.NewServiceRequest(srv)}) if err != nil { log.WithFields(log.Fields{ "service": srv, @@ -73,9 +74,10 @@ func buildStopProfileCommand(key string, profile config.Profile) *cobra.Command Short: `Stops the ` + profile.Name + ` profile`, Long: `Stops the ` + profile.Name + ` profile, stopping the Services that compound it`, Run: func(cmd *cobra.Command, args []string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() - err := serviceManager.StopCompose(context.Background(), true, []string{key}) + err := serviceManager.StopCompose( + context.Background(), true, []deploy.ServiceRequest{deploy.NewServiceRequest(key)}) if err != nil { log.WithFields(log.Fields{ "profile": key, diff --git a/cli/config/compose/profiles/fleet/configurations/kibana.config.yml b/cli/config/compose/profiles/fleet/configurations/kibana.config.yml index 14af5e4868..aad1ec0a6a 100644 --- a/cli/config/compose/profiles/fleet/configurations/kibana.config.yml +++ b/cli/config/compose/profiles/fleet/configurations/kibana.config.yml @@ -16,5 +16,5 @@ xpack.fleet.registryUrl: http://package-registry:8080 xpack.fleet.agents.enabled: true xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200 xpack.fleet.agents.fleet_server.hosts: - - http://kibana:5601 + - http://fleet-server:8220 xpack.fleet.agents.tlsCheckDisabled: true diff --git a/cli/config/compose/profiles/fleet/docker-compose.yml b/cli/config/compose/profiles/fleet/docker-compose.yml index 8f693542df..b919ae05b2 100644 --- a/cli/config/compose/profiles/fleet/docker-compose.yml +++ b/cli/config/compose/profiles/fleet/docker-compose.yml @@ -29,7 +29,7 @@ services: test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null" retries: 600 interval: 1s - image: "docker.elastic.co/${kibanaDockerNamespace:-beats}/kibana:${kibanaVersion:-7.13.0-SNAPSHOT}" + image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-7.13.0-SNAPSHOT}" ports: - "5601:5601" volumes: @@ -40,3 +40,22 @@ services: test: ["CMD", "curl", "-f", "http://localhost:8080"] retries: 300 interval: 1s + + fleet-server: + image: "docker.elastic.co/beats/elastic-agent:${stackVersion:-8.0.0-SNAPSHOT}" + depends_on: + elasticsearch: + condition: service_healthy + kibana: + condition: service_healthy + healthcheck: + test: "curl -f http://127.0.0.1:8220/api/status | grep HEALTHY 2>&1 >/dev/null" + retries: 12 + interval: 5s + environment: + - "FLEET_SERVER_ENABLE=1" + - "FLEET_SERVER_INSECURE_HTTP=1" + - "KIBANA_FLEET_SETUP=1" + - "KIBANA_FLEET_HOST=http://kibana:5601" + - "FLEET_SERVER_HOST=0.0.0.0" + - "FLEET_SERVER_PORT=8220" diff --git a/cli/config/compose/services/centos/docker-compose.yml b/cli/config/compose/services/centos/docker-compose.yml deleted file mode 100644 index fa1c2f2162..0000000000 --- a/cli/config/compose/services/centos/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2.4' -services: - centos: - image: centos:${centosTag:-7} - container_name: ${centosContainerName} - entrypoint: tail -f /dev/null diff --git a/cli/config/compose/services/debian/docker-compose.yml b/cli/config/compose/services/debian/docker-compose.yml deleted file mode 100644 index cb4491e840..0000000000 --- a/cli/config/compose/services/debian/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2.4' -services: - debian: - image: debian:${debianTag:-9} - container_name: ${debianContainerName} - entrypoint: tail -f /dev/null diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml new file mode 100644 index 0000000000..ac1a563a91 --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/apm-server.yml @@ -0,0 +1,21 @@ +monitoring.enabled: true +http.enabled: true +http.port: 5067 +http.host: "0.0.0.0" +apm-server: + host: "0.0.0:8200" + secret_token: "1234" + # Enable APM Server Golang expvar support (https://golang.org/pkg/expvar/). + expvar: + enabled: true + url: "/debug/vars" + kibana: + # For APM Agent configuration in Kibana, enabled must be true. + enabled: true + host: "kibana" + username: "elastic" + password: "changeme" +output.elasticsearch: + hosts: ["http://elasticsearch:9200"] + username: "elastic" + password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml new file mode 100644 index 0000000000..e2ad548a4c --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/capabilities.yml @@ -0,0 +1,5 @@ +capabilities: +- rule: allow + input: fleet-server +- rule: deny + input: "*" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml new file mode 100644 index 0000000000..90e67b0a2a --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/credentials.yml @@ -0,0 +1,10 @@ +fleet_server: + elasticsearch: + host: "elasticsearch" + username: "elastic" + password: "changeme" +kibana: + fleet: + host: "kibana" + username: "elastic" + password: "changeme" diff --git a/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml b/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml new file mode 100644 index 0000000000..30feaf7e85 --- /dev/null +++ b/cli/config/compose/services/elastic-agent/apm-legacy/config/fleet-setup.yml @@ -0,0 +1,9 @@ +fleet: + enroll: true + force: false + insecure: true +fleet_server: + enable: true +kibana: + fleet: + setup: true diff --git a/cli/config/compose/services/centos-systemd/docker-compose.yml b/cli/config/compose/services/elastic-agent/centos/docker-compose.yml similarity index 94% rename from cli/config/compose/services/centos-systemd/docker-compose.yml rename to cli/config/compose/services/elastic-agent/centos/docker-compose.yml index 31ca6eb33d..dbe229fc10 100644 --- a/cli/config/compose/services/centos-systemd/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/centos/docker-compose.yml @@ -1,6 +1,6 @@ version: '2.4' services: - centos-systemd: + elastic-agent: image: centos/systemd:${centos_systemdTag:-latest} container_name: ${centos_systemdContainerName} entrypoint: "/usr/sbin/init" diff --git a/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml b/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml new file mode 100644 index 0000000000..e4613f7f0c --- /dev/null +++ b/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.4' +services: + elastic-agent: + image: docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.0.0-SNAPSHOT} + container_name: ${elasticAgentContainerName} + depends_on: + elasticsearch: + condition: service_healthy + kibana: + condition: service_healthy + environment: + - "FLEET_SERVER_ENABLE=1" + - "FLEET_SERVER_INSECURE_HTTP=1" + - "ELASTIC_AGENT_CLOUD=1" + - "APM_SERVER_PATH=/apm-legacy/apm-server/" + - "STATE_PATH=/apm-legacy/elastic-agent/" + - "CONFIG_PATH=/apm-legacy/config/" + - "DATA_PATH=/apm-legacy/data/" + - "LOGS_PATH=/apm-legacy/logs/" + - "HOME_PATH=/apm-legacy/" + volumes: + - "${apmVolume}:/apm-legacy" + ports: + - "127.0.0.1:8220:8220" + - "127.0.0.1:8200:8200" + - "127.0.0.1:5066:5066" diff --git a/cli/config/compose/services/debian-systemd/docker-compose.yml b/cli/config/compose/services/elastic-agent/debian/docker-compose.yml similarity index 94% rename from cli/config/compose/services/debian-systemd/docker-compose.yml rename to cli/config/compose/services/elastic-agent/debian/docker-compose.yml index 1333569887..2c39275ce1 100644 --- a/cli/config/compose/services/debian-systemd/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/debian/docker-compose.yml @@ -1,6 +1,6 @@ version: '2.4' services: - debian-systemd: + elastic-agent: image: alehaa/debian-systemd:${debian_systemdTag:-stretch} container_name: ${debian_systemdContainerName} entrypoint: "/sbin/init" diff --git a/cli/config/compose/services/elastic-agent/docker-compose.yml b/cli/config/compose/services/elastic-agent/docker-compose.yml index 0ec3c0e5b1..09ab36af2d 100644 --- a/cli/config/compose/services/elastic-agent/docker-compose.yml +++ b/cli/config/compose/services/elastic-agent/docker-compose.yml @@ -9,12 +9,8 @@ services: kibana: condition: service_healthy environment: - - "FLEET_SERVER_ELASTICSEARCH_HOST=http://${elasticsearchHost:-elasticsearch}:${elasticsearchPort:-9200}" - "FLEET_SERVER_ENABLE=${fleetServerMode:-0}" - "FLEET_SERVER_INSECURE_HTTP=${fleetServerMode:-0}" - - "FLEET_SERVER_HOST=0.0.0.0" - - "FLEET_SERVER_ELASTICSEARCH_USERNAME=elastic" - - "FLEET_SERVER_ELASTICSEARCH_PASSWORD=changeme" platform: ${elasticAgentPlatform:-linux/amd64} ports: - "127.0.0.1:8220:8220" diff --git a/cli/config/compose/services/fleet-server-centos/docker-compose.yml b/cli/config/compose/services/fleet-server-centos/docker-compose.yml index 942ad0be4b..8492ce3947 100644 --- a/cli/config/compose/services/fleet-server-centos/docker-compose.yml +++ b/cli/config/compose/services/fleet-server-centos/docker-compose.yml @@ -6,4 +6,5 @@ services: entrypoint: "/usr/sbin/init" privileged: true volumes: + - ${fleet_server_centosAgentBinarySrcPath:-.}:${fleet_server_centosAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/cli/config/compose/services/fleet-server-debian/docker-compose.yml b/cli/config/compose/services/fleet-server-debian/docker-compose.yml index 8b2f3c3d09..bb86c67f83 100644 --- a/cli/config/compose/services/fleet-server-debian/docker-compose.yml +++ b/cli/config/compose/services/fleet-server-debian/docker-compose.yml @@ -6,4 +6,5 @@ services: entrypoint: "/sbin/init" privileged: true volumes: + - ${fleet_server_debianAgentBinarySrcPath:-.}:${fleet_server_debianAgentBinaryTargetPath:-/tmp} - /sys/fs/cgroup:/sys/fs/cgroup:ro diff --git a/cli/config/config.go b/cli/config/config.go index caafaa7575..569ee254d5 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -130,15 +130,19 @@ func Init() { } shell.CheckInstalledSoftware(binaries...) - InitConfig() + initConfig() } -// InitConfig initialises configuration -func InitConfig() { +// initConfig initialises configuration +func initConfig() { if Op != nil { return } + newConfig(OpDir()) +} +// OpDir returns the directory to copy to +func OpDir() string { home, err := homedir.Dir() if err != nil { log.WithFields(log.Fields{ @@ -146,9 +150,7 @@ func InitConfig() { }).Fatal("Could not get current user's HOME dir") } - w := filepath.Join(home, ".op") - - newConfig(w) + return filepath.Join(home, ".op") } // PutServiceEnvironment puts the environment variables for the service, replacing "SERVICE_" diff --git a/cli/config/kubernetes/README.md b/cli/config/kubernetes/README.md new file mode 100644 index 0000000000..3a8015930e --- /dev/null +++ b/cli/config/kubernetes/README.md @@ -0,0 +1,26 @@ +# K8s deployment for Elasticsearch, Kibana, and Fleet-Server + +## Requirements + +- docker +- kind (>= 0.10.0) +- kubectl (>= 1.17) + +## Deployment + +``` +kind create cluster +kubectl apply -k base +``` + +This will allow you to visit `http://localhost:5601` using username: `elastic` password: `changeme` to login + +## Local Ingress (Optional) + +If a local ingress is preferred, apply the following to setup nginx-ingress + +``` +kubectl apply -k overlays/local +``` + +This will allow you to reach the Kibana endpoint at `http://localhost` diff --git a/cli/config/kubernetes/base/elasticsearch/configmap.yaml b/cli/config/kubernetes/base/elasticsearch/configmap.yaml new file mode 100644 index 0000000000..98db4446cb --- /dev/null +++ b/cli/config/kubernetes/base/elasticsearch/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-config +data: + ES_JAVA_OPTS: "-Xms1g -Xmx1g" + network.host: "" + transport.host: "127.0.0.1" + http.host: "0.0.0.0" + indices.id_field_data.enabled: 'true' + xpack.license.self_generated.type: "trial" + xpack.security.enabled: 'true' + xpack.security.authc.api_key.enabled: 'true' + ELASTIC_USERNAME: "elastic" + ELASTIC_PASSWORD: "changeme" diff --git a/cli/config/kubernetes/base/elasticsearch/deployment.yaml b/cli/config/kubernetes/base/elasticsearch/deployment.yaml new file mode 100644 index 0000000000..271cb0742c --- /dev/null +++ b/cli/config/kubernetes/base/elasticsearch/deployment.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch + labels: + app: elasticsearch +spec: + replicas: 1 + selector: + matchLabels: + app: elasticsearch + template: + metadata: + labels: + app: elasticsearch + spec: + containers: + - name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-SNAPSHOT + envFrom: + - configMapRef: + name: elasticsearch-config + ports: + - containerPort: 9200 + name: client diff --git a/cli/config/kubernetes/base/elasticsearch/kustomization.yaml b/cli/config/kubernetes/base/elasticsearch/kustomization.yaml new file mode 100644 index 0000000000..81d586a3cd --- /dev/null +++ b/cli/config/kubernetes/base/elasticsearch/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- deployment.yaml +- service.yaml +- configmap.yaml diff --git a/cli/config/kubernetes/base/elasticsearch/service.yaml b/cli/config/kubernetes/base/elasticsearch/service.yaml new file mode 100644 index 0000000000..f43cdd3f74 --- /dev/null +++ b/cli/config/kubernetes/base/elasticsearch/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + service: elasticsearch +spec: + type: NodePort + ports: + - port: 9200 + name: client + selector: + app: elasticsearch diff --git a/cli/config/kubernetes/base/fleet-server/cluster-role-binding.yaml b/cli/config/kubernetes/base/fleet-server/cluster-role-binding.yaml new file mode 100644 index 0000000000..c0b3d25536 --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: fleet-server-cluster-role-binding +subjects: +- kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: ClusterRole + name: fleet-server-cluster-role + apiGroup: rbac.authorization.k8s.io diff --git a/cli/config/kubernetes/base/fleet-server/cluster-role.yaml b/cli/config/kubernetes/base/fleet-server/cluster-role.yaml new file mode 100644 index 0000000000..3e1bca2f3e --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/cluster-role.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: fleet-server-cluster-role + labels: + app: fleet-server +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + verbs: ["get", "list", "watch"] diff --git a/cli/config/kubernetes/base/fleet-server/deployment.yaml b/cli/config/kubernetes/base/fleet-server/deployment.yaml new file mode 100644 index 0000000000..94597ccd4a --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fleet-server + labels: + app: fleet-server +spec: + replicas: 1 + selector: + matchLabels: + app: fleet-server + template: + metadata: + labels: + app: fleet-server + spec: + containers: + - name: fleet-server + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + env: + - name: FLEET_SERVER_ENABLE + value: "1" + - name: FLEET_SERVER_INSECURE_HTTP + value: "1" + - name: KIBANA_FLEET_SETUP + value: "1" + - name: KIBANA_FLEET_HOST + value: "http://kibana:5601" + - name: FLEET_SERVER_HOST + value: "0.0.0.0" + - name: FLEET_SERVER_PORT + value: "8220" diff --git a/cli/config/kubernetes/base/fleet-server/kustomization.yaml b/cli/config/kubernetes/base/fleet-server/kustomization.yaml new file mode 100644 index 0000000000..f4208b1288 --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- deployment.yaml +- service.yaml +- role.yaml +- role-binding.yaml +- cluster-role.yaml +- cluster-role-binding.yaml diff --git a/cli/config/kubernetes/base/fleet-server/role-binding.yaml b/cli/config/kubernetes/base/fleet-server/role-binding.yaml new file mode 100644 index 0000000000..f0b6586e63 --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: fleet-server-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: fleet-server-role +subjects: +- kind: ServiceAccount + name: default + namespace: default diff --git a/cli/config/kubernetes/base/fleet-server/role.yaml b/cli/config/kubernetes/base/fleet-server/role.yaml new file mode 100644 index 0000000000..540e200627 --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/role.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: fleet-server-role + namespace: default + labels: + app: fleet-server +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update", "list", "watch", "update", "patch", "delete"] diff --git a/cli/config/kubernetes/base/fleet-server/service.yaml b/cli/config/kubernetes/base/fleet-server/service.yaml new file mode 100644 index 0000000000..ddbf6f8e90 --- /dev/null +++ b/cli/config/kubernetes/base/fleet-server/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: fleet-server + labels: + service: fleet-server +spec: + type: NodePort + selector: + app: fleet-server + ports: + - port: 8220 + name: http diff --git a/cli/config/kubernetes/base/kibana/configmap.yaml b/cli/config/kubernetes/base/kibana/configmap.yaml new file mode 100644 index 0000000000..34576b6eab --- /dev/null +++ b/cli/config/kubernetes/base/kibana/configmap.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana +data: + kibana.yml: |- + --- + server.name: kibana + server.host: "0.0.0.0" + + elasticsearch.hosts: [ "http://elasticsearch:9200" ] + elasticsearch.username: elastic + elasticsearch.password: changeme + monitoring.ui.container.elasticsearch.enabled: true + + xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" + + xpack.fleet.enabled: true + xpack.fleet.registryUrl: http://package-registry:8080 + xpack.fleet.agents.enabled: true + xpack.fleet.agents.elasticsearch.host: http://elasticsearch:9200 + xpack.fleet.agents.fleet_server.hosts: ["http://fleet-server:8220"] + xpack.fleet.agents.kibana.host: "http://kibana:5601" + xpack.fleet.agents.tlsCheckDisabled: true diff --git a/cli/config/kubernetes/base/kibana/deployment.yaml b/cli/config/kubernetes/base/kibana/deployment.yaml new file mode 100644 index 0000000000..abbdc1c606 --- /dev/null +++ b/cli/config/kubernetes/base/kibana/deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + labels: + app: kibana +spec: + replicas: 1 + selector: + matchLabels: + app: kibana + template: + metadata: + labels: + app: kibana + spec: + containers: + - name: kibana + image: docker.elastic.co/kibana/kibana:8.0.0-SNAPSHOT + env: + - name: ELASTICSEARCH_URL + value: http://elasticsearch:9200 + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + ports: + - containerPort: 5601 + hostPort: 5601 + name: http + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - mountPath: /usr/share/kibana/config/kibana.yml + name: config + subPath: kibana.yml + volumes: + - name: config + configMap: + name: kibana diff --git a/cli/config/kubernetes/base/kibana/kustomization.yaml b/cli/config/kubernetes/base/kibana/kustomization.yaml new file mode 100644 index 0000000000..81d586a3cd --- /dev/null +++ b/cli/config/kubernetes/base/kibana/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- deployment.yaml +- service.yaml +- configmap.yaml diff --git a/cli/config/kubernetes/base/kibana/service.yaml b/cli/config/kubernetes/base/kibana/service.yaml new file mode 100644 index 0000000000..2ed79c4a29 --- /dev/null +++ b/cli/config/kubernetes/base/kibana/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: kibana + labels: + service: kibana +spec: + type: NodePort + ports: + - port: 5601 + name: http + selector: + app: kibana diff --git a/cli/config/kubernetes/base/kustomization.yaml b/cli/config/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000000..09b83ef350 --- /dev/null +++ b/cli/config/kubernetes/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +commonLabels: + env: test +bases: +- ./elasticsearch +- ./kibana +- ./package-registry +- ./fleet-server diff --git a/cli/config/kubernetes/base/package-registry/deployment.yaml b/cli/config/kubernetes/base/package-registry/deployment.yaml new file mode 100644 index 0000000000..6dae9547a0 --- /dev/null +++ b/cli/config/kubernetes/base/package-registry/deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: package-registry + labels: + app: package-registry +spec: + replicas: 1 + selector: + matchLabels: + app: package-registry + template: + metadata: + labels: + app: package-registry + spec: + containers: + - name: package-registry + image: docker.elastic.co/package-registry/distribution:staging + livenessProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 15 + periodSeconds: 20 diff --git a/cli/config/kubernetes/base/package-registry/kustomization.yaml b/cli/config/kubernetes/base/package-registry/kustomization.yaml new file mode 100644 index 0000000000..a944d005ca --- /dev/null +++ b/cli/config/kubernetes/base/package-registry/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- deployment.yaml +- service.yaml diff --git a/cli/config/kubernetes/base/package-registry/service.yaml b/cli/config/kubernetes/base/package-registry/service.yaml new file mode 100644 index 0000000000..4d7d293c9e --- /dev/null +++ b/cli/config/kubernetes/base/package-registry/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: package-registry + labels: + service: package-registry +spec: + type: NodePort + ports: + - port: 8080 + name: package-registry + selector: + app: package-registry diff --git a/cli/config/kubernetes/kind.yaml b/cli/config/kubernetes/kind.yaml new file mode 100644 index 0000000000..32f50aba9a --- /dev/null +++ b/cli/config/kubernetes/kind.yaml @@ -0,0 +1,26 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - containerPort: 5601 + hostPort: 5601 + protocol: TCP + - containerPort: 9200 + hostPort: 9200 + protocol: TCP + - containerPort: 8220 + hostPort: 8220 + protocol: TCP diff --git a/cli/config/kubernetes/overlays/deployment.yaml b/cli/config/kubernetes/overlays/deployment.yaml new file mode 100644 index 0000000000..9ae3ed7a11 --- /dev/null +++ b/cli/config/kubernetes/overlays/deployment.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastic-agent + labels: + app: elastic-agent +spec: + replicas: 1 + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + containers: + - name: elastic-agent + image: centos/systemd:latest + command: ["/usr/sbin/init"] + securityContext: + allowPrivilegeEscalation: true + runAsUser: 0 + capabilities: + add: ["SYS_ADMIN"] diff --git a/cli/config/kubernetes/overlays/kustomization.yaml b/cli/config/kubernetes/overlays/kustomization.yaml new file mode 100644 index 0000000000..37fdeffa9b --- /dev/null +++ b/cli/config/kubernetes/overlays/kustomization.yaml @@ -0,0 +1,5 @@ +bases: + - ../../base + +resources: +- deployment.yaml diff --git a/cli/config/kubernetes/overlays/local/ingress.yaml b/cli/config/kubernetes/overlays/local/ingress.yaml new file mode 100644 index 0000000000..d894baf0aa --- /dev/null +++ b/cli/config/kubernetes/overlays/local/ingress.yaml @@ -0,0 +1,12 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: kibana-ingress +spec: + rules: + - http: + paths: + - path: / + backend: + serviceName: kibana + servicePort: 5601 diff --git a/cli/config/kubernetes/overlays/local/kustomization.yaml b/cli/config/kubernetes/overlays/local/kustomization.yaml new file mode 100644 index 0000000000..3722f06af4 --- /dev/null +++ b/cli/config/kubernetes/overlays/local/kustomization.yaml @@ -0,0 +1,6 @@ +bases: + - ../../base + +resources: + - ./ingress.yaml + - ./nginx-ingress.yaml diff --git a/cli/config/kubernetes/overlays/local/nginx-ingress.yaml b/cli/config/kubernetes/overlays/local/nginx-ingress.yaml new file mode 100644 index 0000000000..4a242567a9 --- /dev/null +++ b/cli/config/kubernetes/overlays/local/nginx-ingress.yaml @@ -0,0 +1,657 @@ + +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: NodePort + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + minReadySeconds: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.45.0@sha256:c4390c53f348c3bd4e60a5dd6a11c35799ae78c49388090140b9d72ccede1755 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + - --publish-status-address=localhost + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + ports: + - name: http + containerPort: 80 + protocol: TCP + hostPort: 80 + - name: https + containerPort: 443 + protocol: TCP + hostPort: 443 + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi + nodeSelector: + ingress-ready: 'true' + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 0 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + name: ingress-nginx-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + namespace: ingress-nginx +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.27.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.45.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/e2e/_suites/fleet/features/apm_integration.feature b/e2e/_suites/fleet/features/apm_integration.feature new file mode 100644 index 0000000000..cff83f53d7 --- /dev/null +++ b/e2e/_suites/fleet/features/apm_integration.feature @@ -0,0 +1,30 @@ +@apm_server +Feature: APM Integration +Scenarios for APM + +@install +Scenario Outline: Deploying a stand-alone agent with fleet server mode + Given a "" stand-alone agent is deployed with fleet server mode + And the stand-alone agent is listed in Fleet as "online" + When the "Elastic APM" integration is added in the policy + Then the "Elastic APM" datasource is shown in the policy as added + And the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | + + + +@cloud +Scenario Outline: Deploying a stand-alone agent with fleet server mode on cloud + When a "" stand-alone agent is deployed with fleet server mode on cloud + Then the "apm-server" process is in the "started" state on the host + + +@default +Examples: default + | image | + | default | diff --git a/e2e/_suites/fleet/features/backend_processes.feature b/e2e/_suites/fleet/features/backend_processes.feature index e4a2b1f710..4b52594aa1 100644 --- a/e2e/_suites/fleet/features/backend_processes.feature +++ b/e2e/_suites/fleet/features/backend_processes.feature @@ -6,8 +6,8 @@ Feature: Backend Processes Scenario Outline: Deploying the agent Given a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is in the "started" state on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -21,10 +21,10 @@ Examples: Debian @enroll Scenario Outline: Deploying the agent with enroll and then run on rpm and deb - Given a "" agent is deployed to Fleet with "systemd" installer + Given a "" agent is deployed to Fleet When the "elastic-agent" process is in the "started" state on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -57,8 +57,8 @@ Examples: Debian Scenario Outline: Restarting the installed agent Given a "" agent is deployed to Fleet with "tar" installer When the "elastic-agent" process is "restarted" on the host - Then there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + Then there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos @@ -75,8 +75,8 @@ Scenario Outline: Restarting the host with persistent agent restarts backen Given a "" agent is deployed to Fleet with "tar" installer When the host is restarted Then the "elastic-agent" process is in the "started" state on the host - And there are "1" instances of the "filebeat" process in the "started" state - And there are "1" instances of the "metricbeat" process in the "started" state + And there are "2" instances of the "filebeat" process in the "started" state + And there are "2" instances of the "metricbeat" process in the "started" state @centos Examples: Centos diff --git a/e2e/_suites/fleet/features/fleet_mode_agent.feature b/e2e/_suites/fleet/features/fleet_mode_agent.feature index 65c5962460..e5ccf2b2a5 100644 --- a/e2e/_suites/fleet/features/fleet_mode_agent.feature +++ b/e2e/_suites/fleet/features/fleet_mode_agent.feature @@ -21,7 +21,7 @@ Examples: Debian @enroll Scenario Outline: Deploying the agent with enroll and then run on rpm and deb - Given a "" agent is deployed to Fleet with "systemd" installer + Given a "" agent is deployed to Fleet When the "elastic-agent" process is in the "started" state on the host Then the agent is listed in Fleet as "online" And system package dashboards are listed in Fleet diff --git a/e2e/_suites/fleet/features/fleet_server.feature b/e2e/_suites/fleet/features/fleet_server.feature deleted file mode 100644 index ff863764fd..0000000000 --- a/e2e/_suites/fleet/features/fleet_server.feature +++ /dev/null @@ -1,19 +0,0 @@ -@fleet_server -Feature: Fleet Server - Scenarios for Fleet Server, where an Elasticseach and a Kibana instances are already provisioned, - so that the Agent is able to communicate with them - -@start-fleet-server -Scenario Outline: Deploying an Elastic Agent that starts Fleet Server - When a "" agent is deployed to Fleet with "tar" installer in fleet-server mode - Then the agent is listed in Fleet as "online" - -@centos -Examples: Centos -| os | -| centos | - -@debian -Examples: Debian -| os | -| debian | diff --git a/e2e/_suites/fleet/features/stand_alone_agent.feature b/e2e/_suites/fleet/features/stand_alone_agent.feature index e6f085afd9..05941e70e1 100644 --- a/e2e/_suites/fleet/features/stand_alone_agent.feature +++ b/e2e/_suites/fleet/features/stand_alone_agent.feature @@ -51,10 +51,11 @@ Examples: Ubi8 | image | | ubi8 | -@run_fleet_server -Scenario Outline: Deploying a stand-alone agent with fleet server mode +@bootstrap-fleet-server +Scenario Outline: Bootstrapping Fleet Server from a stand-alone Elastic Agent When a "" stand-alone agent is deployed with fleet server mode Then the stand-alone agent is listed in Fleet as "online" + And there are "1" instances of the "fleet-server" process in the "started" state @default Examples: default diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go index a7d54edeb8..6156f13a69 100644 --- a/e2e/_suites/fleet/fleet.go +++ b/e2e/_suites/fleet/fleet.go @@ -10,17 +10,17 @@ import ( "strings" "time" + "github.com/google/uuid" + "github.com/cenkalti/backoff/v4" "github.com/cucumber/godog" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "github.com/google/uuid" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -28,10 +28,12 @@ import ( const actionADDED = "added" const actionREMOVED = "removed" +var deployedAgentsCount = 0 + // FleetTestSuite represents the scenarios for Fleet-mode type FleetTestSuite struct { // integrations - Cleanup bool + StandAlone bool CurrentToken string // current enrollment token CurrentTokenID string // current enrollment tokenID ElasticAgentStopped bool // will be used to signal when the agent process can be called again in the tear-down stage @@ -41,29 +43,34 @@ type FleetTestSuite struct { Installers map[string]installer.ElasticAgentInstaller Integration kibana.IntegrationPackage // the installed integration Policy kibana.Policy - FleetPolicy kibana.Policy PolicyUpdatedAt string // the moment the policy was updated + FleetServerPolicy kibana.Policy Version string // current elastic-agent version kibanaClient *kibana.Client + deployer deploy.Deployment + // date controls for queries + AgentStoppedDate time.Time + RuntimeDependenciesStartDate time.Time } // afterScenario destroys the state created by a scenario func (fts *FleetTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() + defer func() { deployedAgentsCount = 0 }() - serviceName := fts.Image + serviceName := common.ElasticAgentServiceName - if log.IsLevelEnabled(log.DebugLevel) { + if !fts.StandAlone { agentInstaller := fts.getInstaller() - err := agentInstaller.PrintLogsFn(fts.Hostname) - if err != nil { - log.WithFields(log.Fields{ - "containerName": fts.Hostname, - "error": err, - }).Warn("Could not get agent logs in the container") + if log.IsLevelEnabled(log.DebugLevel) { + err := agentInstaller.PrintLogsFn(fts.Hostname) + if err != nil { + log.WithFields(log.Fields{ + "containerName": fts.Hostname, + "error": err, + }).Warn("Could not get agent logs in the container") + } } - // only call it when the elastic-agent is present if !fts.ElasticAgentStopped { err := agentInstaller.UninstallFn() @@ -71,6 +78,8 @@ func (fts *FleetTestSuite) afterScenario() { log.Warnf("Could not uninstall the agent after the scenario: %v", err) } } + } else if log.IsLevelEnabled(log.DebugLevel) { + _ = fts.getContainerLogs() } err := fts.unenrollHostname() @@ -83,7 +92,18 @@ func (fts *FleetTestSuite) afterScenario() { developerMode := shell.GetEnvBool("DEVELOPER_MODE") if !developerMode { - _ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName + "-systemd"}, common.ProfileEnv) + image := "" + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + image = agentInstaller.Image + } + + _ = fts.deployer.Remove( + []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(serviceName).WithFlavour(image), + }, + common.ProfileEnv) } else { log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") } @@ -96,40 +116,22 @@ func (fts *FleetTestSuite) afterScenario() { }).Warn("The enrollment token could not be deleted") } - // Cleanup all package policies - packagePolicies, err := fts.kibanaClient.ListPackagePolicies() - if err != nil { - log.WithFields(log.Fields{ - "err": err, - "policy": fts.FleetPolicy, - }).Error("The package policies could not be found") - } - for _, pkgPolicy := range packagePolicies { - // Do not remove the fleet server package integration otherwise fleet server fails to bootstrap - if !strings.Contains(pkgPolicy.Name, "fleet_server") && pkgPolicy.PolicyID == fts.FleetPolicy.ID { - err = fts.kibanaClient.DeleteIntegrationFromPolicy(pkgPolicy) - if err != nil { - log.WithFields(log.Fields{ - "err": err, - "packagePolicy": pkgPolicy, - }).Error("The integration could not be deleted from the configuration") - } - } - } + fts.kibanaClient.DeleteAllPolicies() // clean up fields fts.CurrentTokenID = "" fts.CurrentToken = "" fts.Image = "" fts.Hostname = "" + fts.StandAlone = false } // beforeScenario creates the state needed by a scenario func (fts *FleetTestSuite) beforeScenario() { - fts.Cleanup = false + fts.StandAlone = false fts.ElasticAgentStopped = false - fts.Version = common.AgentVersion + fts.Version = common.BeatVersion policy, err := fts.kibanaClient.GetDefaultPolicy(false) if err != nil { @@ -139,19 +141,11 @@ func (fts *FleetTestSuite) beforeScenario() { } fts.Policy = policy - - fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) - if err != nil { - log.WithFields(log.Fields{ - "err": err, - }).Warn("The default fleet server policy could not be obtained") - } - fts.FleetPolicy = fleetPolicy - } func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { - s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstallerInFleetMode) + s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet) + s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller) s.Step(`^a "([^"]*)" agent "([^"]*)" is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller) s.Step(`^agent is in version "([^"]*)"$`, fts.agentInVersion) s.Step(`^agent is upgraded to version "([^"]*)"$`, fts.anAgentIsUpgraded) @@ -177,8 +171,40 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) { s.Step(`^the policy is updated to have "([^"]*)" in "([^"]*)" mode$`, fts.thePolicyIsUpdatedToHaveMode) s.Step(`^the policy will reflect the change in the Security App$`, fts.thePolicyWillReflectTheChangeInTheSecurityApp) - // fleet server steps - s.Step(`^a "([^"]*)" agent is deployed to Fleet with "([^"]*)" installer in fleet-server mode$`, fts.anAgentIsDeployedToFleetWithInstallerInFleetMode) + // stand-alone only steps + s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, fts.aStandaloneAgentIsDeployed) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, fts.bootstrapFleetServerFromAStandaloneAgent) + s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode on cloud$`, fts.aStandaloneAgentIsDeployedWithFleetServerModeOnCloud) + s.Step(`^there is new data in the index from agent$`, fts.thereIsNewDataInTheIndexFromAgent) + s.Step(`^the "([^"]*)" docker container is stopped$`, fts.theDockerContainerIsStopped) + s.Step(`^there is no new data in the index after agent shuts down$`, fts.thereIsNoNewDataInTheIndexAfterAgentShutsDown) + s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, fts.theStandaloneAgentIsListedInFleetWithStatus) +} + +func (fts *FleetTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { + waitForAgents := func() error { + agents, err := fts.kibanaClient.ListAgents() + if err != nil { + return err + } + + if len(agents) == 0 { + return errors.New("No agents found") + } + + agentZero := agents[0] + hostname := agentZero.LocalMetadata.Host.HostName + + return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) + } + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) + + err := backoff.Retry(waitForAgents, exp) + if err != nil { + return err + } + return nil } func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, version, installerType string) error { @@ -206,7 +232,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion default: version = common.AgentStaleVersion } @@ -215,7 +241,7 @@ func (fts *FleetTestSuite) anStaleAgentIsDeployedToFleetWithInstaller(image, ver // prepare installer for stale version if fts.Version != agentVersionBackup { - i := installer.GetElasticAgentInstaller(image, installerType, fts.Version) + i := installer.GetElasticAgentInstaller(image, installerType, fts.Version, deployedAgentsCount) fts.Installers[fmt.Sprintf("%s-%s-%s", image, installerType, version)] = i } @@ -228,7 +254,7 @@ func (fts *FleetTestSuite) installCerts() error { log.WithFields(log.Fields{ "installer": agentInstaller, "version": fts.Version, - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, }).Error("No installer found") return errors.New("no installer found") @@ -237,7 +263,7 @@ func (fts *FleetTestSuite) installCerts() error { err := agentInstaller.InstallCertsFn() if err != nil { log.WithFields(log.Fields{ - "agentVersion": common.AgentVersion, + "agentVersion": common.BeatVersion, "agentStaleVersion": common.AgentStaleVersion, "error": err, "installer": agentInstaller, @@ -254,9 +280,9 @@ func (fts *FleetTestSuite) anAgentIsUpgraded(desiredVersion string) error { case "stale": desiredVersion = common.AgentStaleVersion case "latest": - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion default: - desiredVersion = common.AgentVersion + desiredVersion = common.BeatVersion } return fts.kibanaClient.UpgradeAgent(fts.Hostname, desiredVersion) @@ -267,7 +293,7 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { case "stale": version = common.AgentStaleVersion case "latest": - version = common.AgentVersion + version = common.BeatVersion } agentInVersionFn := func() error { @@ -288,36 +314,45 @@ func (fts *FleetTestSuite) agentInVersion(version string) error { return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) return backoff.Retry(agentInVersionFn, exp) } -// supported installers: tar, systemd +// this step infers the installer type from the underlying OS image +// supported images: centos and debian +func (fts *FleetTestSuite) anAgentIsDeployedToFleet(image string) error { + installerType := "rpm" + if image == "debian" { + installerType = "deb" + } + + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType) +} + +// supported installers: tar, rpm, deb func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(image string, installerType string) error { - return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, false) + return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType) } -func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(image string, installerType string, bootstrapFleetServer bool) error { +func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(image string, installerType string) error { log.WithFields(log.Fields{ - "bootstrapFleetServer": bootstrapFleetServer, - "image": image, - "installer": installerType, - }).Trace("Deploying an agent to Fleet with base image and fleet server") + "image": image, + "installer": installerType, + }).Trace("Deploying an agent to Fleet with base image using an already bootstrapped Fleet Server") + + deployedAgentsCount++ fts.Image = image fts.InstallerType = installerType agentInstaller := fts.getInstaller() - profile := agentInstaller.Profile // name of the runtime dependencies compose file - - serviceName := common.ElasticAgentServiceName // name of the service - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", serviceName, 1) // name of the container + containerName := fts.getContainerName(agentInstaller) // name of the container // enroll the agent with a new token - enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(fts.FleetPolicy) + enrollmentKey, err := fts.kibanaClient.CreateEnrollmentAPIKey(fts.Policy) if err != nil { return err } @@ -325,9 +360,8 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i fts.CurrentTokenID = enrollmentKey.ID var fleetConfig *kibana.FleetConfig - fleetConfig, err = deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, bootstrapFleetServer) + fleetConfig, err = deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken) - fts.Cleanup = true if err != nil { return err } @@ -341,7 +375,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i } // get container hostname once - hostname, err := docker.GetContainerHostname(containerName) + hostname, err := deploy.GetContainerHostname(containerName) if err != nil { return err } @@ -350,16 +384,30 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i return err } +// getContainerName returns the current container name for the service: +// we are using the Docker client instead of docker-compose because it does not support +// returning the output of a command: it simply returns error level +func (fts *FleetTestSuite) getContainerName(i installer.ElasticAgentInstaller) string { + return fmt.Sprintf("%s_%s_%d", i.Profile, common.ElasticAgentServiceName, deployedAgentsCount) +} + +// getServiceName returns the current service name, the one defined at the docker compose +func (fts *FleetTestSuite) getServiceName(i installer.ElasticAgentInstaller) string { + return i.Image +} + func (fts *FleetTestSuite) getInstaller() installer.ElasticAgentInstaller { + key := fmt.Sprintf("%s-%s-%s-%d", fts.Image, fts.InstallerType, fts.Version, deployedAgentsCount) // check if the agent is already cached - if i, exists := fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version]; exists { + if i, exists := fts.Installers[key]; exists { return i } - agentInstaller := installer.GetElasticAgentInstaller(fts.Image, fts.InstallerType, fts.Version) + // setting current index for the installer + agentInstaller := installer.GetElasticAgentInstaller(fts.Image, fts.InstallerType, fts.Version, deployedAgentsCount) // cache the new installer - fts.Installers[fts.Image+"-"+fts.InstallerType+"-"+fts.Version] = agentInstaller + fts.Installers[key] = agentInstaller return agentInstaller } @@ -371,17 +419,20 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st serviceName := agentInstaller.Service // name of the service + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image) + if state == "started" { - return installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start") + return installer.SystemctlRun(profileService, imageService, serviceName, "start") } else if state == "restarted" { - err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop") + err := installer.SystemctlRun(profileService, imageService, serviceName, "stop") if err != nil { return err } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) - err = installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "start") + err = installer.SystemctlRun(profileService, imageService, serviceName, "start") if err != nil { return err } @@ -407,7 +458,7 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st "process": process, }).Trace("Stopping process on the service") - err := installer.SystemctlRun(profile, agentInstaller.Image, serviceName, "stop") + err := installer.SystemctlRun(profileService, imageService, serviceName, "stop") if err != nil { log.WithFields(log.Fields{ "action": state, @@ -419,13 +470,9 @@ func (fts *FleetTestSuite) processStateChangedOnTheHost(process string, state st return err } - // name of the container for the service: - // we are using the Docker client instead of docker-compose - // because it does not support returning the output of a - // command: it simply returns error level - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", common.ElasticAgentServiceName, 1) + containerName := fts.getContainerName(agentInstaller) - return docker.CheckProcessStateOnTheHost(containerName, process, "stopped", 1, common.TimeoutFactor) + return CheckProcessState(fts.deployer, containerName, process, "stopped", 1, utils.TimeoutFactor) } func (fts *FleetTestSuite) setup() error { @@ -450,10 +497,10 @@ func theAgentIsListedInFleetWithStatus(desiredStatus string, hostname string) er if err != nil { return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentOnlineFn := func() error { agentID, err := kibanaClient.GetAgentIDByHostname(hostname) @@ -520,13 +567,7 @@ func theAgentIsListedInFleetWithStatus(desiredStatus string, hostname string) er func (fts *FleetTestSuite) theFileSystemAgentFolderIsEmpty() error { agentInstaller := fts.getInstaller() - profile := agentInstaller.Profile // name of the runtime dependencies compose file - - // name of the container for the service: - // we are using the Docker client instead of docker-compose - // because it does not support returning the output of a - // command: it simply returns error level - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", common.ElasticAgentServiceName, 1) + containerName := fts.getContainerName(agentInstaller) content, err := agentInstaller.ListElasticAgentWorkingDirContent(containerName) if err != nil { @@ -543,32 +584,31 @@ func (fts *FleetTestSuite) theFileSystemAgentFolderIsEmpty() error { func (fts *FleetTestSuite) theHostIsRestarted() error { agentInstaller := fts.getInstaller() - profile := agentInstaller.Profile // name of the runtime dependencies compose file - image := agentInstaller.Image // image of the service - service := agentInstaller.Service // name of the service - - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", common.ElasticAgentServiceName, 1) + containerName := fts.getContainerName(agentInstaller) _, err := shell.Execute(context.Background(), ".", "docker", "stop", containerName) if err != nil { log.WithFields(log.Fields{ - "image": image, - "service": service, + "containerName": containerName, + "image": agentInstaller.Image, + "service": agentInstaller.Service, }).Error("Could not stop the service") } - utils.Sleep(time.Duration(common.TimeoutFactor) * 10 * time.Second) + utils.Sleep(time.Duration(utils.TimeoutFactor) * 10 * time.Second) _, err = shell.Execute(context.Background(), ".", "docker", "start", containerName) if err != nil { log.WithFields(log.Fields{ - "image": image, - "service": service, + "containerName": containerName, + "image": agentInstaller.Image, + "service": agentInstaller.Service, }).Error("Could not start the service") } log.WithFields(log.Fields{ - "image": image, - "service": service, + "containerName": containerName, + "image": agentInstaller.Image, + "service": agentInstaller.Service, }).Debug("The service has been restarted") return nil } @@ -577,10 +617,10 @@ func (fts *FleetTestSuite) systemPackageDashboardsAreListedInFleet() error { log.Trace("Checking system Package dashboards in Fleet") dataStreamsCount := 0 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) countDataStreamsFn := func() error { dataStreams, err := fts.kibanaClient.GetDataStreams() @@ -646,7 +686,7 @@ func (fts *FleetTestSuite) theAgentIsReenrolledOnTheHost() error { // during an unenroll the fleet server exits as there is no longer // and agent id associated with the enrollment. When fleet server // restarts it needs a new agent to associate with the boostrap - cfg, err := kibana.NewFleetConfig(fts.CurrentToken, true, false) + cfg, err := kibana.NewFleetConfig(fts.CurrentToken) if err != nil { return err } @@ -678,50 +718,18 @@ func (fts *FleetTestSuite) theEnrollmentTokenIsRevoked() error { return nil } -func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { - log.WithFields(log.Fields{ - "policyID": fts.FleetPolicy.ID, - "package": packageName, - }).Trace("Checking if the policy shows the package added") - - maxTimeout := time.Minute - retryCount := 1 - - exp := common.GetExponentialBackOff(maxTimeout) - - configurationIsPresentFn := func() error { - packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.FleetPolicy) - if err != nil { - log.WithFields(log.Fields{ - "packagePolicy": packagePolicy, - "policy": fts.FleetPolicy, - "retry": retryCount, - "error": err, - }).Warn("The integration was not found in the policy") - retryCount++ - return err - } - - retryCount++ - return err - } - - err := backoff.Retry(configurationIsPresentFn, exp) - if err != nil { - return err - } - - return nil +func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { + return theIntegrationIsOperatedInThePolicy(fts.kibanaClient, fts.Policy, packageName, action) } -func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName string, action string) error { +func theIntegrationIsOperatedInThePolicy(client *kibana.Client, policy kibana.Policy, packageName string, action string) error { log.WithFields(log.Fields{ "action": action, - "policy": fts.FleetPolicy, + "policy": policy, "package": packageName, }).Trace("Doing an operation for a package on a policy") - integration, err := fts.kibanaClient.GetIntegrationByPackageName(packageName) + integration, err := client.GetIntegrationByPackageName(packageName) if err != nil { return err } @@ -731,44 +739,20 @@ func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName strin Name: integration.Name, Description: integration.Title, Namespace: "default", - PolicyID: fts.FleetPolicy.ID, + PolicyID: policy.ID, Enabled: true, Package: integration, Inputs: []kibana.Input{}, } + packageDataStream.Inputs = inputs(integration.Name) - if strings.EqualFold(integration.Name, "linux") { - packageDataStream.Inputs = []kibana.Input{ - { - Type: "linux/metrics", - Enabled: true, - Streams: []interface{}{ - map[string]interface{}{ - "id": "linux/metrics-linux.memory-" + uuid.New().String(), - "enabled": true, - "data_stream": map[string]interface{}{ - "dataset": "linux.memory", - "type": "metrics", - }, - }, - }, - Vars: map[string]kibana.Var{ - "period": { - Value: "1s", - Type: "string", - }, - }, - }, - } - } - - return fts.kibanaClient.AddIntegrationToPolicy(packageDataStream) + return client.AddIntegrationToPolicy(packageDataStream) } else if strings.ToLower(action) == actionREMOVED { - packageDataStream, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(integration.Name, fts.FleetPolicy) + packageDataStream, err := client.GetIntegrationFromAgentPolicy(integration.Name, policy) if err != nil { return err } - return fts.kibanaClient.DeleteIntegrationFromPolicy(packageDataStream) + return client.DeleteIntegrationFromPolicy(packageDataStream) } return nil @@ -777,10 +761,10 @@ func (fts *FleetTestSuite) theIntegrationIsOperatedInThePolicy(packageName strin func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() error { log.Trace("Checking if the hostname is not shown in the Administration view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { host, err := fts.kibanaClient.IsAgentListedInSecurityApp(fts.Hostname) @@ -817,10 +801,10 @@ func (fts *FleetTestSuite) theHostNameIsNotShownInTheAdminViewInTheSecurityApp() func (fts *FleetTestSuite) theHostNameIsShownInTheAdminViewInTheSecurityApp(status string) error { log.Trace("Checking if the hostname is shown in the Admin view in the Security App") - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) agentListedInSecurityFn := func() error { matches, err := fts.kibanaClient.IsAgentListedInSecurityAppWithStatus(fts.Hostname, status) @@ -857,13 +841,8 @@ func (fts *FleetTestSuite) theHostNameIsShownInTheAdminViewInTheSecurityApp(stat return nil } -func (fts *FleetTestSuite) anIntegrationIsSuccessfullyDeployedWithAgentAndInstaller(integration string, image string, agentInstaller string) error { - err := fts.anAgentIsDeployedToFleetWithInstallerInFleetMode(image, agentInstaller) - if err != nil { - return err - } - - err = fts.theAgentIsListedInFleetWithStatus("online") +func (fts *FleetTestSuite) anIntegrationIsSuccessfullyDeployedWithAgentAndInstaller(integration string, image string, installerType string) error { + err := fts.anAgentIsDeployedToFleetWithInstaller(image, installerType) if err != nil { return err } @@ -877,10 +856,10 @@ func (fts *FleetTestSuite) thePolicyResponseWillBeShownInTheSecurityApp() error return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { listed, err := fts.kibanaClient.IsPolicyResponseListedInSecurityApp(agentID) @@ -937,7 +916,7 @@ func (fts *FleetTestSuite) thePolicyIsUpdatedToHaveMode(name string, mode string return godog.ErrPending } - packageDS, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.FleetPolicy) + packageDS, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.Policy) if err != nil { return err @@ -971,15 +950,15 @@ func (fts *FleetTestSuite) thePolicyWillReflectTheChangeInTheSecurityApp() error return err } - pkgPolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.FleetPolicy) + pkgPolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy("endpoint", fts.Policy) if err != nil { return err } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 retryCount := 1 - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) getEventsFn := func() error { err := fts.kibanaClient.GetAgentEvents("endpoint-security", agentID, pkgPolicy.ID, fts.PolicyUpdatedAt) @@ -1033,13 +1012,15 @@ func (fts *FleetTestSuite) theVersionOfThePackageIsInstalled(version string, pac func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error { log.Trace("Enrolling a new agent with an revoked token") + // increase the number of agents + deployedAgentsCount++ + agentInstaller := fts.getInstaller() - profile := agentInstaller.Profile // name of the runtime dependencies compose file + containerName := fts.getContainerName(agentInstaller) // name of the new container - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, fts.Image+"-systemd", common.ElasticAgentServiceName, 2) // name of the new container + fleetConfig, err := deployAgentToFleet(agentInstaller, fts.deployer, containerName, fts.CurrentToken) - fleetConfig, err := deployAgentToFleet(agentInstaller, containerName, fts.CurrentToken, false) // the installation process for TAR includes the enrollment if agentInstaller.InstallerType != "tar" { if err != nil { @@ -1177,7 +1158,7 @@ func (fts *FleetTestSuite) checkDataStream() error { return err } -func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containerName string, token string, bootstrapFleetServer bool) (*kibana.FleetConfig, error) { +func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, deployer deploy.Deployment, containerName string, token string) (*kibana.FleetConfig, error) { profile := agentInstaller.Profile // name of the runtime dependencies compose file service := agentInstaller.Service // name of the service serviceTag := agentInstaller.Tag // docker tag of the service @@ -1189,9 +1170,11 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe // we are setting the container name because Centos service could be reused by any other test suite common.ProfileEnv[envVarsPrefix+"ContainerName"] = containerName - serviceManager := compose.NewServiceManager() + agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(agentInstaller.Image).WithScale(deployedAgentsCount) + + services := []deploy.ServiceRequest{deploy.NewServiceRequest(profile), agentService} - err := serviceManager.AddServicesToCompose(context.Background(), profile, []string{service}, common.ProfileEnv) + err := deployer.Add(services, common.ProfileEnv) if err != nil { log.WithFields(log.Fields{ "service": service, @@ -1204,7 +1187,7 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe targetFile := "/" // copy downloaded agent to the root dir of the container - err = docker.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar) + err = deploy.CopyFileToContainer(context.Background(), containerName, agentInstaller.BinaryPath, targetFile, isTar) if err != nil { return nil, err } @@ -1214,7 +1197,7 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return nil, err } - cfg, cfgError := kibana.NewFleetConfig(token, bootstrapFleetServer, false) + cfg, cfgError := kibana.NewFleetConfig(token) if cfgError != nil { return nil, cfgError } @@ -1226,3 +1209,75 @@ func deployAgentToFleet(agentInstaller installer.ElasticAgentInstaller, containe return cfg, agentInstaller.PostInstallFn() } + +func inputs(integration string) []kibana.Input { + switch integration { + case "apm": + return []kibana.Input{ + { + Type: "apm", + Enabled: true, + Streams: []interface{}{}, + Vars: map[string]kibana.Var{ + "apm-server": { + Value: "host", + Type: "localhost:8200", + }, + }, + }, + } + case "linux": + return []kibana.Input{ + { + Type: "linux/metrics", + Enabled: true, + Streams: []interface{}{ + map[string]interface{}{ + "id": "linux/metrics-linux.memory-" + uuid.New().String(), + "enabled": true, + "data_stream": map[string]interface{}{ + "dataset": "linux.memory", + "type": "metrics", + }, + }, + }, + Vars: map[string]kibana.Var{ + "period": { + Value: "1s", + Type: "string", + }, + }, + }, + } + } + return []kibana.Input{} +} + +func (fts *FleetTestSuite) getContainerLogs() error { + serviceManager := deploy.NewServiceManager() + + image := "" + if !fts.StandAlone { + agentInstaller := fts.getInstaller() + image = agentInstaller.Image + } + + profile := deploy.NewServiceRequest(common.FleetProfileName) + serviceName := common.ElasticAgentServiceName + + services := []deploy.ServiceRequest{ + profile, // profile name + deploy.NewServiceRequest(serviceName).WithFlavour(image), // agent service + } + err := serviceManager.RunCommand(profile, services, []string{"logs", serviceName}, common.ProfileEnv) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "service": serviceName, + }).Error("Could not retrieve Elastic Agent logs") + + return err + } + + return nil +} diff --git a/e2e/_suites/fleet/fleet_server.go b/e2e/_suites/fleet/fleet_server.go deleted file mode 100644 index e634ade08e..0000000000 --- a/e2e/_suites/fleet/fleet_server.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package main - -func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerInFleetMode(image string, installerType string) error { - fts.ElasticAgentStopped = true - return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(image, installerType, true) -} diff --git a/e2e/_suites/fleet/ingest_manager_test.go b/e2e/_suites/fleet/ingest_manager_test.go index d378f17742..e9df20c969 100644 --- a/e2e/_suites/fleet/ingest_manager_test.go +++ b/e2e/_suites/fleet/ingest_manager_test.go @@ -5,18 +5,14 @@ package main import ( - "context" "os" - "strings" "time" "github.com/cucumber/godog" "github.com/cucumber/messages-go/v10" "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" - "github.com/elastic/e2e-testing/internal/elasticsearch" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" @@ -34,43 +30,14 @@ func setUpSuite() { log.Error(err) os.Exit(1) } + + common.Provider = shell.GetEnv("PROVIDER", common.Provider) developerMode := shell.GetEnvBool("DEVELOPER_MODE") if developerMode { log.Info("Running in Developer mode 💻: runtime dependencies between different test runs will be reused to speed up dev cycle") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(common.AgentVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersionBase, - }).Fatal("Failed to get agent base version, aborting") - } - common.AgentVersionBase = v - - common.TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", common.TimeoutFactor) - common.AgentVersion = shell.GetEnv("BEAT_VERSION", common.AgentVersionBase) - - // check if version is an alias - v, err = utils.GetElasticArtifactVersion(common.AgentVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.AgentVersion, - }).Fatal("Failed to get agent version, aborting") - } - common.AgentVersion = v - - common.StackVersion = shell.GetEnv("STACK_VERSION", common.StackVersion) - v, err = utils.GetElasticArtifactVersion(common.StackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": common.StackVersion, - }).Fatal("Failed to get stack version, aborting") - } - common.StackVersion = v + common.InitVersions() common.KibanaVersion = shell.GetEnv("KIBANA_VERSION", "") if common.KibanaVersion == "" { @@ -88,127 +55,75 @@ func setUpSuite() { imts = IngestManagerTestSuite{ Fleet: &FleetTestSuite{ kibanaClient: kibanaClient, + deployer: deploy.New(common.Provider), Installers: map[string]installer.ElasticAgentInstaller{}, // do not pre-initialise the map }, - StandAlone: &StandAloneTestSuite{ - kibanaClient: kibanaClient, - }, } } func InitializeIngestManagerTestScenario(ctx *godog.ScenarioContext) { ctx.BeforeScenario(func(*messages.Pickle) { log.Trace("Before Fleet scenario") - - imts.StandAlone.Cleanup = false - imts.Fleet.beforeScenario() }) ctx.AfterScenario(func(*messages.Pickle, error) { log.Trace("After Fleet scenario") - - if imts.StandAlone.Cleanup { - imts.StandAlone.afterScenario() - } - - if imts.Fleet.Cleanup { - imts.Fleet.afterScenario() - } + imts.Fleet.afterScenario() }) ctx.Step(`^the "([^"]*)" process is in the "([^"]*)" state on the host$`, imts.processStateOnTheHost) ctx.Step(`^there are "([^"]*)" instances of the "([^"]*)" process in the "([^"]*)" state$`, imts.thereAreInstancesOfTheProcessInTheState) imts.Fleet.contributeSteps(ctx) - imts.StandAlone.contributeSteps(ctx) } func InitializeIngestManagerTestSuite(ctx *godog.TestSuiteContext) { - serviceManager := compose.NewServiceManager() + developerMode := shell.GetEnvBool("DEVELOPER_MODE") ctx.BeforeSuite(func() { setUpSuite() - log.Trace("Installing Fleet runtime dependencies") - - common.ProfileEnv = map[string]string{ - "kibanaVersion": common.KibanaVersion, - "stackVersion": common.StackVersion, - } + log.Trace("Bootstrapping Fleet Server") if !shell.GetEnvBool("SKIP_PULL") { images := []string{ - "docker.elastic.co/beats/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/beats/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/beats/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/beats/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/elasticsearch/elasticsearch:" + common.StackVersion, "docker.elastic.co/kibana/kibana:" + common.KibanaVersion, - "docker.elastic.co/observability-ci/elastic-agent:" + common.AgentVersion, - "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.AgentVersion, + "docker.elastic.co/observability-ci/elastic-agent:" + common.BeatVersion, + "docker.elastic.co/observability-ci/elastic-agent-ubi8:" + common.BeatVersion, "docker.elastic.co/observability-ci/elasticsearch:" + common.StackVersion, "docker.elastic.co/observability-ci/elasticsearch-ubi8:" + common.StackVersion, "docker.elastic.co/observability-ci/kibana:" + common.KibanaVersion, "docker.elastic.co/observability-ci/kibana-ubi8:" + common.KibanaVersion, } - docker.PullImages(images) - } - - common.ProfileEnv["kibanaDockerNamespace"] = "kibana" - if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) { - // because it comes from a PR - common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci" + deploy.PullImages(images) } - profile := common.FleetProfileName - err := serviceManager.RunCompose(context.Background(), true, []string{profile}, common.ProfileEnv) - if err != nil { - log.WithFields(log.Fields{ - "profile": profile, - }).Fatal("Could not run the runtime dependencies for the profile.") - } - - minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute - healthy, err := elasticsearch.WaitForElasticsearch(context.Background(), minutesToBeHealthy) - if !healthy { - log.WithFields(log.Fields{ - "error": err, - "minutes": minutesToBeHealthy, - }).Fatal("The Elasticsearch cluster could not get the healthy status") - } - - kibanaClient, err := kibana.NewClient() - if err != nil { - log.WithFields(log.Fields{ - "error": err, - }).Fatal("Unable to create kibana client") - } - - healthyKibana, err := kibanaClient.WaitForReady(minutesToBeHealthy) - if !healthyKibana { - log.WithFields(log.Fields{ - "error": err, - "minutes": minutesToBeHealthy, - }).Fatal("The Kibana instance could not get the healthy status") - } - - imts.Fleet.setup() + deployer := deploy.New(common.Provider) + deployer.Bootstrap(func() error { + kibanaClient, err := kibana.NewClient() + if err != nil { + log.WithField("error", err).Fatal("Unable to create kibana client") + } + err = kibanaClient.WaitForFleet() + if err != nil { + log.WithField("error", err).Fatal("Fleet could not be initialized") + } + return nil + }) - imts.StandAlone.RuntimeDependenciesStartDate = time.Now().UTC() + imts.Fleet.Version = common.BeatVersionBase + imts.Fleet.RuntimeDependenciesStartDate = time.Now().UTC() }) ctx.AfterSuite(func() { - developerMode := shell.GetEnvBool("DEVELOPER_MODE") if !developerMode { log.Debug("Destroying Fleet runtime dependencies") - profile := common.FleetProfileName - - err := serviceManager.StopCompose(context.Background(), true, []string{profile}) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "profile": profile, - }).Warn("Could not destroy the runtime dependencies for the profile.") - } + deployer := deploy.New(common.Provider) + deployer.Destroy() } installers := imts.Fleet.Installers diff --git a/e2e/_suites/fleet/stand-alone.go b/e2e/_suites/fleet/stand-alone.go index fd4d723cfe..b60d0ff10d 100644 --- a/e2e/_suites/fleet/stand-alone.go +++ b/e2e/_suites/fleet/stand-alone.go @@ -7,99 +7,98 @@ package main import ( "context" "fmt" + "path" "strings" "time" "github.com/cenkalti/backoff/v4" - "github.com/cucumber/godog" + "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" - "github.com/elastic/e2e-testing/internal/elasticsearch" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/installer" - "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" - "github.com/pkg/errors" + + "github.com/elastic/e2e-testing/internal/elasticsearch" log "github.com/sirupsen/logrus" ) -// StandAloneTestSuite represents the scenarios for Stand-alone-mode -type StandAloneTestSuite struct { - Cleanup bool - Hostname string - Image string - // date controls for queries - AgentStoppedDate time.Time - RuntimeDependenciesStartDate time.Time - kibanaClient *kibana.Client +func (fts *FleetTestSuite) aStandaloneAgentIsDeployed(image string) error { + return fts.startStandAloneAgent(image, "", nil) } -// afterScenario destroys the state created by a scenario -func (sats *StandAloneTestSuite) afterScenario() { - serviceManager := compose.NewServiceManager() - serviceName := common.ElasticAgentServiceName - - if log.IsLevelEnabled(log.DebugLevel) { - _ = sats.getContainerLogs() +func (fts *FleetTestSuite) bootstrapFleetServerFromAStandaloneAgent(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err } - developerMode := shell.GetEnvBool("DEVELOPER_MODE") - if !developerMode { - _ = serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - } else { - log.WithField("service", serviceName).Info("Because we are running in development mode, the service won't be stopped") - } + fts.FleetServerPolicy = fleetPolicy + return fts.startStandAloneAgent(image, "", map[string]string{"fleetServerMode": "1"}) } -func (sats *StandAloneTestSuite) contributeSteps(s *godog.ScenarioContext) { - s.Step(`^a "([^"]*)" stand-alone agent is deployed$`, sats.aStandaloneAgentIsDeployed) - s.Step(`^a "([^"]*)" stand-alone agent is deployed with fleet server mode$`, sats.aStandaloneAgentIsDeployedWithFleetServerMode) - s.Step(`^there is new data in the index from agent$`, sats.thereIsNewDataInTheIndexFromAgent) - s.Step(`^the "([^"]*)" docker container is stopped$`, sats.theDockerContainerIsStopped) - s.Step(`^there is no new data in the index after agent shuts down$`, sats.thereIsNoNewDataInTheIndexAfterAgentShutsDown) - s.Step(`^the stand-alone agent is listed in Fleet as "([^"]*)"$`, sats.theStandaloneAgentIsListedInFleetWithStatus) +func (fts *FleetTestSuite) aStandaloneAgentIsDeployedWithFleetServerModeOnCloud(image string) error { + fleetPolicy, err := fts.kibanaClient.GetDefaultPolicy(true) + if err != nil { + return err + } + fts.FleetServerPolicy = fleetPolicy + volume := path.Join(config.OpDir(), "compose", "services", "elastic-agent", "apm-legacy") + return fts.startStandAloneAgent(image, "cloud", map[string]string{"apmVolume": volume}) } -func (sats *StandAloneTestSuite) theStandaloneAgentIsListedInFleetWithStatus(desiredStatus string) error { - waitForAgents := func() error { - agents, err := sats.kibanaClient.ListAgents() - if err != nil { - return err - } +func (fts *FleetTestSuite) thereIsNewDataInTheIndexFromAgent() error { + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + minimumHitsCount := 50 - if len(agents) == 0 { - return errors.New("No agents found") - } + result, err := searchAgentData(fts.Hostname, fts.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) + if err != nil { + return err + } - agentZero := agents[0] - hostname := agentZero.LocalMetadata.Host.HostName + log.Tracef("Search result: %v", result) - return theAgentIsListedInFleetWithStatus(desiredStatus, hostname) - } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + return elasticsearch.AssertHitsArePresent(result) +} - err := backoff.Retry(waitForAgents, exp) +func (fts *FleetTestSuite) theDockerContainerIsStopped(serviceName string) error { + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(serviceName), + } + err := fts.deployer.Remove(services, common.ProfileEnv) if err != nil { return err } + fts.AgentStoppedDate = time.Now().UTC() + return nil } -func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployedWithFleetServerMode(image string) error { - return sats.startAgent(image, map[string]string{"fleetServerMode": "1"}) -} +func (fts *FleetTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { + maxTimeout := time.Duration(30) * time.Second + minimumHitsCount := 1 -func (sats *StandAloneTestSuite) aStandaloneAgentIsDeployed(image string) error { - return sats.startAgent(image, nil) -} + result, err := searchAgentData(fts.Hostname, fts.AgentStoppedDate, minimumHitsCount, maxTimeout) + if err != nil { + if strings.Contains(err.Error(), "type:index_not_found_exception") { + return err + } + + log.WithFields(log.Fields{ + "error": err, + }).Info("No documents were found for the Agent in the index after it stopped") + return nil + } -func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) error { + return elasticsearch.AssertHitsAreNotPresent(result) +} +func (fts *FleetTestSuite) startStandAloneAgent(image string, flavour string, env map[string]string) error { + fts.StandAlone = true log.Trace("Deploying an agent to Fleet") - dockerImageTag := common.AgentVersion + dockerImageTag := common.BeatVersion useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") @@ -107,15 +106,13 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) // load the docker images that were already: // a. downloaded from the GCP bucket // b. fetched from the local beats binaries - dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.AgentVersion) + dockerInstaller := installer.GetElasticAgentInstaller("docker", image, common.BeatVersion, deployedAgentsCount) dockerInstaller.PreInstallFn() dockerImageTag += "-amd64" } - serviceManager := compose.NewServiceManager() - common.ProfileEnv["elasticAgentDockerImageSuffix"] = "" if image != "default" { common.ProfileEnv["elasticAgentDockerImageSuffix"] = "-" + image @@ -133,23 +130,26 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) common.ProfileEnv[k] = v } - err := serviceManager.AddServicesToCompose(context.Background(), common.FleetProfileName, []string{common.ElasticAgentServiceName}, common.ProfileEnv) + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest(common.FleetProfileName), + deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(flavour), + } + err := fts.deployer.Add(services, common.ProfileEnv) if err != nil { log.Error("Could not deploy the elastic-agent") return err } // get container hostname once - hostname, err := docker.GetContainerHostname(containerName) + hostname, err := deploy.GetContainerHostname(containerName) if err != nil { return err } - sats.Image = image - sats.Hostname = hostname - sats.Cleanup = true + fts.Image = image + fts.Hostname = hostname - err = sats.installTestTools(containerName) + err = fts.installTestTools(containerName) if err != nil { return err } @@ -157,23 +157,36 @@ func (sats *StandAloneTestSuite) startAgent(image string, env map[string]string) return nil } -func (sats *StandAloneTestSuite) getContainerLogs() error { - serviceManager := compose.NewServiceManager() +func (fts *FleetTestSuite) thePolicyShowsTheDatasourceAdded(packageName string) error { + log.WithFields(log.Fields{ + "policyID": fts.Policy.ID, + "package": packageName, + }).Trace("Checking if the policy shows the package added") + + maxTimeout := time.Minute + retryCount := 1 - profile := common.FleetProfileName - serviceName := common.ElasticAgentServiceName + exp := utils.GetExponentialBackOff(maxTimeout) - composes := []string{ - profile, // profile name - serviceName, // agent service + configurationIsPresentFn := func() error { + packagePolicy, err := fts.kibanaClient.GetIntegrationFromAgentPolicy(packageName, fts.Policy) + if err != nil { + log.WithFields(log.Fields{ + "packagePolicy": packagePolicy, + "policy": fts.Policy, + "retry": retryCount, + "error": err, + }).Warn("The integration was not found in the policy") + retryCount++ + return err + } + + retryCount++ + return err } - err := serviceManager.RunCommand(profile, composes, []string{"logs", serviceName}, common.ProfileEnv) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "service": serviceName, - }).Error("Could not retrieve Elastic Agent logs") + err := backoff.Retry(configurationIsPresentFn, exp) + if err != nil { return err } @@ -183,8 +196,8 @@ func (sats *StandAloneTestSuite) getContainerLogs() error { // installTestTools we need the container name because we use the Docker Client instead of Docker Compose // we are going to install those tools we use in the test framework for checking // and verifications -func (sats *StandAloneTestSuite) installTestTools(containerName string) error { - if sats.Image != "ubi8" { +func (fts *FleetTestSuite) installTestTools(containerName string) error { + if fts.Image != "ubi8" { return nil } @@ -195,7 +208,7 @@ func (sats *StandAloneTestSuite) installTestTools(containerName string) error { "containerName": containerName, }).Trace("Installing test tools ") - _, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + _, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { log.WithFields(log.Fields{ "command": cmd, @@ -213,51 +226,6 @@ func (sats *StandAloneTestSuite) installTestTools(containerName string) error { return nil } -func (sats *StandAloneTestSuite) thereIsNewDataInTheIndexFromAgent() error { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - minimumHitsCount := 50 - - result, err := searchAgentData(sats.Hostname, sats.RuntimeDependenciesStartDate, minimumHitsCount, maxTimeout) - if err != nil { - return err - } - - log.Tracef("Search result: %v", result) - - return elasticsearch.AssertHitsArePresent(result) -} - -func (sats *StandAloneTestSuite) theDockerContainerIsStopped(serviceName string) error { - serviceManager := compose.NewServiceManager() - - err := serviceManager.RemoveServicesFromCompose(context.Background(), common.FleetProfileName, []string{serviceName}, common.ProfileEnv) - if err != nil { - return err - } - sats.AgentStoppedDate = time.Now().UTC() - - return nil -} - -func (sats *StandAloneTestSuite) thereIsNoNewDataInTheIndexAfterAgentShutsDown() error { - maxTimeout := time.Duration(30) * time.Second - minimumHitsCount := 1 - - result, err := searchAgentData(sats.Hostname, sats.AgentStoppedDate, minimumHitsCount, maxTimeout) - if err != nil { - if strings.Contains(err.Error(), "type:index_not_found_exception") { - return err - } - - log.WithFields(log.Fields{ - "error": err, - }).Info("No documents were found for the Agent in the index after it stopped") - return nil - } - - return elasticsearch.AssertHitsAreNotPresent(result) -} - func searchAgentData(hostname string, startDate time.Time, minimumHitsCount int, maxTimeout time.Duration) (elasticsearch.SearchResult, error) { timezone := "America/New_York" diff --git a/e2e/_suites/fleet/world.go b/e2e/_suites/fleet/world.go index 4f8d395efe..4e02804303 100644 --- a/e2e/_suites/fleet/world.go +++ b/e2e/_suites/fleet/world.go @@ -7,15 +7,19 @@ package main import ( "fmt" "strconv" + "strings" + "time" + "github.com/cenkalti/backoff/v4" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" ) // IngestManagerTestSuite represents a test suite, holding references to the pieces needed to run the tests type IngestManagerTestSuite struct { - Fleet *FleetTestSuite - StandAlone *StandAloneTestSuite + Fleet *FleetTestSuite } func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state string) error { @@ -24,11 +28,14 @@ func (imts *IngestManagerTestSuite) processStateOnTheHost(process string, state func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocurrences string, process string, state string) error { profile := common.FleetProfileName - serviceName := common.ElasticAgentServiceName - containerName := fmt.Sprintf("%s_%s_%s_%d", profile, imts.Fleet.Image+"-systemd", serviceName, 1) - if imts.StandAlone.Hostname != "" { - containerName = fmt.Sprintf("%s_%s_%d", profile, serviceName, 1) + var containerName string + + if imts.Fleet.StandAlone { + containerName = fmt.Sprintf("%s_%s_%d", profile, common.ElasticAgentServiceName, 1) + } else { + agentInstaller := imts.Fleet.getInstaller() + containerName = imts.Fleet.getContainerName(agentInstaller) } count, err := strconv.Atoi(ocurrences) @@ -36,5 +43,201 @@ func (imts *IngestManagerTestSuite) thereAreInstancesOfTheProcessInTheState(ocur return err } - return docker.CheckProcessStateOnTheHost(containerName, process, state, count, common.TimeoutFactor) + return CheckProcessState(imts.Fleet.deployer, containerName, process, state, count, utils.TimeoutFactor) +} + +// CheckProcessState checks if a process is in the desired state in a container +// name of the container for the service: +// we are using the underlying deployer to run the commands in the container/service +func CheckProcessState(deployer deploy.Deployment, service string, process string, state string, occurrences int, timeoutFactor int) error { + timeout := time.Duration(utils.TimeoutFactor) * time.Minute + + err := waitForProcess(deployer, service, process, state, occurrences, timeout) + if err != nil { + if state == "started" { + log.WithFields(log.Fields{ + "service ": service, + "error": err, + "timeout": timeout, + }).Error("The process was not found but should be present") + } else { + log.WithFields(log.Fields{ + "service": service, + "error": err, + "timeout": timeout, + }).Error("The process was found but shouldn't be present") + } + + return err + } + + return nil +} + +// waitForProcess polls a container executing "ps" command until the process is in the desired state (present or not), +// or a timeout happens +func waitForProcess(deployer deploy.Deployment, service string, process string, desiredState string, ocurrences int, maxTimeout time.Duration) error { + exp := utils.GetExponentialBackOff(maxTimeout) + + mustBePresent := false + if desiredState == "started" { + mustBePresent = true + } + retryCount := 1 + + // wrap service into a request for the deployer + serviceRequest := deploy.NewServiceRequest(service) + + processStatus := func() error { + log.WithFields(log.Fields{ + "desiredState": desiredState, + "ocurrences": ocurrences, + "process": process, + }).Trace("Checking process desired state on the container") + + // pgrep -d: -d, --delimiter specify output delimiter + //i.e. "pgrep -d , metricbeat": 483,519 + cmds := []string{"pgrep", "-d", ",", process} + output, err := deployer.ExecIn(serviceRequest, cmds) + if err != nil { + log.WithFields(log.Fields{ + "cmds": cmds, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": ocurrences, + "process": process, + "retry": retryCount, + }).Warn("Could not get number of processes in the container") + + retryCount++ + + return err + } + + // tokenize the pids to get each pid's state, adding them to an array if they match the desired state + // From Split docs: + // If output does not contain sep and sep is not empty, Split returns a + // slice of length 1 whose only element is s, that's why we first initialise to the empty array + pids := strings.Split(output, ",") + if len(pids) == 1 && pids[0] == "" { + pids = []string{} + } + + log.WithFields(log.Fields{ + "count": len(pids), + "desiredState": desiredState, + "mustBePresent": mustBePresent, + "pids": pids, + "process": process, + }).Tracef("Pids for process found") + + desiredStatePids := []string{} + + for _, pid := range pids { + pidStateCmds := []string{"ps", "-q", pid, "-o", "state", "--no-headers"} + pidState, err := deployer.ExecIn(serviceRequest, pidStateCmds) + if err != nil { + log.WithFields(log.Fields{ + "cmds": cmds, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": ocurrences, + "pid": pid, + "process": process, + "retry": retryCount, + }).Warn("Could not check pid status in the container") + + retryCount++ + + return err + } + + log.WithFields(log.Fields{ + "desiredState": desiredState, + "mustBePresent": mustBePresent, + "pid": pid, + "pidState": pidState, + "process": process, + }).Tracef("Checking if process is in the S state") + + // if the process must be present, then check for the S state + // From 'man ps': + // D uninterruptible sleep (usually IO) + // R running or runnable (on run queue) + // S interruptible sleep (waiting for an event to complete) + // T stopped by job control signal + // t stopped by debugger during the tracing + // W paging (not valid since the 2.6.xx kernel) + // X dead (should never be seen) + // Z defunct ("zombie") process, terminated but not reaped by its parent + if mustBePresent && pidState == "S" { + desiredStatePids = append(desiredStatePids, pid) + } else if !mustBePresent { + desiredStatePids = append(desiredStatePids, pid) + } + } + + occurrencesMatched := (len(desiredStatePids) == ocurrences) + + // both true or both false + if mustBePresent == occurrencesMatched { + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "desiredState": desiredState, + "service": service, + "mustBePresent": mustBePresent, + "ocurrences": len(desiredStatePids), + "process": process, + }).Infof("Process desired state checked") + + return nil + } + + if mustBePresent { + err = fmt.Errorf("%s process is not running in the container with the desired number of occurrences (%d) yet", process, ocurrences) + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "desiredState": desiredState, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "ocurrences": len(desiredStatePids), + "process": process, + "retry": retryCount, + }).Warn(err.Error()) + + retryCount++ + + return err + } + + err = fmt.Errorf("%s process is still running in the container", process) + log.WithFields(log.Fields{ + "desiredOcurrences": ocurrences, + "elapsedTime": exp.GetElapsedTime(), + "error": err, + "service": service, + "ocurrences": len(desiredStatePids), + "process": process, + "state": desiredState, + "retry": retryCount, + }).Warn(err.Error()) + + retryCount++ + + return err + } + + err := backoff.Retry(processStatus, exp) + if err != nil { + return err + } + + return nil } diff --git a/e2e/_suites/helm/helm_charts_test.go b/e2e/_suites/helm/helm_charts_test.go index 180d583fad..8ab65ce860 100644 --- a/e2e/_suites/helm/helm_charts_test.go +++ b/e2e/_suites/helm/helm_charts_test.go @@ -15,7 +15,7 @@ import ( "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/e2e/steps" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/helm" "github.com/elastic/e2e-testing/internal/kubectl" "github.com/elastic/e2e-testing/internal/shell" @@ -37,10 +37,6 @@ var elasticAPMActive = false var helmManager helm.Manager -// timeoutFactor a multiplier for the max timeout when doing backoff retries. -// It can be overriden by TIMEOUT_FACTOR env var -var timeoutFactor = 2 - //nolint:unused var kubectlClient kubectl.Kubectl @@ -53,10 +49,6 @@ var helmChartVersion = "7.11.2" // kubernetesVersion represents the default version used for Kubernetes var kubernetesVersion = "1.18.2" -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = "7.13.0-SNAPSHOT" - var testSuite HelmChartTestSuite var tx *apm.Transaction @@ -80,17 +72,8 @@ func setupSuite() { helmVersion = shell.GetEnv("HELM_VERSION", helmVersion) helmChartVersion = shell.GetEnv("HELM_CHART_VERSION", helmChartVersion) kubernetesVersion = shell.GetEnv("KUBERNETES_VERSION", kubernetesVersion) - timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor) - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err := utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v + common.InitVersions() h, err := helm.Factory(helmVersion) if err != nil { @@ -187,9 +170,9 @@ func (ts *HelmChartTestSuite) aResourceWillExposePods(resourceType string) error return err } - maxTimeout := time.Duration(timeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 checkEndpointsFn := func() error { @@ -412,7 +395,7 @@ func (ts *HelmChartTestSuite) install(ctx context.Context, chart string) error { "chart": ts.Name, }).Info("Rancher Local Path Provisioner and local-path storage class for Elasticsearch volumes installed") - maxTimeout := common.TimeoutFactor * 100 + maxTimeout := utils.TimeoutFactor * 100 log.Debug("Applying workaround to use Rancher's local-path storage class for Elasticsearch volumes") flags = []string{"--wait", fmt.Sprintf("--timeout=%ds", maxTimeout), "--values", "https://raw.githubusercontent.com/elastic/helm-charts/master/elasticsearch/examples/kubernetes-kind/values.yaml"} @@ -675,20 +658,22 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { suiteContext = apm.ContextWithSpan(suiteContext, suiteParentSpan) defer suiteParentSpan.End() - if elasticAPMActive { - serviceManager := compose.NewServiceManager() + elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") + if elasticAPMActive && elasticAPMEnvironment == "local" { + serviceManager := deploy.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - err := serviceManager.RunCompose(suiteContext, true, []string{"helm"}, env) + err := serviceManager.RunCompose( + suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("helm")}, env) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", }).Warn("Could not run the profile.") } - steps.AddAPMServicesForInstrumentation(suiteContext, "helm", stackVersion, true, env) + steps.AddAPMServicesForInstrumentation(suiteContext, "helm", common.StackVersion, true, env) } err := testSuite.createCluster(suiteContext, testSuite.KubernetesVersion) @@ -730,8 +715,8 @@ func InitializeHelmChartTestSuite(ctx *godog.TestSuiteContext) { } if elasticAPMActive { - serviceManager := compose.NewServiceManager() - err := serviceManager.StopCompose(suiteContext, true, []string{"helm"}) + serviceManager := deploy.NewServiceManager() + err := serviceManager.StopCompose(suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("helm")}) if err != nil { log.WithFields(log.Fields{ "profile": "helm", diff --git a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go index 6984d07cf1..b27318a14b 100644 --- a/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go +++ b/e2e/_suites/kubernetes-autodiscover/autodiscover_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -18,22 +22,32 @@ import ( messages "github.com/cucumber/messages-go/v10" log "github.com/sirupsen/logrus" + "github.com/elastic/e2e-testing/cli/config" + "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/deploy" + "github.com/elastic/e2e-testing/internal/kubernetes" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" ) -const defaultBeatVersion = "7.13.0-SNAPSHOT" -const defaultEventsWaitTimeout = 120 * time.Second -const defaultDeployWaitTimeout = 120 * time.Second +var beatVersions = map[string]string{} + +var defaultEventsWaitTimeout = 60 * time.Second +var defaultDeployWaitTimeout = 60 * time.Second type podsManager struct { - kubectl kubernetesControl + kubectl kubernetes.Control ctx context.Context } func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, options []string) error { path := filepath.Join("testdata/templates", sanitizeName(podName)+".yml.tmpl") + err := m.configureDockerImage(podName) + if err != nil { + return err + } + usedOptions := make(map[string]bool) funcs := template.FuncMap{ "option": func(o string) bool { @@ -49,7 +63,7 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio return utils.GetDockerNamespaceEnvVar("beats") }, "beats_version": func() string { - return shell.GetEnv("GITHUB_CHECK_SHA1", shell.GetEnv("BEAT_VERSION", defaultBeatVersion)) + return beatVersions[podName] }, "namespace": func() string { return m.kubectl.Namespace @@ -85,6 +99,62 @@ func (m *podsManager) executeTemplateFor(podName string, writer io.Writer, optio return nil } +func (m *podsManager) configureDockerImage(podName string) error { + if podName != "filebeat" && podName != "heartbeat" && podName != "metricbeat" { + log.Debugf("Not processing custom binaries for pod: %s. Only [filebeat, heartbeat, metricbeat] will be processed", podName) + return nil + } + + // we are caching the versions by pod to avoid downloading and loading/tagging the Docker image multiple times + if beatVersions[podName] != "" { + log.Tracef("The beat version was already loaded: %s", beatVersions[podName]) + return nil + } + + beatVersion := common.BeatVersion + "-amd64" + + useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") + beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") + if useCISnapshots || beatsLocalPath != "" { + log.Debugf("Configuring Docker image for %s", podName) + + // this method will detect if the GITHUB_CHECK_SHA1 variable is set + artifactName := utils.BuildArtifactName(podName, common.BeatVersion, common.BeatVersionBase, "linux", "amd64", "tar.gz", true) + + imagePath, err := utils.FetchBeatsBinary(artifactName, podName, common.BeatVersion, common.BeatVersionBase, utils.TimeoutFactor, true) + if err != nil { + return err + } + + // load the TAR file into the docker host as a Docker image + err = deploy.LoadImage(imagePath) + if err != nil { + return err + } + + // tag the image with the proper docker tag, including platform + err = deploy.TagImage( + "docker.elastic.co/beats/"+podName+":"+common.BeatVersionBase, + "docker.elastic.co/observability-ci/"+podName+":"+beatVersion, + ) + if err != nil { + return err + } + + // load PR image into kind + err = cluster.LoadImage(m.ctx, "docker.elastic.co/observability-ci/"+podName+":"+beatVersion) + if err != nil { + return err + } + + } + + log.Tracef("Caching beat version '%s' for %s", beatVersion, podName) + beatVersions[podName] = beatVersion + + return nil +} + func (m *podsManager) isDeleted(podName string, options []string) error { var buf bytes.Buffer err := m.executeTemplateFor(podName, &buf, options) @@ -380,24 +450,32 @@ func waitDuration(ctx context.Context, duration string) error { return nil } -var cluster kubernetesCluster +var cluster kubernetes.Cluster func InitializeTestSuite(ctx *godog.TestSuiteContext) { suiteContext, cancel := context.WithCancel(context.Background()) log.DeferExitHandler(cancel) ctx.BeforeSuite(func() { - err := cluster.initialize(suiteContext) + // init logger + config.Init() + + common.InitVersions() + + defaultEventsWaitTimeout = defaultEventsWaitTimeout * time.Duration(utils.TimeoutFactor) + defaultDeployWaitTimeout = defaultDeployWaitTimeout * time.Duration(utils.TimeoutFactor) + + err := cluster.Initialize(suiteContext, "testdata/kind.yml") if err != nil { log.WithError(err).Fatal("Failed to initialize cluster") } log.DeferExitHandler(func() { - cluster.cleanup(suiteContext) + cluster.Cleanup(suiteContext) }) }) ctx.AfterSuite(func() { - cluster.cleanup(suiteContext) + cluster.Cleanup(suiteContext) cancel() }) } @@ -406,12 +484,12 @@ func InitializeScenario(ctx *godog.ScenarioContext) { scenarioCtx, cancel := context.WithCancel(context.Background()) log.DeferExitHandler(cancel) - var kubectl kubernetesControl + var kubectl kubernetes.Control var pods podsManager - ctx.BeforeScenario(func(*messages.Pickle) { + ctx.BeforeScenario(func(p *messages.Pickle) { kubectl = cluster.Kubectl().WithNamespace(scenarioCtx, "") if kubectl.Namespace != "" { - log.Debugf("Running scenario in namespace: %s", kubectl.Namespace) + log.Debugf("Running scenario %s in namespace: %s", p.Name, kubectl.Namespace) } pods.kubectl = kubectl pods.ctx = scenarioCtx diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl index 70cfce3cb6..6bf3627d4d 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/filebeat.yml.tmpl @@ -45,6 +45,7 @@ spec: containers: - name: filebeat image: docker.elastic.co/{{ beats_namespace }}/filebeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/filebeat.yml", "-e", diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl index aa02eca6ad..88e6e9dab4 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/heartbeat.yml.tmpl @@ -64,6 +64,7 @@ spec: containers: - name: heartbeat image: docker.elastic.co/{{ beats_namespace }}/heartbeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/heartbeat.yml", "-e", diff --git a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl index 042bb56bd7..a21a9ea43e 100644 --- a/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl +++ b/e2e/_suites/kubernetes-autodiscover/testdata/templates/metricbeat.yml.tmpl @@ -41,6 +41,7 @@ spec: containers: - name: metricbeat image: docker.elastic.co/{{ beats_namespace }}/metricbeat:{{ beats_version }} + imagePullPolicy: IfNotPresent args: [ "-c", "/etc/metricbeat.yml", "-e", diff --git a/e2e/_suites/metricbeat/metricbeat_test.go b/e2e/_suites/metricbeat/metricbeat_test.go index 2cf3956cfe..3900248c11 100644 --- a/e2e/_suites/metricbeat/metricbeat_test.go +++ b/e2e/_suites/metricbeat/metricbeat_test.go @@ -17,8 +17,7 @@ import ( "github.com/elastic/e2e-testing/cli/config" "github.com/elastic/e2e-testing/e2e/steps" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/elasticsearch" "github.com/elastic/e2e-testing/internal/shell" "github.com/elastic/e2e-testing/internal/utils" @@ -34,17 +33,7 @@ var developerMode = false var elasticAPMActive = false -var metricbeatVersionBase = "7.13.0-SNAPSHOT" - -// metricbeatVersion is the version of the metricbeat to use -// It can be overriden by BEAT_VERSION env var -var metricbeatVersion = metricbeatVersionBase - -var serviceManager compose.ServiceManager - -// stackVersion is the version of the stack to use -// It can be overriden by STACK_VERSION env var -var stackVersion = metricbeatVersionBase +var serviceManager deploy.ServiceManager var testSuite MetricbeatTestSuite @@ -66,29 +55,9 @@ func setupSuite() { }).Info("Current execution will be instrumented 🛠") } - // check if base version is an alias - v, err := utils.GetElasticArtifactVersion(metricbeatVersionBase) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": metricbeatVersionBase, - }).Fatal("Failed to get metricbeat base version, aborting") - } - metricbeatVersionBase = v + common.InitVersions() - metricbeatVersion = shell.GetEnv("BEAT_VERSION", metricbeatVersionBase) - - stackVersion = shell.GetEnv("STACK_VERSION", stackVersion) - v, err = utils.GetElasticArtifactVersion(stackVersion) - if err != nil { - log.WithFields(log.Fields{ - "error": err, - "version": stackVersion, - }).Fatal("Failed to get stack version, aborting") - } - stackVersion = v - - serviceManager = compose.NewServiceManager() + serviceManager = deploy.NewServiceManager() testSuite = MetricbeatTestSuite{ Query: elasticsearch.Query{}, @@ -153,7 +122,7 @@ func (mts *MetricbeatTestSuite) CleanUp() error { testSuite.currentContext = apm.ContextWithSpan(context.Background(), span) defer span.End() - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() fn := func(ctx context.Context) { err := elasticsearch.DeleteIndex(ctx, mts.getIndexName()) @@ -167,15 +136,15 @@ func (mts *MetricbeatTestSuite) CleanUp() error { defer fn(context.Background()) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - services := []string{"metricbeat"} + services := []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")} if mts.ServiceName != "" { - services = append(services, mts.ServiceName) + services = append(services, deploy.NewServiceRequest(mts.ServiceName)) } - err := serviceManager.RemoveServicesFromCompose(mts.currentContext, "metricbeat", services, env) + err := serviceManager.RemoveServicesFromCompose(mts.currentContext, deploy.NewServiceRequest("metricbeat"), services, env) if mts.cleanUpTmpFiles { if _, err := os.Stat(mts.configurationFile); err == nil { @@ -259,20 +228,21 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { suiteContext = apm.ContextWithSpan(suiteContext, suiteParentSpan) defer suiteParentSpan.End() - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } - err := serviceManager.RunCompose(suiteContext, true, []string{"metricbeat"}, env) + err := serviceManager.RunCompose( + suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}, env) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", }).Fatal("Could not run the profile.") } - minutesToBeHealthy := time.Duration(common.TimeoutFactor) * time.Minute + minutesToBeHealthy := time.Duration(utils.TimeoutFactor) * time.Minute healthy, err := elasticsearch.WaitForElasticsearch(suiteContext, minutesToBeHealthy) if !healthy { log.WithFields(log.Fields{ @@ -281,8 +251,9 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { }).Fatal("The Elasticsearch cluster could not get the healthy status") } - if elasticAPMActive { - steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", stackVersion, true, env) + elasticAPMEnvironment := shell.GetEnv("ELASTIC_APM_ENVIRONMENT", "ci") + if elasticAPMActive && elasticAPMEnvironment == "local" { + steps.AddAPMServicesForInstrumentation(suiteContext, "metricbeat", common.StackVersion, true, env) } }) @@ -304,8 +275,8 @@ func InitializeMetricbeatTestSuite(ctx *godog.TestSuiteContext) { defer suiteParentSpan.End() if !developerMode { - serviceManager := compose.NewServiceManager() - err := serviceManager.StopCompose(suiteContext, true, []string{"metricbeat"}) + serviceManager := deploy.NewServiceManager() + err := serviceManager.StopCompose(suiteContext, true, []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}) if err != nil { log.WithFields(log.Fields{ "profile": "metricbeat", @@ -319,7 +290,7 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForModule(serviceType stri serviceType = strings.ToLower(serviceType) // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() mts.ServiceType = serviceType @@ -372,16 +343,10 @@ func (mts *MetricbeatTestSuite) installedAndConfiguredForVariantModule(serviceVa } func (mts *MetricbeatTestSuite) installedUsingConfiguration(configuration string) error { - // restore initial state - metricbeatVersionBackup := metricbeatVersion - defer func() { metricbeatVersion = metricbeatVersionBackup }() - // at this point we have everything to define the index name - mts.Version = metricbeatVersion + mts.Version = common.BeatVersion mts.setIndexName() - metricbeatVersion = utils.CheckPRVersion(metricbeatVersion, metricbeatVersionBase) - configurationFilePath, err := steps.FetchBeatConfiguration(false, "metricbeat", configuration+".yml") if err != nil { return err @@ -406,22 +371,22 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { useCISnapshots := shell.GetEnvBool("BEATS_USE_CI_SNAPSHOTS") beatsLocalPath := shell.GetEnv("BEATS_LOCAL_PATH", "") if useCISnapshots || beatsLocalPath != "" { - artifactName := utils.BuildArtifactName("metricbeat", mts.Version, metricbeatVersionBase, "linux", "amd64", "tar.gz", true) + artifactName := utils.BuildArtifactName("metricbeat", mts.Version, common.BeatVersionBase, "linux", "amd64", "tar.gz", true) - imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, metricbeatVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, "metricbeat", mts.Version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return err } - err = docker.LoadImage(imagePath) + err = deploy.LoadImage(imagePath) if err != nil { return err } mts.Version = mts.Version + "-amd64" - err = docker.TagImage( - "docker.elastic.co/beats/metricbeat:"+metricbeatVersionBase, + err = deploy.TagImage( + "docker.elastic.co/beats/metricbeat:"+common.BeatVersionBase, "docker.elastic.co/observability-ci/metricbeat:"+mts.Version, ) if err != nil { @@ -430,7 +395,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { } // this is needed because, in general, the target service (apache, mysql, redis) does not have a healthcheck - waitForService := time.Duration(common.TimeoutFactor) * 10 * time.Second + waitForService := time.Duration(utils.TimeoutFactor) * 10 * time.Second if mts.ServiceName == "ceph" { // see https://github.com/elastic/beats/blob/ef6274d0d1e36308a333cbed69846a1bd63528ae/metricbeat/module/ceph/mgr_osd_tree/mgr_osd_tree_integration_test.go#L35 // Ceph service needs more time to start up @@ -439,7 +404,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { utils.Sleep(waitForService) - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() logLevel := log.GetLevel().String() if log.GetLevel() == log.TraceLevel { @@ -452,7 +417,7 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { "logLevel": logLevel, "metricbeatConfigFile": mts.configurationFile, "metricbeatTag": mts.Version, - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, mts.ServiceName + "Tag": mts.ServiceVersion, "serviceName": mts.ServiceName, } @@ -460,7 +425,11 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { env["metricbeatDockerNamespace"] = utils.GetDockerNamespaceEnvVar("beats") env["metricbeatPlatform"] = "linux/amd64" - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{"metricbeat"}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, + deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest("metricbeat")}, + env) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -491,13 +460,13 @@ func (mts *MetricbeatTestSuite) runMetricbeatService() error { } if log.IsLevelEnabled(log.DebugLevel) { - composes := []string{ - "metricbeat", // profile name - "metricbeat", // metricbeat service + services := []deploy.ServiceRequest{ + deploy.NewServiceRequest("metricbeat"), // profile name + deploy.NewServiceRequest("metricbeat"), // metricbeat service } if developerMode { - err = serviceManager.RunCommand("metricbeat", composes, []string{"logs", "metricbeat"}, env) + err = serviceManager.RunCommand(deploy.NewServiceRequest("metricbeat"), services, []string{"logs", "metricbeat"}, env) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -516,11 +485,14 @@ func (mts *MetricbeatTestSuite) serviceIsRunningForMetricbeat(serviceType string serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceEnvironment(env, serviceType, serviceVersion) - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{serviceType}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest(serviceType)}, + env) if err != nil { log.WithFields(log.Fields{ "service": serviceType, @@ -541,11 +513,14 @@ func (mts *MetricbeatTestSuite) serviceVariantIsRunningForMetricbeat( serviceType = strings.ToLower(serviceType) env := map[string]string{ - "stackVersion": stackVersion, + "stackVersion": common.StackVersion, } env = config.PutServiceVariantEnvironment(env, serviceType, serviceVariant, serviceVersion) - err := serviceManager.AddServicesToCompose(testSuite.currentContext, "metricbeat", []string{serviceType}, env) + err := serviceManager.AddServicesToCompose( + testSuite.currentContext, deploy.NewServiceRequest("metricbeat"), + []deploy.ServiceRequest{deploy.NewServiceRequest(serviceType)}, + env) if err != nil { log.WithFields(log.Fields{ "service": serviceType, @@ -576,7 +551,7 @@ func (mts *MetricbeatTestSuite) thereAreEventsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { @@ -612,7 +587,7 @@ func (mts *MetricbeatTestSuite) thereAreNoErrorsInTheIndex() error { } minimumHitsCount := 5 - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute result, err := elasticsearch.WaitForNumberOfHits(mts.currentContext, mts.getIndexName(), esQuery, minimumHitsCount, maxTimeout) if err != nil { diff --git a/e2e/steps/befores.go b/e2e/steps/befores.go index d4cb3a2e3b..200c45095b 100644 --- a/e2e/steps/befores.go +++ b/e2e/steps/befores.go @@ -8,24 +8,24 @@ import ( "context" "strings" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/shell" log "github.com/sirupsen/logrus" ) // AddAPMServicesForInstrumentation adds a Kibana and APM Server instances to the running project func AddAPMServicesForInstrumentation(ctx context.Context, profile string, stackVersion string, needsKibana bool, env map[string]string) { - serviceManager := compose.NewServiceManager() + serviceManager := deploy.NewServiceManager() apmServerURL := shell.GetEnv("APM_SERVER_URL", "") if strings.HasPrefix(apmServerURL, "http://localhost") { - apmServices := []string{ - "apm-server", + apmServices := []deploy.ServiceRequest{ + deploy.NewServiceRequest("apm-server"), } if needsKibana { env["kibanaTag"] = stackVersion - apmServices = append(apmServices, "kibana") + apmServices = append(apmServices, deploy.NewServiceRequest("kibana")) } log.WithFields(log.Fields{ @@ -34,7 +34,7 @@ func AddAPMServicesForInstrumentation(ctx context.Context, profile string, stack }).Info("Starting local APM services for instrumentation") env["apmServerTag"] = stackVersion - err := serviceManager.AddServicesToCompose(ctx, profile, apmServices, env) + err := serviceManager.AddServicesToCompose(ctx, deploy.NewServiceRequest(profile), apmServices, env) if err != nil { log.WithFields(log.Fields{ "error": err, diff --git a/internal/common/defaults.go b/internal/common/defaults.go index 8d7db430f5..54ec149a6a 100644 --- a/internal/common/defaults.go +++ b/internal/common/defaults.go @@ -4,6 +4,12 @@ package common +import ( + "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" +) + // ElasticAgentProcessName the name of the process for the Elastic Agent const ElasticAgentProcessName = "elastic-agent" @@ -19,25 +25,70 @@ const ElasticEndpointIntegrationTitle = "Endpoint Security" // FleetProfileName the name of the profile to run the runtime, backend services const FleetProfileName = "fleet" -// AgentVersionBase is the base version of the agent to use -var AgentVersionBase = "7.13.0-SNAPSHOT" +// FleetServerAgentServiceName the name of the service for the Elastic Agent +const FleetServerAgentServiceName = "fleet-server" + +// BeatVersionBase is the base version of the Beat to use +var BeatVersionBase = "7.13.0-SNAPSHOT" -// AgentVersion is the version of the agent to use +// BeatVersion is the version of the Beat to use // It can be overriden by BEAT_VERSION env var -var AgentVersion = AgentVersionBase +var BeatVersion = BeatVersionBase // AgentStaleVersion is the version of the agent to use as a base during upgrade // It can be overriden by ELASTIC_AGENT_STALE_VERSION env var. Using latest GA as a default. -var AgentStaleVersion = "7.11-SNAPSHOT" +var AgentStaleVersion = "7.12-SNAPSHOT" // StackVersion is the version of the stack to use // It can be overriden by STACK_VERSION env var -var StackVersion = AgentVersionBase +var StackVersion = BeatVersionBase // KibanaVersion is the version of kibana to use // It can be override by KIBANA_VERSION -var KibanaVersion = AgentVersionBase +var KibanaVersion = BeatVersionBase // ProfileEnv is the environment to be applied to any execution // affecting the runtime dependencies (or profile) var ProfileEnv map[string]string + +// Provider is the deployment provider used, currently docker is supported +var Provider = "docker" + +// InitVersions initialise default versions. We do not want to do it in the init phase +// supporting lazy-loading the versions when needed. Basically, the CLI part does not +// need to load them +func InitVersions() { + v, err := utils.GetElasticArtifactVersion(BeatVersionBase) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersionBase, + }).Fatal("Failed to get Beat base version, aborting") + } + BeatVersionBase = v + + BeatVersion = shell.GetEnv("BEAT_VERSION", BeatVersionBase) + + // check if version is an alias + v, err = utils.GetElasticArtifactVersion(BeatVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": BeatVersion, + }).Fatal("Failed to get Beat version, aborting") + } + BeatVersion = v + + // detects if the BeatVersion is set by the GITHUB_CHECK_SHA1 variable + BeatVersion = utils.CheckPRVersion(BeatVersion, BeatVersionBase) + + StackVersion = shell.GetEnv("STACK_VERSION", BeatVersionBase) + v, err = utils.GetElasticArtifactVersion(StackVersion) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "version": StackVersion, + }).Fatal("Failed to get stack version, aborting") + } + StackVersion = v +} diff --git a/internal/deploy/base.go b/internal/deploy/base.go new file mode 100644 index 0000000000..4500c6ee2c --- /dev/null +++ b/internal/deploy/base.go @@ -0,0 +1,71 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "strings" +) + +// Deployment interface for operations dealing with deployments of the bits +// required for testing +type Deployment interface { + Add(services []ServiceRequest, env map[string]string) error // adds a service to deployment + Bootstrap(waitCB func() error) error // will bootstrap or reuse existing cluster if kubernetes is selected + Destroy() error // Teardown deployment + ExecIn(service ServiceRequest, cmd []string) (string, error) // Execute arbitrary commands in service + Inspect(service ServiceRequest) (*ServiceManifest, error) // inspects service + Remove(services []ServiceRequest, env map[string]string) error // Removes services from deployment +} + +// ServiceManifest information about a service in a deployment +type ServiceManifest struct { + ID string + Name string + Connection string // a string representing how to connect to service + Hostname string +} + +// ServiceRequest represents the service to be created using the provider +type ServiceRequest struct { + Name string + Flavour string // optional, configured using builder method + Scale int // default: 1 +} + +// NewServiceRequest creates a request for a service +func NewServiceRequest(n string) ServiceRequest { + return ServiceRequest{ + Name: n, + Scale: 1, + } +} + +// WithFlavour adds a flavour for the service, resulting in a look-up of the service in the config directory, +// using flavour as a subdir of the service +func (sr ServiceRequest) WithFlavour(f string) ServiceRequest { + sr.Flavour = f + return sr +} + +// WithScale adds the scale index to the service +func (sr ServiceRequest) WithScale(s int) ServiceRequest { + if s < 1 { + s = 1 + } + + sr.Scale = s + return sr +} + +// New creates a new deployment +func New(provider string) Deployment { + if strings.EqualFold(provider, "docker") { + return newDockerDeploy() + } + if strings.EqualFold(provider, "kubernetes") { + return newK8sDeploy() + } + return nil +} diff --git a/internal/compose/compose.go b/internal/deploy/compose.go similarity index 55% rename from internal/compose/compose.go rename to internal/deploy/compose.go index cc6374282a..2a52128c18 100644 --- a/internal/compose/compose.go +++ b/internal/deploy/compose.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package compose +package deploy import ( "context" @@ -19,12 +19,12 @@ import ( // ServiceManager manages lifecycle of a service type ServiceManager interface { - AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error - ExecCommandInService(profile string, image string, serviceName string, cmds []string, env map[string]string, detach bool) error - RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error - RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error - RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error - StopCompose(ctx context.Context, isProfile bool, composeNames []string) error + AddServicesToCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error + ExecCommandInService(profile ServiceRequest, image ServiceRequest, serviceName string, cmds []string, env map[string]string, detach bool) error + RemoveServicesFromCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error + RunCommand(profile ServiceRequest, services []ServiceRequest, composeArgs []string, env map[string]string) error + RunCompose(ctx context.Context, isProfile bool, services []ServiceRequest, env map[string]string) error + StopCompose(ctx context.Context, isProfile bool, services []ServiceRequest) error } // DockerServiceManager implementation of the service manager interface @@ -37,7 +37,7 @@ func NewServiceManager() ServiceManager { } // AddServicesToCompose adds services to a running docker compose -func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Add services to Docker Compose", "docker-compose.services.add", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -45,18 +45,30 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil log.WithFields(log.Fields{ "profile": profile, - "services": composeNames, + "services": services, }).Trace("Adding services to compose") - newComposeNames := []string{profile} - newComposeNames = append(newComposeNames, composeNames...) + scaleCmds := []string{} + newServices := []ServiceRequest{profile} + for _, srv := range services { + newServices = append(newServices, srv) + if srv.Scale > 1 { + scaleCmds = append(scaleCmds, fmt.Sprintf("%s=%d", srv.Name, srv.Scale)) + } + } - persistedEnv := state.Recover(profile+"-profile", config.Op.Workspace) + persistedEnv := state.Recover(profile.Name+"-profile", config.Op.Workspace) for k, v := range env { persistedEnv[k] = v } - err := executeCompose(sm, true, newComposeNames, []string{"up", "-d"}, persistedEnv) + cmds := []string{"up", "-d"} + if len(scaleCmds) > 0 { + cmds = append(cmds, "--scale") + cmds = append(cmds, scaleCmds...) + } + + err := executeCompose(true, newServices, cmds, persistedEnv) if err != nil { return err } @@ -65,8 +77,8 @@ func (sm *DockerServiceManager) AddServicesToCompose(ctx context.Context, profil } // ExecCommandInService executes a command in a service from a profile -func (sm *DockerServiceManager) ExecCommandInService(profile string, image string, serviceName string, cmds []string, env map[string]string, detach bool) error { - composes := []string{ +func (sm *DockerServiceManager) ExecCommandInService(profile ServiceRequest, image ServiceRequest, serviceName string, cmds []string, env map[string]string, detach bool) error { + services := []ServiceRequest{ profile, // profile name image, // image for the service } @@ -74,10 +86,11 @@ func (sm *DockerServiceManager) ExecCommandInService(profile string, image strin if detach { composeArgs = append(composeArgs, "-d") } + composeArgs = append(composeArgs, "--index", fmt.Sprintf("%d", image.Scale)) composeArgs = append(composeArgs, serviceName) composeArgs = append(composeArgs, cmds...) - err := sm.RunCommand(profile, composes, composeArgs, env) + err := sm.RunCommand(profile, services, composeArgs, env) if err != nil { log.WithFields(log.Fields{ "command": cmds, @@ -92,7 +105,7 @@ func (sm *DockerServiceManager) ExecCommandInService(profile string, image strin } // RemoveServicesFromCompose removes services from a running docker compose -func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, profile string, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, profile ServiceRequest, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Remove services from Docker Compose", "docker-compose.services.remove", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) @@ -100,33 +113,33 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p log.WithFields(log.Fields{ "profile": profile, - "services": composeNames, + "services": services, }).Trace("Removing services from compose") - newComposeNames := []string{profile} - newComposeNames = append(newComposeNames, composeNames...) + newServices := []ServiceRequest{profile} + newServices = append(newServices, services...) - persistedEnv := state.Recover(profile+"-profile", config.Op.Workspace) + persistedEnv := state.Recover(profile.Name+"-profile", config.Op.Workspace) for k, v := range env { persistedEnv[k] = v } - for _, composeName := range composeNames { + for _, srv := range services { command := []string{"rm", "-fvs"} - command = append(command, composeName) + command = append(command, srv.Name) - err := executeCompose(sm, true, newComposeNames, command, persistedEnv) + err := executeCompose(true, newServices, command, persistedEnv) if err != nil { log.WithFields(log.Fields{ "command": command, - "service": composeName, + "service": srv, "profile": profile, }).Error("Could not remove service from compose") return err } log.WithFields(log.Fields{ "profile": profile, - "service": composeName, + "service": srv, }).Debug("Service removed from compose") } @@ -134,48 +147,54 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(ctx context.Context, p } // RunCommand executes a docker-compose command in a running a docker compose -func (sm *DockerServiceManager) RunCommand(profile string, composeNames []string, composeArgs []string, env map[string]string) error { - return executeCompose(sm, true, composeNames, composeArgs, env) +func (sm *DockerServiceManager) RunCommand(profile ServiceRequest, services []ServiceRequest, composeArgs []string, env map[string]string) error { + return executeCompose(true, services, composeArgs, env) } // RunCompose runs a docker compose by its name -func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, composeNames []string, env map[string]string) error { +func (sm *DockerServiceManager) RunCompose(ctx context.Context, isProfile bool, services []ServiceRequest, env map[string]string) error { span, _ := apm.StartSpanOptions(ctx, "Starting Docker Compose files", "docker-compose.services.up", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() - return executeCompose(sm, isProfile, composeNames, []string{"up", "-d"}, env) + return executeCompose(isProfile, services, []string{"up", "-d"}, env) } // StopCompose stops a docker compose by its name -func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, composeNames []string) error { +func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, services []ServiceRequest) error { span, _ := apm.StartSpanOptions(ctx, "Stopping Docker Compose files", "docker-compose.services.down", apm.SpanOptions{ Parent: apm.SpanFromContext(ctx).TraceContext(), }) defer span.End() - composeFilePaths := make([]string, len(composeNames)) - for i, composeName := range composeNames { + composeFilePaths := make([]string, len(services)) + for i, srv := range services { b := isProfile - if i == 0 && !isProfile && (len(composeName) == 1) { + if i == 0 && !isProfile && (len(services) == 1) { b = true } - composeFilePath, err := config.GetComposeFile(b, composeName) + serviceIncludingFlavour := srv.Name + if srv.Flavour != "" { + // discover the flavour in the subdir + serviceIncludingFlavour = filepath.Join(srv.Name, srv.Flavour) + } + + composeFilePath, err := config.GetComposeFile(b, serviceIncludingFlavour) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } composeFilePaths[i] = composeFilePath } - ID := composeNames[0] + "-service" + ID := services[0].Name + "-service" if isProfile { - ID = composeNames[0] + "-profile" + ID = services[0].Name + "-profile" } persistedEnv := state.Recover(ID, config.Op.Workspace) - err := executeCompose(sm, isProfile, composeNames, []string{"down", "--remove-orphans"}, persistedEnv) + err := executeCompose(isProfile, services, []string{"down", "--remove-orphans"}, persistedEnv) if err != nil { return fmt.Errorf("Could not stop compose file: %v - %v", composeFilePaths, err) } @@ -183,28 +202,34 @@ func (sm *DockerServiceManager) StopCompose(ctx context.Context, isProfile bool, log.WithFields(log.Fields{ "composeFilePath": composeFilePaths, - "profile": composeNames[0], + "profile": services[0].Name, }).Trace("Docker compose down.") return nil } -func executeCompose(sm *DockerServiceManager, isProfile bool, composeNames []string, command []string, env map[string]string) error { - composeFilePaths := make([]string, len(composeNames)) - for i, composeName := range composeNames { +func executeCompose(isProfile bool, services []ServiceRequest, command []string, env map[string]string) error { + composeFilePaths := make([]string, len(services)) + for i, srv := range services { b := false if i == 0 && isProfile { b = true } - composeFilePath, err := config.GetComposeFile(b, composeName) + serviceIncludingFlavour := srv.Name + if srv.Flavour != "" { + // discover the flavour in the subdir + serviceIncludingFlavour = filepath.Join(srv.Name, srv.Flavour) + } + + composeFilePath, err := config.GetComposeFile(b, serviceIncludingFlavour) if err != nil { return fmt.Errorf("Could not get compose file: %s - %v", composeFilePath, err) } composeFilePaths[i] = composeFilePath } - compose := tc.NewLocalDockerCompose(composeFilePaths, composeNames[0]) + compose := tc.NewLocalDockerCompose(composeFilePaths, services[0].Name) execError := compose. WithCommand(command). WithEnv(env). @@ -225,7 +250,7 @@ func executeCompose(sm *DockerServiceManager, isProfile bool, composeNames []str "cmd": command, "composeFilePaths": composeFilePaths, "env": env, - "profile": composeNames[0], + "profile": services[0].Name, }).Debug("Docker compose executed.") return nil diff --git a/internal/deploy/docker.go b/internal/deploy/docker.go new file mode 100644 index 0000000000..27faf4860b --- /dev/null +++ b/internal/deploy/docker.go @@ -0,0 +1,102 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "context" + "strings" + + "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" + log "github.com/sirupsen/logrus" +) + +// DockerDeploymentManifest deploy manifest for docker +type dockerDeploymentManifest struct { + Context context.Context +} + +func newDockerDeploy() Deployment { + return &dockerDeploymentManifest{Context: context.Background()} +} + +// Add adds services deployment +func (c *dockerDeploymentManifest) Add(services []ServiceRequest, env map[string]string) error { + serviceManager := NewServiceManager() + + return serviceManager.AddServicesToCompose(c.Context, services[0], services[1:], env) +} + +// Bootstrap sets up environment with docker compose +func (c *dockerDeploymentManifest) Bootstrap(waitCB func() error) error { + serviceManager := NewServiceManager() + common.ProfileEnv = map[string]string{ + "kibanaVersion": common.KibanaVersion, + "stackVersion": common.StackVersion, + } + + common.ProfileEnv["kibanaDockerNamespace"] = "kibana" + if strings.HasPrefix(common.KibanaVersion, "pr") || utils.IsCommit(common.KibanaVersion) { + // because it comes from a PR + common.ProfileEnv["kibanaDockerNamespace"] = "observability-ci" + } + + profile := NewServiceRequest(common.FleetProfileName) + err := serviceManager.RunCompose(c.Context, true, []ServiceRequest{profile}, common.ProfileEnv) + if err != nil { + log.WithFields(log.Fields{ + "profile": profile, + "error": err.Error(), + }).Fatal("Could not run the runtime dependencies for the profile.") + } + err = waitCB() + if err != nil { + return err + } + return nil +} + +// Destroy teardown docker environment +func (c *dockerDeploymentManifest) Destroy() error { + serviceManager := NewServiceManager() + err := serviceManager.StopCompose(c.Context, true, []ServiceRequest{NewServiceRequest(common.FleetProfileName)}) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + "profile": common.FleetProfileName, + }).Fatal("Could not destroy the runtime dependencies for the profile.") + } + return nil +} + +// ExecIn execute command in service +func (c *dockerDeploymentManifest) ExecIn(service ServiceRequest, cmd []string) (string, error) { + output, err := ExecCommandIntoContainer(c.Context, service, "root", cmd) + if err != nil { + return "", err + } + return output, nil +} + +// Inspect inspects a service +func (c *dockerDeploymentManifest) Inspect(service ServiceRequest) (*ServiceManifest, error) { + inspect, err := InspectContainer(service) + if err != nil { + return &ServiceManifest{}, err + } + return &ServiceManifest{ + ID: inspect.ID, + Name: strings.TrimPrefix(inspect.Name, "/"), + Connection: service.Name, + Hostname: inspect.NetworkSettings.Networks["fleet_default"].Aliases[0], + }, nil +} + +// Remove remove services from deployment +func (c *dockerDeploymentManifest) Remove(services []ServiceRequest, env map[string]string) error { + serviceManager := NewServiceManager() + + return serviceManager.RemoveServicesFromCompose(c.Context, services[0], services[1:], env) +} diff --git a/internal/docker/docker.go b/internal/deploy/docker_client.go similarity index 61% rename from internal/docker/docker.go rename to internal/deploy/docker_client.go index 2a48078ad4..f6a6b4f7b4 100644 --- a/internal/docker/docker.go +++ b/internal/deploy/docker_client.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package docker +package deploy import ( "archive/tar" @@ -23,7 +23,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" ) @@ -68,36 +68,6 @@ func buildTarForDeployment(file *os.File) (bytes.Buffer, error) { return buffer, nil } -// CheckProcessStateOnTheHost checks if a process is in the desired state in a container -// name of the container for the service: -// we are using the Docker client instead of docker-compose -// because it does not support returning the output of a -// command: it simply returns error level -func CheckProcessStateOnTheHost(containerName string, process string, state string, occurrences int, timeoutFactor int) error { - timeout := time.Duration(common.TimeoutFactor) * time.Minute - - err := WaitForProcess(containerName, process, state, occurrences, timeout) - if err != nil { - if state == "started" { - log.WithFields(log.Fields{ - "container ": containerName, - "error": err, - "timeout": timeout, - }).Error("The process was not found but should be present") - } else { - log.WithFields(log.Fields{ - "container": containerName, - "error": err, - "timeout": timeout, - }).Error("The process was found but shouldn't be present") - } - - return err - } - - return nil -} - // CopyFileToContainer copies a file to the running container func CopyFileToContainer(ctx context.Context, containerName string, srcPath string, parentDir string, isTar bool) error { dockerClient := getDockerClient() @@ -165,17 +135,19 @@ func CopyFileToContainer(ctx context.Context, containerName string, srcPath stri } // ExecCommandIntoContainer executes a command, as a user, into a container -func ExecCommandIntoContainer(ctx context.Context, containerName string, user string, cmd []string) (string, error) { - return ExecCommandIntoContainerWithEnv(ctx, containerName, user, cmd, []string{}) +func ExecCommandIntoContainer(ctx context.Context, container ServiceRequest, user string, cmd []string) (string, error) { + return ExecCommandIntoContainerWithEnv(ctx, container, user, cmd, []string{}) } // ExecCommandIntoContainerWithEnv executes a command, as a user, with env, into a container -func ExecCommandIntoContainerWithEnv(ctx context.Context, containerName string, user string, cmd []string, env []string) (string, error) { +func ExecCommandIntoContainerWithEnv(ctx context.Context, container ServiceRequest, user string, cmd []string, env []string) (string, error) { dockerClient := getDockerClient() detach := false tty := false + containerName := container.Name + log.WithFields(log.Fields{ "container": containerName, "command": cmd, @@ -296,7 +268,7 @@ func GetContainerHostname(containerName string) (string, error) { "containerName": containerName, }).Trace("Retrieving container name from the Docker client") - hostname, err := ExecCommandIntoContainer(context.Background(), containerName, "root", []string{"cat", "/etc/hostname"}) + hostname, err := ExecCommandIntoContainer(context.Background(), NewServiceRequest(containerName), "root", []string{"cat", "/etc/hostname"}) if err != nil { log.WithFields(log.Fields{ "containerName": containerName, @@ -315,14 +287,13 @@ func GetContainerHostname(containerName string) (string, error) { // InspectContainer returns the JSON representation of the inspection of a // Docker container, identified by its name -func InspectContainer(name string) (*types.ContainerJSON, error) { +func InspectContainer(service ServiceRequest) (*types.ContainerJSON, error) { dockerClient := getDockerClient() ctx := context.Background() labelFilters := filters.NewArgs() - labelFilters.Add("label", "service.owner=co.elastic.observability") - labelFilters.Add("label", "service.container.name="+name) + labelFilters.Add("name", service.Name) containers, err := dockerClient.ContainerList(context.Background(), types.ContainerListOptions{All: true, Filters: labelFilters}) if err != nil { @@ -340,6 +311,18 @@ func InspectContainer(name string) (*types.ContainerJSON, error) { return &inspect, nil } +// ListContainers returns a list of running containers +func ListContainers() ([]types.Container, error) { + dockerClient := getDockerClient() + ctx := context.Background() + + containers, err := dockerClient.ContainerList(ctx, types.ContainerListOptions{}) + if err != nil { + return []types.Container{}, err + } + return containers, nil +} + // RemoveContainer removes a container identified by its container name func RemoveContainer(containerName string) error { dockerClient := getDockerClient() @@ -403,8 +386,8 @@ func LoadImage(imagePath string) error { func TagImage(src string, target string) error { dockerClient := getDockerClient() - maxTimeout := 15 * time.Second - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := 5 * time.Second * time.Duration(utils.TimeoutFactor) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 0 tagImageFn := func() error { @@ -455,171 +438,6 @@ func RemoveDevNetwork() error { return nil } -// WaitForProcess polls a container executing "ps" command until the process is in the desired state (present or not), -// or a timeout happens -func WaitForProcess(containerName string, process string, desiredState string, ocurrences int, maxTimeout time.Duration) error { - exp := common.GetExponentialBackOff(maxTimeout) - - mustBePresent := false - if desiredState == "started" { - mustBePresent = true - } - retryCount := 1 - - processStatus := func() error { - log.WithFields(log.Fields{ - "desiredState": desiredState, - "ocurrences": ocurrences, - "process": process, - }).Trace("Checking process desired state on the container") - - // pgrep -d: -d, --delimiter specify output delimiter - //i.e. "pgrep -d , metricbeat": 483,519 - cmds := []string{"pgrep", "-d", ",", process} - output, err := ExecCommandIntoContainer(context.Background(), containerName, "root", cmds) - if err != nil { - log.WithFields(log.Fields{ - "cmds": cmds, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": ocurrences, - "process": process, - "retry": retryCount, - }).Warn("Could not get number of processes in the container") - - retryCount++ - - return err - } - - // tokenize the pids to get each pid's state, adding them to an array if they match the desired state - // From Split docs: - // If output does not contain sep and sep is not empty, Split returns a - // slice of length 1 whose only element is s, that's why we first initialise to the empty array - pids := strings.Split(output, ",") - if len(pids) == 1 && pids[0] == "" { - pids = []string{} - } - - log.WithFields(log.Fields{ - "count": len(pids), - "desiredState": desiredState, - "mustBePresent": mustBePresent, - "pids": pids, - "process": process, - }).Tracef("Pids for process found") - - desiredStatePids := []string{} - - for _, pid := range pids { - pidStateCmds := []string{"ps", "-q", pid, "-o", "state", "--no-headers"} - pidState, err := ExecCommandIntoContainer(context.Background(), containerName, "root", pidStateCmds) - if err != nil { - log.WithFields(log.Fields{ - "cmds": cmds, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": ocurrences, - "pid": pid, - "process": process, - "retry": retryCount, - }).Warn("Could not check pid status in the container") - - retryCount++ - - return err - } - - log.WithFields(log.Fields{ - "desiredState": desiredState, - "mustBePresent": mustBePresent, - "pid": pid, - "pidState": pidState, - "process": process, - }).Tracef("Checking if process is in the S state") - - // if the process must be present, then check for the S state - // From 'man ps': - // D uninterruptible sleep (usually IO) - // R running or runnable (on run queue) - // S interruptible sleep (waiting for an event to complete) - // T stopped by job control signal - // t stopped by debugger during the tracing - // W paging (not valid since the 2.6.xx kernel) - // X dead (should never be seen) - // Z defunct ("zombie") process, terminated but not reaped by its parent - if mustBePresent && pidState == "S" { - desiredStatePids = append(desiredStatePids, pid) - } else if !mustBePresent { - desiredStatePids = append(desiredStatePids, pid) - } - } - - occurrencesMatched := (len(desiredStatePids) == ocurrences) - - // both true or both false - if mustBePresent == occurrencesMatched { - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "desiredState": desiredState, - "container": containerName, - "mustBePresent": mustBePresent, - "ocurrences": len(desiredStatePids), - "process": process, - }).Infof("Process desired state checked") - - return nil - } - - if mustBePresent { - err = fmt.Errorf("%s process is not running in the container with the desired number of occurrences (%d) yet", process, ocurrences) - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "desiredState": desiredState, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "ocurrences": len(desiredStatePids), - "process": process, - "retry": retryCount, - }).Warn(err.Error()) - - retryCount++ - - return err - } - - err = fmt.Errorf("%s process is still running in the container", process) - log.WithFields(log.Fields{ - "desiredOcurrences": ocurrences, - "elapsedTime": exp.GetElapsedTime(), - "error": err, - "container": containerName, - "ocurrences": len(desiredStatePids), - "process": process, - "state": desiredState, - "retry": retryCount, - }).Warn(err.Error()) - - retryCount++ - - return err - } - - err := backoff.Retry(processStatus, exp) - if err != nil { - return err - } - - return nil -} - func getDockerClient() *client.Client { if instance != nil { return instance diff --git a/internal/docker/docker_test.go b/internal/deploy/docker_client_test.go similarity index 89% rename from internal/docker/docker_test.go rename to internal/deploy/docker_client_test.go index 2451a8309e..f44d420244 100644 --- a/internal/docker/docker_test.go +++ b/internal/deploy/docker_client_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package docker +package deploy import ( "context" @@ -40,7 +40,7 @@ func Test_CopyFile(t *testing.T) { err = CopyFileToContainer(ctx, containerName, src, target, false) assert.Nil(t, err) - output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"cat", "/tmp/dockerCopy.txt"}) + output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"cat", "/tmp/dockerCopy.txt"}) assert.Nil(t, err) assert.True(t, strings.HasSuffix(output, "OK!"), "File contains the 'OK!' string") }) @@ -76,7 +76,7 @@ func Test_CopyFile(t *testing.T) { err = CopyFileToContainer(ctx, containerName, src, target, true) assert.Nil(t, err) - output, err := ExecCommandIntoContainer(ctx, containerName, "root", []string{"ls", "/project/txtr/kermit.jpg"}) + output, err := ExecCommandIntoContainer(ctx, NewServiceRequest(containerName), "root", []string{"ls", "/project/txtr/kermit.jpg"}) assert.Nil(t, err) assert.True(t, strings.Contains(output, "/project/txtr/kermit.jpg"), "File '/project/txtr/kermit.jpg' should be present") }) diff --git a/internal/deploy/kubernetes.go b/internal/deploy/kubernetes.go new file mode 100644 index 0000000000..2b13196712 --- /dev/null +++ b/internal/deploy/kubernetes.go @@ -0,0 +1,119 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package deploy + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/elastic/e2e-testing/internal/kubernetes" + "github.com/pkg/errors" +) + +var cluster kubernetes.Cluster +var kubectl kubernetes.Control + +// KubernetesDeploymentManifest deploy manifest for kubernetes +type kubernetesDeploymentManifest struct { + Context context.Context +} + +func newK8sDeploy() Deployment { + return &kubernetesDeploymentManifest{Context: context.Background()} +} + +// Add adds services deployment +func (c *kubernetesDeploymentManifest) Add(services []ServiceRequest, env map[string]string) error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + + for _, service := range services { + _, err := kubectl.Run(c.Context, "apply", "-k", fmt.Sprintf("../../../cli/config/kubernetes/overlays/%s", service.Name)) + if err != nil { + return err + } + } + return nil +} + +// Bootstrap sets up environment with kind +func (c *kubernetesDeploymentManifest) Bootstrap(waitCB func() error) error { + err := cluster.Initialize(c.Context, "../../../cli/config/kubernetes/kind.yaml") + if err != nil { + return err + } + + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + _, err = kubectl.Run(c.Context, "apply", "-k", "../../../cli/config/kubernetes/base") + if err != nil { + return err + } + err = waitCB() + if err != nil { + return err + } + return nil +} + +// Destroy teardown kubernetes environment +func (c *kubernetesDeploymentManifest) Destroy() error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + cluster.Cleanup(c.Context) + return nil +} + +// ExecIn execute command in service +func (c *kubernetesDeploymentManifest) ExecIn(service ServiceRequest, cmd []string) (string, error) { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + args := []string{"exec", "deployment/" + service.Name, "--"} + for _, arg := range cmd { + args = append(cmd, arg) + } + output, err := kubectl.Run(c.Context, args...) + if err != nil { + return "", err + } + return output, nil +} + +type kubernetesServiceManifest struct { + Metadata struct { + Name string `json:"name"` + ID string `json:"uid"` + } `json:"metadata"` +} + +// Inspect inspects a service +func (c *kubernetesDeploymentManifest) Inspect(service ServiceRequest) (*ServiceManifest, error) { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + out, err := kubectl.Run(c.Context, "get", "deployment/"+service.Name, "-o", "json") + if err != nil { + return &ServiceManifest{}, err + } + var inspect kubernetesServiceManifest + if err = json.Unmarshal([]byte(out), &inspect); err != nil { + return &ServiceManifest{}, errors.Wrap(err, "Could not convert metadata to JSON") + } + return &ServiceManifest{ + ID: inspect.Metadata.ID, + Name: strings.TrimPrefix(inspect.Metadata.Name, "/"), + Connection: service.Name, + Hostname: service.Name, + }, nil +} + +// Remove remove services from deployment +func (c *kubernetesDeploymentManifest) Remove(services []ServiceRequest, env map[string]string) error { + kubectl = cluster.Kubectl().WithNamespace(c.Context, "default") + + for _, service := range services { + _, err := kubectl.Run(c.Context, "delete", "-k", fmt.Sprintf("../../../cli/config/kubernetes/overlays/%s", service.Name)) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/elasticsearch/client.go b/internal/elasticsearch/client.go index c8b32e48f8..4de42d8a8e 100644 --- a/internal/elasticsearch/client.go +++ b/internal/elasticsearch/client.go @@ -13,9 +13,9 @@ import ( "time" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" es "github.com/elastic/go-elasticsearch/v8" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -202,7 +202,7 @@ func WaitForElasticsearch(ctx context.Context, maxTimeoutMinutes time.Duration) // WaitForElasticsearchFromHostPort waits for an elasticsearch running in a host:port to be healthy, returning false // if elasticsearch does not get healthy status in a defined number of minutes. func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int, maxTimeoutMinutes time.Duration) (bool, error) { - exp := common.GetExponentialBackOff(maxTimeoutMinutes) + exp := utils.GetExponentialBackOff(maxTimeoutMinutes) retryCount := 1 @@ -251,7 +251,7 @@ func WaitForElasticsearchFromHostPort(ctx context.Context, host string, port int // WaitForIndices waits for the elasticsearch indices to return the list of indices. func WaitForIndices() (string, error) { - exp := common.GetExponentialBackOff(60 * time.Second) + exp := utils.GetExponentialBackOff(60 * time.Second) retryCount := 1 body := "" @@ -294,7 +294,7 @@ func WaitForIndices() (string, error) { // WaitForNumberOfHits waits for an elasticsearch query to return more than a number of hits, // returning false if the query does not reach that number in a defined number of time. func WaitForNumberOfHits(ctx context.Context, indexName string, query map[string]interface{}, desiredHits int, maxTimeout time.Duration) (SearchResult, error) { - exp := common.GetExponentialBackOff(maxTimeout) + exp := utils.GetExponentialBackOff(maxTimeout) retryCount := 1 result := SearchResult{} diff --git a/internal/installer/base.go b/internal/installer/base.go index 1178c46909..f65a11ae77 100644 --- a/internal/installer/base.go +++ b/internal/installer/base.go @@ -10,8 +10,7 @@ import ( "strings" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" log "github.com/sirupsen/logrus" ) @@ -38,8 +37,11 @@ type BasePackage struct { // extractPackage depends on the underlying OS, so 'cmds' must contain the specific instructions for the OS func (i *BasePackage) extractPackage(cmds []string) error { - sm := compose.NewServiceManager() - err := sm.ExecCommandInService(i.profile, i.image, i.service, cmds, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err := sm.ExecCommandInService( + deploy.NewServiceRequest(i.profile), imageService, i.service, cmds, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ "command": cmds, @@ -56,11 +58,14 @@ func (i *BasePackage) extractPackage(cmds []string) error { // Postinstall executes operations after installing a DEB package func (i *BasePackage) Postinstall() error { - err := SystemctlRun(i.profile, i.image, i.service, "enable") + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err := SystemctlRun(profileService, imageService, i.service, "enable") if err != nil { return err } - return SystemctlRun(i.profile, i.image, i.service, "start") + return SystemctlRun(profileService, imageService, i.service, "start") } // PrintLogs prints logs for the agent @@ -74,8 +79,11 @@ func (i *BasePackage) PrintLogs(containerName string) error { "cat", i.logFile, } - sm := compose.NewServiceManager() - err = sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.image) + + err = sm.ExecCommandInService( + deploy.NewServiceRequest(i.profile), imageService, i.service, cmd, common.ProfileEnv, false) if err != nil { return err } @@ -110,7 +118,7 @@ func getElasticAgentHash(containerName string, commitFile string) (string, error "cat", commitFile, } - fullHash, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + fullHash, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { return "", err } @@ -129,9 +137,9 @@ func getElasticAgentHash(containerName string, commitFile string) (string, error } // SystemctlRun runs systemctl in profile or service -func SystemctlRun(profile string, image string, service string, command string) error { +func SystemctlRun(profile deploy.ServiceRequest, image deploy.ServiceRequest, service string, command string) error { cmd := []string{"systemctl", command, common.ElasticAgentProcessName} - sm := compose.NewServiceManager() + sm := deploy.NewServiceManager() err := sm.ExecCommandInService(profile, image, service, cmd, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/deb.go b/internal/installer/deb.go index 5ec4c07c2b..08c15b9f8f 100644 --- a/internal/installer/deb.go +++ b/internal/installer/deb.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -40,14 +40,17 @@ func (i *DEBPackage) InstallCerts() error { return installCertsForDebian(i.profile, i.image, i.service) } func installCertsForDebian(profile string, image string, service string) error { - sm := compose.NewServiceManager() - if err := sm.ExecCommandInService(profile, image, service, []string{"apt-get", "update"}, common.ProfileEnv, false); err != nil { + sm := deploy.NewServiceManager() + serviceProfile := deploy.NewServiceRequest(profile) + serviceImage := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(image) + + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"apt-get", "update"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"apt", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"apt", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-certificates", "-f"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-certificates", "-f"}, common.ProfileEnv, false); err != nil { return err } return nil @@ -67,8 +70,7 @@ func (i *DEBPackage) Uninstall() error { // newDebianInstaller returns an instance of the Debian installer for a specific version func newDebianInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { - image = image + "-systemd" // we want to consume systemd boxes - service := image + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -77,7 +79,7 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI arch := "amd64" extension := "deb" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -91,8 +93,11 @@ func newDebianInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour("debian") + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, image, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } workingDir := "/var/lib/elastic-agent" diff --git a/internal/installer/docker.go b/internal/installer/docker.go index 1877dde9e6..bd6cd1162e 100644 --- a/internal/installer/docker.go +++ b/internal/installer/docker.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -55,14 +55,14 @@ func (i *DockerPackage) InstallCerts() error { // Preinstall executes operations before installing a Docker package func (i *DockerPackage) Preinstall() error { - err := docker.LoadImage(i.installerPath) + err := deploy.LoadImage(i.installerPath) if err != nil { return err } // we need to tag the loaded image because its tag relates to the target branch - return docker.TagImage( - "docker.elastic.co/beats/"+i.artifact+":"+common.AgentVersionBase, + return deploy.TagImage( + "docker.elastic.co/beats/"+i.artifact+":"+common.BeatVersionBase, "docker.elastic.co/observability-ci/"+i.artifact+":"+i.originalVersion+"-amd64", ) } @@ -99,14 +99,14 @@ func (i *DockerPackage) WithOS(OS string) *DockerPackage { // WithVersion sets the version func (i *DockerPackage) WithVersion(version string) *DockerPackage { - i.version = utils.CheckPRVersion(version, common.AgentVersionBase) // sanitize version + i.version = utils.CheckPRVersion(version, common.BeatVersionBase) // sanitize version i.originalVersion = version return i } // newDockerInstaller returns an instance of the Docker installer func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error) { - image := "elastic-agent" + image := common.ElasticAgentServiceName service := image profile := common.FleetProfileName @@ -123,7 +123,7 @@ func newDockerInstaller(ubi8 bool, version string) (ElasticAgentInstaller, error arch := "amd64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifactName, version, common.AgentVersionBase, os, arch, extension, true) + binaryName := utils.BuildArtifactName(artifactName, version, common.BeatVersionBase, os, arch, extension, true) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/installer/elasticagent.go b/internal/installer/elasticagent.go index 8399089158..258e9fb778 100644 --- a/internal/installer/elasticagent.go +++ b/internal/installer/elasticagent.go @@ -9,8 +9,7 @@ import ( "fmt" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" - "github.com/elastic/e2e-testing/internal/docker" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -47,7 +46,7 @@ func (i *ElasticAgentInstaller) ListElasticAgentWorkingDirContent(containerName "ls", "-l", i.workingDir, } - content, err := docker.ExecCommandIntoContainer(context.Background(), containerName, "root", cmd) + content, err := deploy.ExecCommandIntoContainer(context.Background(), deploy.NewServiceRequest(containerName), "root", cmd) if err != nil { return "", err } @@ -62,9 +61,9 @@ func (i *ElasticAgentInstaller) ListElasticAgentWorkingDirContent(containerName } // runElasticAgentCommandEnv runs a command for the elastic-agent -func runElasticAgentCommandEnv(profile string, image string, service string, process string, command string, arguments []string, env map[string]string) error { +func runElasticAgentCommandEnv(profile deploy.ServiceRequest, image deploy.ServiceRequest, service string, process string, command string, arguments []string, env map[string]string) error { cmds := []string{ - "timeout", fmt.Sprintf("%dm", common.TimeoutFactor), process, command, + "timeout", fmt.Sprintf("%dm", utils.TimeoutFactor), process, command, } cmds = append(cmds, arguments...) @@ -72,7 +71,7 @@ func runElasticAgentCommandEnv(profile string, image string, service string, pro common.ProfileEnv[k] = v } - sm := compose.NewServiceManager() + sm := deploy.NewServiceManager() err := sm.ExecCommandInService(profile, image, service, cmds, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ @@ -97,7 +96,7 @@ func runElasticAgentCommandEnv(profile string, image string, service string, pro // Else, if the environment variable BEATS_USE_CI_SNAPSHOTS is set, then the artifact // to be downloaded will be defined by the latest snapshot produced by the Beats CI. func downloadAgentBinary(artifactName string, artifact string, version string) (string, error) { - imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.AgentVersionBase, common.TimeoutFactor, true) + imagePath, err := utils.FetchBeatsBinary(artifactName, artifact, version, common.BeatVersionBase, utils.TimeoutFactor, true) if err != nil { return "", err } @@ -106,21 +105,22 @@ func downloadAgentBinary(artifactName string, artifact string, version string) ( } // GetElasticAgentInstaller returns an installer from a docker image -func GetElasticAgentInstaller(image string, installerType string, version string) ElasticAgentInstaller { +func GetElasticAgentInstaller(image string, installerType string, version string, index int) ElasticAgentInstaller { log.WithFields(log.Fields{ "image": image, "installer": installerType, + "version": version, }).Debug("Configuring installer for the agent") var installer ElasticAgentInstaller var err error if "centos" == image && "tar" == installerType { - installer, err = newTarInstaller("centos", "latest", version) - } else if "centos" == image && "systemd" == installerType { + installer, err = newTarInstaller("centos", "latest", version, index) + } else if "centos" == image && "rpm" == installerType { installer, err = newCentosInstaller("centos", "latest", version) } else if "debian" == image && "tar" == installerType { - installer, err = newTarInstaller("debian", "stretch", version) - } else if "debian" == image && "systemd" == installerType { + installer, err = newTarInstaller("debian", "stretch", version, index) + } else if "debian" == image && "deb" == installerType { installer, err = newDebianInstaller("debian", "stretch", version) } else if "docker" == image && "default" == installerType { installer, err = newDockerInstaller(false, version) diff --git a/internal/installer/rpm.go b/internal/installer/rpm.go index acc7c554fb..dc320db349 100644 --- a/internal/installer/rpm.go +++ b/internal/installer/rpm.go @@ -6,7 +6,7 @@ package installer import ( "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -41,17 +41,20 @@ func (i *RPMPackage) InstallCerts() error { return installCertsForCentos(i.profile, i.image, i.service) } func installCertsForCentos(profile string, image string, service string) error { - sm := compose.NewServiceManager() - if err := sm.ExecCommandInService(profile, image, service, []string{"yum", "check-update"}, common.ProfileEnv, false); err != nil { + sm := deploy.NewServiceManager() + serviceProfile := deploy.NewServiceRequest(profile) + serviceImage := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(image) + + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"yum", "check-update"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"yum", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"yum", "install", "ca-certificates", "-y"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-trust", "force-enable"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-trust", "force-enable"}, common.ProfileEnv, false); err != nil { return err } - if err := sm.ExecCommandInService(profile, image, service, []string{"update-ca-trust", "extract"}, common.ProfileEnv, false); err != nil { + if err := sm.ExecCommandInService(serviceProfile, serviceImage, service, []string{"update-ca-trust", "extract"}, common.ProfileEnv, false); err != nil { return err } return nil @@ -71,8 +74,7 @@ func (i *RPMPackage) Uninstall() error { // newCentosInstaller returns an instance of the Centos installer for a specific version func newCentosInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { - image = image + "-systemd" // we want to consume systemd boxes - service := image + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -81,7 +83,7 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI arch := "x86_64" extension := "rpm" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -95,8 +97,11 @@ func newCentosInstaller(image string, tag string, version string) (ElasticAgentI return ElasticAgentInstaller{}, err } + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour("centos") + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, image, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } workingDir := "/var/lib/elastic-agent" diff --git a/internal/installer/tar.go b/internal/installer/tar.go index 66789fe9d5..b853832b4e 100644 --- a/internal/installer/tar.go +++ b/internal/installer/tar.go @@ -8,7 +8,7 @@ import ( "fmt" "github.com/elastic/e2e-testing/internal/common" - "github.com/elastic/e2e-testing/internal/compose" + "github.com/elastic/e2e-testing/internal/deploy" "github.com/elastic/e2e-testing/internal/kibana" "github.com/elastic/e2e-testing/internal/utils" log "github.com/sirupsen/logrus" @@ -20,6 +20,7 @@ type TARPackage struct { // optional fields arch string artifact string + index int OS string OSFlavour string // at this moment, centos or debian version string @@ -45,7 +46,10 @@ func (i *TARPackage) Install(cfg *kibana.FleetConfig) error { binary := fmt.Sprintf("/elastic-agent/%s", i.artifact) args := cfg.Flags() - err := runElasticAgentCommandEnv(i.profile, i.image, i.service, binary, "install", args, map[string]string{}) + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + + err := runElasticAgentCommandEnv(profileService, imageService, i.service, binary, "install", args, map[string]string{}) if err != nil { return fmt.Errorf("Failed to install the agent with subcommand: %v", err) } @@ -83,9 +87,13 @@ func (i *TARPackage) Preinstall() error { {"rm", "-fr", "/elastic-agent"}, {"mv", fmt.Sprintf("/%s-%s-%s-%s", i.artifact, i.version, i.OS, i.arch), "/elastic-agent"}, } + + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + for _, cmd := range cmds { - sm := compose.NewServiceManager() - err := sm.ExecCommandInService(i.profile, i.image, i.service, cmd, common.ProfileEnv, false) + sm := deploy.NewServiceManager() + err := sm.ExecCommandInService(profileService, imageService, i.service, cmd, common.ProfileEnv, false) if err != nil { log.WithFields(log.Fields{ "command": cmd, @@ -106,7 +114,10 @@ func (i *TARPackage) Preinstall() error { func (i *TARPackage) Uninstall() error { args := []string{"-f"} - return runElasticAgentCommandEnv(i.profile, i.image, i.service, common.ElasticAgentProcessName, "uninstall", args, map[string]string{}) + profileService := deploy.NewServiceRequest(i.profile) + imageService := deploy.NewServiceRequest(common.ElasticAgentServiceName).WithFlavour(i.OSFlavour).WithScale(i.index) + + return runElasticAgentCommandEnv(profileService, imageService, i.service, common.ElasticAgentProcessName, "uninstall", args, map[string]string{}) } // WithArch sets the architecture @@ -121,6 +132,12 @@ func (i *TARPackage) WithArtifact(artifact string) *TARPackage { return i } +// WithIndex sets the index of the agent +func (i *TARPackage) WithIndex(index int) *TARPackage { + i.index = index + return i +} + // WithOS sets the OS func (i *TARPackage) WithOS(OS string) *TARPackage { i.OS = OS @@ -140,9 +157,8 @@ func (i *TARPackage) WithVersion(version string) *TARPackage { } // newTarInstaller returns an instance of the Debian installer for a specific version -func newTarInstaller(image string, tag string, version string) (ElasticAgentInstaller, error) { - dockerImage := image + "-systemd" // we want to consume systemd boxes - service := dockerImage +func newTarInstaller(image string, tag string, version string, index int) (ElasticAgentInstaller, error) { + service := common.ElasticAgentServiceName profile := common.FleetProfileName // extract the agent in the box, as it's mounted as a volume @@ -151,7 +167,7 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst arch := "x86_64" extension := "tar.gz" - binaryName := utils.BuildArtifactName(artifact, version, common.AgentVersionBase, os, arch, extension, false) + binaryName := utils.BuildArtifactName(artifact, version, common.BeatVersionBase, os, arch, extension, false) binaryPath, err := downloadAgentBinary(binaryName, artifact, version) if err != nil { log.WithFields(log.Fields{ @@ -173,17 +189,21 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst logFileName := "elastic-agent-json.log" logFile := logsDir + "/" + logFileName + profileService := deploy.NewServiceRequest(profile) + imageService := deploy.NewServiceRequest(service).WithFlavour(image) + enrollFn := func(cfg *kibana.FleetConfig) error { - return runElasticAgentCommandEnv(profile, dockerImage, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) + return runElasticAgentCommandEnv(profileService, imageService, service, common.ElasticAgentProcessName, "enroll", cfg.Flags(), map[string]string{}) } // - installerPackage := NewTARPackage(binaryName, profile, dockerImage, service, commitFile, logFile). + installerPackage := NewTARPackage(binaryName, profile, image, service, commitFile, logFile). WithArch(arch). WithArtifact(artifact). + WithIndex(index). WithOS(os). WithOSFlavour(image). - WithVersion(utils.CheckPRVersion(version, common.AgentVersionBase)) // sanitize version + WithVersion(utils.CheckPRVersion(version, common.BeatVersionBase)) // sanitize version return ElasticAgentInstaller{ artifactArch: arch, @@ -193,7 +213,7 @@ func newTarInstaller(image string, tag string, version string) (ElasticAgentInst artifactVersion: version, BinaryPath: binaryPath, EnrollFn: enrollFn, - Image: dockerImage, + Image: image, InstallFn: installerPackage.Install, InstallCertsFn: installerPackage.InstallCerts, InstallerType: "tar", diff --git a/internal/kibana/fleet.go b/internal/kibana/fleet.go index 95ee36610f..64ec8dbbea 100644 --- a/internal/kibana/fleet.go +++ b/internal/kibana/fleet.go @@ -7,7 +7,6 @@ package kibana import ( "fmt" - "github.com/elastic/e2e-testing/internal/common" log "github.com/sirupsen/logrus" ) @@ -21,19 +20,11 @@ type FleetConfig struct { KibanaURI string FleetServerPort int FleetServerURI string - // server - BootstrapFleetServer bool - ServerPolicyID string } -// NewFleetConfig builds a new configuration for the fleet agent, defaulting ES credentials, URI and port. -// If the 'bootstrappFleetServer' flag is true, the it will create the config for the initial fleet server -// used to bootstrap Fleet Server -// If the 'fleetServerMode' flag is true, the it will create the config for an agent using an existing Fleet -// Server to connect to Fleet. It will also retrieve the default policy ID for fleet server -func NewFleetConfig(token string, bootstrapFleetServer bool, fleetServerMode bool) (*FleetConfig, error) { +// NewFleetConfig builds a new configuration for the fleet agent, defaulting fleet-server host, ES credentials, URI and port. +func NewFleetConfig(token string) (*FleetConfig, error) { cfg := &FleetConfig{ - BootstrapFleetServer: bootstrapFleetServer, EnrollmentToken: token, ElasticsearchCredentials: "elastic:changeme", ElasticsearchPort: 9200, @@ -41,43 +32,20 @@ func NewFleetConfig(token string, bootstrapFleetServer bool, fleetServerMode boo KibanaPort: 5601, KibanaURI: "kibana", FleetServerPort: 8220, - FleetServerURI: "localhost", + FleetServerURI: "fleet-server", } - client, err := NewClient() - if err != nil { - return cfg, err - } - - if fleetServerMode { - defaultFleetServerPolicy, err := client.GetDefaultPolicy(true) - if err != nil { - return nil, err - } - - cfg.ServerPolicyID = defaultFleetServerPolicy.ID - - log.WithFields(log.Fields{ - "elasticsearch": cfg.ElasticsearchURI, - "elasticsearchPort": cfg.ElasticsearchPort, - "policyID": cfg.ServerPolicyID, - "token": cfg.EnrollmentToken, - }).Debug("Fleet Server config created") - } + log.WithFields(log.Fields{ + "elasticsearch": cfg.ElasticsearchURI, + "elasticsearchPort": cfg.ElasticsearchPort, + "token": cfg.EnrollmentToken, + }).Debug("Fleet Server config created") return cfg, nil } // Flags bootstrap flags for fleet server func (cfg FleetConfig) Flags() []string { - if cfg.BootstrapFleetServer { - // TO-DO: remove all code to calculate the fleet-server policy, because it's inferred by the fleet-server - return []string{ - "--force", - "--fleet-server-es", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), - } - } - /* // agent using an already bootstrapped fleet-server fleetServerHost := "https://hostname_of_the_bootstrapped_fleet_server:8220" @@ -89,14 +57,10 @@ func (cfg FleetConfig) Flags() []string { } */ - baseFlags := []string{"-e", "-v", "--force", "--insecure", "--enrollment-token=" + cfg.EnrollmentToken} - if common.AgentVersionBase == "7.13.0-SNAPSHOT" { - return append(baseFlags, "--url", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.FleetServerURI, cfg.FleetServerPort)) - } - - if cfg.ServerPolicyID != "" { - baseFlags = append(baseFlags, "--fleet-server-insecure-http", "--fleet-server", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.ElasticsearchURI, cfg.ElasticsearchPort), "--fleet-server-host=http://0.0.0.0", "--fleet-server-policy", cfg.ServerPolicyID) + flags := []string{ + "-e", "-v", "--force", "--insecure", "--enrollment-token=" + cfg.EnrollmentToken, + "--url", fmt.Sprintf("http://%s:%d", cfg.FleetServerURI, cfg.FleetServerPort), } - return append(baseFlags, "--kibana-url", fmt.Sprintf("http://%s@%s:%d", cfg.ElasticsearchCredentials, cfg.KibanaURI, cfg.KibanaPort)) + return flags } diff --git a/internal/kibana/integrations.go b/internal/kibana/integrations.go index 2c63dea9a4..8807cc287a 100644 --- a/internal/kibana/integrations.go +++ b/internal/kibana/integrations.go @@ -52,7 +52,7 @@ func (c *Client) DeleteIntegrationFromPolicy(packageDS PackageDataStream) error // GetIntegrations returns all available integrations func (c *Client) GetIntegrations() ([]IntegrationPackage, error) { - statusCode, respBody, err := c.get(fmt.Sprintf("%s/epm/packages", FleetAPI)) + statusCode, respBody, err := c.get(fmt.Sprintf("%s/epm/packages?experimental=true", FleetAPI)) if err != nil { log.WithFields(log.Fields{ diff --git a/internal/kibana/policies.go b/internal/kibana/policies.go index 7e7402b416..d96baaf533 100644 --- a/internal/kibana/policies.go +++ b/internal/kibana/policies.go @@ -7,6 +7,7 @@ package kibana import ( "encoding/json" "fmt" + "strings" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -75,6 +76,30 @@ func (c *Client) ListPolicies() ([]Policy, error) { return resp.Items, nil } +// DeleteAllPolicies deletes all policies except fleet_server and system +func (c *Client) DeleteAllPolicies() { + // Cleanup all package policies + packagePolicies, err := c.ListPackagePolicies() + if err != nil { + log.WithFields(log.Fields{ + "err": err, + }).Error("The package policies could not be found") + } + for _, pkgPolicy := range packagePolicies { + // Do not remove the fleet server package integration otherwise fleet server fails to bootstrap + if !strings.Contains(pkgPolicy.Name, "fleet_server") && !strings.Contains(pkgPolicy.Name, "system") { + log.WithField("pkgPolicy", pkgPolicy.Name).Trace("Removing package policy") + err = c.DeleteIntegrationFromPolicy(pkgPolicy) + if err != nil { + log.WithFields(log.Fields{ + "err": err, + "packagePolicy": pkgPolicy, + }).Error("The integration could not be deleted from the configuration") + } + } + } +} + // Var represents a single variable at the package or // data stream level, encapsulating the data type of the // variable and it's value. diff --git a/internal/kibana/server.go b/internal/kibana/server.go index cd37d7ae92..0ff3ae8f91 100644 --- a/internal/kibana/server.go +++ b/internal/kibana/server.go @@ -12,7 +12,7 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" + "github.com/elastic/e2e-testing/internal/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "go.elastic.co/apm" @@ -193,8 +193,8 @@ func (c *Client) RecreateFleet() error { }).Info("Fleet setup done") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -238,14 +238,14 @@ func (c *Client) WaitForFleet() error { "body": jsonResponse, "error": err, "statusCode": statusCode, - }).Error("Kibana has not been initialized") - return errors.New("Kibana has not been initialized") + }).Warn("Fleet is not ready") + return errors.New("Fleet is not ready") } - log.Info("Kibana setup initialized") + log.Info("Fleet setup complete") return nil } - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) err := backoff.Retry(waitForFleet, exp) if err != nil { @@ -257,8 +257,8 @@ func (c *Client) WaitForFleet() error { // WaitForReady waits for Kibana to be healthy and accept connections func (c *Client) WaitForReady(maxTimeoutMinutes time.Duration) (bool, error) { - maxTimeout := time.Duration(common.TimeoutFactor) * time.Minute * 2 - exp := common.GetExponentialBackOff(maxTimeout) + maxTimeout := time.Duration(utils.TimeoutFactor) * time.Minute * 2 + exp := utils.GetExponentialBackOff(maxTimeout) ctx := context.Background() diff --git a/e2e/_suites/kubernetes-autodiscover/kubernetes.go b/internal/kubernetes/kubernetes.go similarity index 59% rename from e2e/_suites/kubernetes-autodiscover/kubernetes.go rename to internal/kubernetes/kubernetes.go index 6143392c86..dbbf5fa65c 100644 --- a/e2e/_suites/kubernetes-autodiscover/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -1,4 +1,8 @@ -package main +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package kubernetes import ( "context" @@ -13,23 +17,26 @@ import ( "github.com/google/uuid" log "github.com/sirupsen/logrus" - "github.com/elastic/e2e-testing/internal/common" "github.com/elastic/e2e-testing/internal/shell" + "github.com/elastic/e2e-testing/internal/utils" ) -type kubernetesControl struct { +// Control struct for k8s cluster +type Control struct { config string Namespace string NamespaceUID string createdNamespace bool } -func (c kubernetesControl) WithConfig(config string) kubernetesControl { +// WithConfig config setter +func (c Control) WithConfig(config string) Control { c.config = config return c } -func (c kubernetesControl) WithNamespace(ctx context.Context, namespace string) kubernetesControl { +// WithNamespace namespace setter +func (c Control) WithNamespace(ctx context.Context, namespace string) Control { if namespace == "" { namespace = "test-" + uuid.New().String() err := c.createNamespace(ctx, namespace) @@ -47,7 +54,7 @@ func (c kubernetesControl) WithNamespace(ctx context.Context, namespace string) return c } -func (c kubernetesControl) createNamespace(ctx context.Context, namespace string) error { +func (c Control) createNamespace(ctx context.Context, namespace string) error { if namespace == "" { return nil } @@ -60,7 +67,7 @@ func (c kubernetesControl) createNamespace(ctx context.Context, namespace string // Wait for default account to be available, if not it is not possible to // deploy pods in this namespace. timeout := 60 * time.Second - exp := backoff.WithContext(common.GetExponentialBackOff(timeout), ctx) + exp := backoff.WithContext(utils.GetExponentialBackOff(timeout), ctx) return backoff.Retry(func() error { _, err := c.Run(ctx, "get", "serviceaccount", "default") if err != nil { @@ -70,7 +77,8 @@ func (c kubernetesControl) createNamespace(ctx context.Context, namespace string }, exp) } -func (c kubernetesControl) Cleanup(ctx context.Context) error { +// Cleanup deletes k8s namespace +func (c Control) Cleanup(ctx context.Context) error { if c.createdNamespace && c.Namespace != "" { output, err := c.Run(ctx, "delete", "namespace", c.Namespace) if err != nil { @@ -80,11 +88,13 @@ func (c kubernetesControl) Cleanup(ctx context.Context) error { return nil } -func (c kubernetesControl) Run(ctx context.Context, runArgs ...string) (output string, err error) { +// Run ability to run kubectl commands +func (c Control) Run(ctx context.Context, runArgs ...string) (output string, err error) { return c.RunWithStdin(ctx, nil, runArgs...) } -func (c kubernetesControl) RunWithStdin(ctx context.Context, stdin io.Reader, runArgs ...string) (output string, err error) { +// RunWithStdin run kubectl commands passing in options from stdin +func (c Control) RunWithStdin(ctx context.Context, stdin io.Reader, runArgs ...string) (output string, err error) { shell.CheckInstalledSoftware("kubectl") var args []string if c.config != "" { @@ -97,23 +107,26 @@ func (c kubernetesControl) RunWithStdin(ctx context.Context, stdin io.Reader, ru return shell.ExecuteWithStdin(ctx, ".", stdin, "kubectl", args...) } -type kubernetesCluster struct { +// Cluster kind structure definition +type Cluster struct { kindName string kubeconfig string tmpDir string } -func (c kubernetesCluster) Kubectl() kubernetesControl { - return kubernetesControl{}.WithConfig(c.kubeconfig) +// Kubectl executable reference to kubectl with applied kubeconfig +func (c Cluster) Kubectl() Control { + return Control{}.WithConfig(c.kubeconfig) } -func (c kubernetesCluster) isAvailable(ctx context.Context) error { +func (c Cluster) isAvailable(ctx context.Context) error { _, err := c.Kubectl().Run(ctx, "api-versions") return err } -func (c *kubernetesCluster) initialize(ctx context.Context) error { +// Initialize detect existing cluster contexts, otherwise will create one via Kind +func (c *Cluster) Initialize(ctx context.Context, kindConfigPath string) error { err := c.isAvailable(ctx) if err == nil { return nil @@ -138,7 +151,7 @@ func (c *kubernetesCluster) initialize(ctx context.Context) error { args := []string{ "create", "cluster", "--name", name, - "--config", "testdata/kind.yml", + "--config", kindConfigPath, "--kubeconfig", c.kubeconfig, } if version, ok := os.LookupEnv("KUBERNETES_VERSION"); ok && version != "" { @@ -157,7 +170,8 @@ func (c *kubernetesCluster) initialize(ctx context.Context) error { return nil } -func (c *kubernetesCluster) cleanup(ctx context.Context) { +// Cleanup deletes the kind cluster if available +func (c *Cluster) Cleanup(ctx context.Context) { if c.kindName != "" { _, err := shell.Execute(ctx, ".", "kind", "delete", "cluster", "--name", c.kindName) if err != nil { @@ -173,3 +187,27 @@ func (c *kubernetesCluster) cleanup(ctx context.Context) { } } } + +// LoadImage loads a Docker image into Kind runtime, using it fully qualified name. +// It does not check cluster availability because a pull error could be present in the pod, +// which will need the load of the requested image, causing a chicken-egg error. +func (c *Cluster) LoadImage(ctx context.Context, image string) error { + shell.CheckInstalledSoftware("kind") + + loadArgs := []string{"load", "docker-image", image} + // default cluster name is equals to 'kind' + if c.kindName != "" { + loadArgs = append(loadArgs, "--name", c.kindName) + } + + result, err := shell.Execute(ctx, ".", "kind", loadArgs...) + if err != nil { + log.WithError(err).Fatal("Failed to load archive into kind") + } + log.WithFields(log.Fields{ + "image": image, + "result": result, + }).Info("Image has been loaded into Kind runtime") + + return nil +} diff --git a/internal/common/retry.go b/internal/utils/retry.go similarity index 87% rename from internal/common/retry.go rename to internal/utils/retry.go index fa38cb95a1..c1ce828388 100644 --- a/internal/common/retry.go +++ b/internal/utils/retry.go @@ -2,18 +2,23 @@ // or more contributor license agreements. Licensed under the Elastic License; // you may not use this file except in compliance with the Elastic License. -package common +package utils import ( "time" backoff "github.com/cenkalti/backoff/v4" + "github.com/elastic/e2e-testing/internal/shell" ) // TimeoutFactor a multiplier for the max timeout when doing backoff retries. // It can be overriden by TIMEOUT_FACTOR env var var TimeoutFactor = 3 +func init() { + TimeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", TimeoutFactor) +} + // GetExponentialBackOff returns a preconfigured exponential backoff instance func GetExponentialBackOff(elapsedTime time.Duration) *backoff.ExponentialBackOff { var ( diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 3059381535..ce942d2ac7 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -18,7 +18,6 @@ import ( "github.com/Jeffail/gabs/v2" backoff "github.com/cenkalti/backoff/v4" - "github.com/elastic/e2e-testing/internal/common" curl "github.com/elastic/e2e-testing/internal/curl" internalio "github.com/elastic/e2e-testing/internal/io" "github.com/elastic/e2e-testing/internal/shell" @@ -26,6 +25,11 @@ import ( log "github.com/sirupsen/logrus" ) +// to avoid fetching the same Elastic artifacts version, we are adding this map to cache the version of the Elastic artifacts, +// using as key the URL of the version. If another request is trying to fetch the same URL, it will return the string version +// of the already requested one. +var elasticVersionsCache = map[string]string{} + // to avoid downloading the same artifacts, we are adding this map to cache the URL of the downloaded binaries, using as key // the URL of the artifact. If another installer is trying to download the same URL, it will return the location of the // already downloaded artifact. @@ -187,7 +191,17 @@ func getGCPBucketCoordinates(fileName string, artifact string, version string, f // If the version is a PR, then it will return the version without checking the artifacts API // i.e. GetElasticArtifactVersion("$VERSION") func GetElasticArtifactVersion(version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + cacheKey := fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version) + + if val, ok := elasticVersionsCache[cacheKey]; ok { + log.WithFields(log.Fields{ + "URL": cacheKey, + "version": val, + }).Debug("Retrieving version from local cache") + return val, nil + } + + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -195,7 +209,7 @@ func GetElasticArtifactVersion(version string) (string, error) { apiStatus := func() error { r := curl.HTTPRequest{ - URL: fmt.Sprintf("https://artifacts-api.elastic.co/v1/versions/%s/?x-elastic-no-kpi=true", version), + URL: cacheKey, } response, err := curl.Get(r) @@ -247,6 +261,8 @@ func GetElasticArtifactVersion(version string) (string, error) { "version": latestVersion, }).Debug("Latest version for current version obtained") + elasticVersionsCache[cacheKey] = latestVersion + return latestVersion, nil } @@ -256,7 +272,7 @@ func GetElasticArtifactVersion(version string) (string, error) { // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-x86_64.rpm", "elastic-agent","$VERSION") // i.e. GetElasticArtifactURL("elastic-agent-$VERSION-linux-amd64.tar.gz", "elastic-agent","$VERSION") func GetElasticArtifactURL(artifactName string, artifact string, version string) (string, error) { - exp := common.GetExponentialBackOff(time.Minute) + exp := GetExponentialBackOff(time.Minute) retryCount := 1 @@ -320,7 +336,7 @@ func GetElasticArtifactURL(artifactName string, artifact string, version string) // GetObjectURLFromBucket extracts the media URL for the desired artifact from the // Google Cloud Storage bucket used by the CI to push snapshots func GetObjectURLFromBucket(bucket string, prefix string, object string, maxtimeout time.Duration) (string, error) { - exp := common.GetExponentialBackOff(maxtimeout) + exp := GetExponentialBackOff(maxtimeout) retryCount := 1 @@ -447,7 +463,7 @@ func DownloadFile(url string) (string, error) { filepath := tempFile.Name() - exp := common.GetExponentialBackOff(3) + exp := GetExponentialBackOff(3) retryCount := 1 var fileReader io.ReadCloser