Kubernetes autodiscover suite

.ci/.e2e-tests.yaml

@@ -62,3 +62,13 @@ SUITES:
         tags: "integrations && redisenterprise"
       - name: "vSphere"
         tags: "integrations && vsphere"
+  - suite: "kubernetes-autodiscover"
+    platforms:
+      - "ubuntu-18.04"
+    scenarios:
+      - name: "kubernetes autodiscover with filebeat"
+        tags: "kubernetes-autodiscover && filebeat"
+      - name: "kubernetes autodiscover with heartbeat"
+        tags: "kubernetes-autodiscover && heartbeat"
+      - name: "kubernetes autodiscover with metricbeat"
+        tags: "kubernetes-autodiscover && metricbeat"

.ci/Jenkinsfile

@@ -50,8 +50,8 @@ pipeline {
     string(name: 'STACK_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
     string(name: 'HELM_CHART_VERSION', defaultValue: '7.11.2', description: 'SemVer version of Helm chart to be used.')
     string(name: 'HELM_VERSION', defaultValue: '3.5.2', description: 'SemVer version of Helm to be used.')
-    string(name: 'HELM_KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.')
-    string(name: 'HELM_KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
+    string(name: 'KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.')
+    string(name: 'KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
     string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.')
     string(name: 'GITHUB_CHECK_REPO', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
     string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the Beats upstream project (branch or PR)')
@@ -76,8 +76,8 @@ pipeline {
         FORCE_SKIP_PRESUBMIT = "${params.forceSkipPresubmit}"
         HELM_CHART_VERSION = "${params.HELM_CHART_VERSION.trim()}"
         HELM_VERSION = "${params.HELM_VERSION.trim()}"
-        HELM_KIND_VERSION = "${params.HELM_KIND_VERSION.trim()}"
-        HELM_KUBERNETES_VERSION = "${params.HELM_KUBERNETES_VERSION.trim()}"
+        KIND_VERSION = "${params.KIND_VERSION.trim()}"
+        KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
         LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
         TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}"
       }

.ci/scripts/install-helm-test-dependencies.sh

@@ -10,29 +10,29 @@ set -euxo pipefail
 #
 # Parameters:
 #   - HELM_VERSION - that's the Helm version which will be installed and enabled.
-#   - HELM_KIND_VERSION - that's the Kind version which will be installed and enabled.
-#   - HELM_KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
+#   - KIND_VERSION - that's the Kind version which will be installed and enabled.
+#   - KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
 #
 
 MSG="parameter missing."
 HOME=${HOME:?$MSG}
 
 HELM_VERSION="${HELM_VERSION:-"3.5.2"}"
 HELM_TAR_GZ_FILE="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
-HELM_KIND_VERSION="v${HELM_KIND_VERSION:-"0.10.0"}"
-HELM_KUBERNETES_VERSION="${HELM_KUBERNETES_VERSION:-"1.18.2"}"
+KIND_VERSION="v${KIND_VERSION:-"0.10.0"}"
+KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.18.2"}"
 
 HELM_CMD="${HOME}/bin/helm"
 KBC_CMD="${HOME}/bin/kubectl"
 
 # Install kind as a Go binary
-GO111MODULE="on" go get sigs.k8s.io/kind@${HELM_KIND_VERSION}
+GO111MODULE="on" go get sigs.k8s.io/kind@${KIND_VERSION}
 
 mkdir -p "${HOME}/bin" "${HOME}/.kube"
 touch "${HOME}/.kube/config"
 
 # Install kubectl
-curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${HELM_KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
 chmod +x "${KBC_CMD}"
 ${KBC_CMD} version --client
 

.ci/scripts/install-kubernetes-autodiscover-test-dependencies.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+## Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+## or more contributor license agreements. Licensed under the Elastic License;
+## you may not use this file except in compliance with the Elastic License.
+
+set -euxo pipefail
+#
+# Install the dependencies using the install and test make goals.
+#
+# Parameters:
+#   - KIND_VERSION - that's the Kind version which will be installed and enabled.
+#   - KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
+#
+
+MSG="parameter missing."
+HOME=${HOME:?$MSG}
+
+KIND_VERSION="v${KIND_VERSION:-"0.10.0"}"
+KUBERNETES_VERSION="${KUBERNETES_VERSION:-"1.18.2"}"
+
+KUBECTL_CMD="${HOME}/bin/kubectl"
+
+# Install kind as a Go binary
+GO111MODULE="on" go get sigs.k8s.io/kind@${KIND_VERSION}
+
+mkdir -p "${HOME}/bin"
+
+# Install kubectl
+curl -sSLo "${KUBECTL_CMD}" "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KUBECTL_CMD}"
+${KUBECTL_CMD} version --client

.pre-commit-config.yaml

@@ -45,6 +45,6 @@ repos:
     -   id: check-jjbb
     -   id: check-gherkin-lint
         args: [
-            "--disable", "AvoidOutlineForSingleExample,TooClumsy,TooLongStep,TooManyDifferentTags,TooManySteps",
+            "--disable", "AvoidOutlineForSingleExample,TooClumsy,TooLongStep,TooManyDifferentTags,TooManySteps,UseBackground",
             "e2e/_suites/**/*.feature",
         ]

cli/config/config.go

@@ -128,7 +128,7 @@ func Init() {
 		"docker",
 		"docker-compose",
 	}
-	shell.CheckInstalledSoftware(binaries)
+	shell.CheckInstalledSoftware(binaries...)
 
 	InitConfig()
 }

e2e/Makefile

@@ -85,7 +85,7 @@ functional-test: install-godog
 
 .PHONY: lint
 lint:
-	@docker run -t --rm -v $(PWD):/src -w /src gherkin/lint **/*.feature --disable AvoidOutlineForSingleExample,TooClumsy,TooManySteps,TooManyDifferentTags,TooLongStep
+	@docker run -t --rm -v $(PWD):/src -w /src gherkin/lint **/*.feature --disable AvoidOutlineForSingleExample,TooClumsy,TooManySteps,TooManyDifferentTags,TooLongStep,UseBackground
 
 ## Test examples
 

e2e/README.md

@@ -128,8 +128,13 @@ We are going to enumerate the variables that will affect the product versions us
 #### Helm charts
 - `HELM_CHART_VERSION`. Set this environment variable to the proper version of the Helm charts to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L43
 - `HELM_VERSION`. Set this environment variable to the proper version of Helm to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L44
-- `HELM_KIND_VERSION`. Set this environment variable to the proper version of Kind (Kubernetes in Docker) to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L45
-- `HELM_KUBERNETES_VERSION`. Set this environment variable to the proper version of Kubernetes to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L46
+- `KIND_VERSION`. Set this environment variable to the proper version of Kind (Kubernetes in Docker) to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L45
+- `KUBERNETES_VERSION`. Set this environment variable to the proper version of Kubernetes to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L46
+
+#### Kubernetes autodiscover charts
+- `BEAT_VERSION`. Set this environment variable to the proper version of the Metricbeat to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44
+- `KIND_VERSION`. Set this environment variable to the proper version of Kind (Kubernetes in Docker) to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L45
+- `KUBERNETES_VERSION`. Set this environment variable to the proper version of Kubernetes to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/0446248bae1ff604219735998841a21a7576bfdd/.ci/Jenkinsfile#L46
 
 #### Metricbeat
 - `BEAT_VERSION`. Set this environment variable to the proper version of the Metricbeat to be used in the current execution. Default: See https://github.com/elastic/e2e-testing/blob/70b1d3ddaf39567aeb4c322054b93ad7ce53e825/.ci/Jenkinsfile#L44

e2e/_suites/helm/README.md

@@ -35,7 +35,7 @@ This is an example of the optional configuration:
    # Depending on the versions used,
    export HELM_VERSION="3.5.2"        # Helm version: for Helm v2.x.x we have to initialise Tiller right after the k8s cluster
    export HELM_CHART_VERSION="7.11.2"  # version of the Elastic's Observability Helm charts
-   export HELM_KUBERNETES_VERSION="1.18.2" # version of the cluster to be passed to kind
+   export KUBERNETES_VERSION="1.18.2" # version of the cluster to be passed to kind
    ```
 
 3. Install dependencies.

e2e/_suites/helm/helm_charts_test.go

@@ -79,7 +79,7 @@ func setupSuite() {
 
 	helmVersion = shell.GetEnv("HELM_VERSION", helmVersion)
 	helmChartVersion = shell.GetEnv("HELM_CHART_VERSION", helmChartVersion)
-	kubernetesVersion = shell.GetEnv("HELM_KUBERNETES_VERSION", kubernetesVersion)
+	kubernetesVersion = shell.GetEnv("KUBERNETES_VERSION", kubernetesVersion)
 	timeoutFactor = shell.GetEnvInteger("TIMEOUT_FACTOR", timeoutFactor)
 
 	stackVersion = shell.GetEnv("STACK_VERSION", stackVersion)
@@ -751,5 +751,5 @@ func toolsAreInstalled() {
 		"helm",
 	}
 
-	shell.CheckInstalledSoftware(binaries)
+	shell.CheckInstalledSoftware(binaries...)
 }

e2e/_suites/kubernetes-autodiscover/README.md

@@ -0,0 +1,174 @@
+# Observability Helm charts End-To-End tests
+
+## Motivation
+
+Kubernetes autodiscover is a key feature of any observability solution for this
+orchestrator, resources change dynamically and observers configurations have to
+adapt to these changes.
+Discovery of resources in Kubernetes poses some challenges, there are several
+corner cases that need to be handled, that involve keeping track of changes of
+state that are not always deterministic. This complicates the implementation and
+its testing. With lack of good test coverage and many cases to cover, it is easy to
+introduce regressions even in basic use cases.
+This suite covers a set of use cases that are better tested with real Kubernetes
+implementations.
+
+## How do the tests work?
+
+At the topmost level, the test framework uses a BDD framework written in Go, where
+we set the expected behavior of use cases in a feature file using Gherkin, and
+implementing the steps in Go code.
+
+`kubectl` is used to configure resources in a Kubernetes cluster. `kind` can be
+used to provide a local cluster.
+
+The tests will follow this general high-level approach:
+
+1. It uses `kubectl` to interact with a Kubernetes cluster.
+1. If there is no configured Kubernetes cluster in kubectl, a new one
+   is deployed using `kind`. If a cluster is created, it is also removed after
+   the suite is executed.
+1. Execute BDD steps representing each scenario. Each scenario is executed in a
+   different Kubernetes namespace, so it is easier to cleanup and avoid one
+   scenarios affecting the others. These namespaces are created and destroyed
+   before and after each scenario.
+1. New scenarios can be configured providing the Gherkin definition and
+   templatized Kubernetes manifests.
+
+### Adding new scenarios
+
+Scenarios defined in this suite are based in a sequence of actions and
+expectations defined in the feature files, and in some templates of resources to
+deploy in Kubernetes. Templates are stored in `testdata/templates`, and must
+have the `.yml.tmpl` extension.
+
+Several of the available steps can be parameterized with template names, these
+names can be written as the name of the template without the extension. Spaces
+in these names are replaced with hyphens.
+
+There are steps intended to define a desired state for the resources in the
+template, such as the following ones:
+* `"filebeat" is running` deploys the template `filebeat.yml.tmpl` and waits for
+  filebeat pods to be running. This step expects some pod to be labeled with
+  `k8s-app:filebeat`.
+* `"a service" is deployed` deploys the resources in the template
+  `a-service.yml.tmpl`, and continues without expecting any state of the
+  deployed resources.
+* `"a pod" is deleted` deletes the resources defined in the template
+  `a-pod.yml.tmpl`.
+
+Any of these steps can be parameterized with an option that can be used to
+select different configuration blocks in the template. For example the following
+step would select the configuration block marked as `monitor annotations` in
+the template: 
+```shell
+  `"a service" is deployed with "monitor annotations"
+```
+
+These option blocks can be defined in the template like this:
+```yaml
+meta:
+  annotations:
+{{ if option "monitor annotations" }}
+    co.elastic.monitor/type: tcp
+    co.elastic.monitor/hosts: "${data.host}:6379"
+{{ end }}
+```
+
+Steps defining expectations are mostly based in checking the events generated by
+the deployed observers. Steps available for that are like the following ones:
+* `"filebeat" collects events with "kubernetes.pod.name:a-pod"` checks that the
+  filebeat pod has collected at least one event with the field
+  `kubernetes.pod.name` set to a `pod`.
+* `"metricbeat" does not collect events with "kubernetes.pod.name:a-pod" during "30s"`
+  expects to have a period of time of 30 seconds without collecting events with
+  the given field and value.
+
+These steps expect to find the events in the `/tmp/beats-events` file in pods marked
+with the label `k8s-app`.
+
+There are other more specific steps. Examples for them can be found in the
+feature files.
+
+
+### Diagnosing test failures
+
+The first step in determining the exact failure is to try and reproduce the test run
+locally, ideally using the DEBUG log level to enhance the log output. Once you've
+done that, look at the output from the test run.
+
+### Running the tests
+
+1. Clone this repository, say into a folder named `e2e-testing`.
+
+   ``` shell
+   git clone git@github.com:elastic/e2e-testing.git
+   ```
+
+2. Configure the version of the tools you want to test (Optional).
+
+This is an example of the optional configuration:
+
+   ```shell
+   # Depending on the versions used,
+   export BEAT_VERSION=7.12.0 # version of beats to use
+   export BEATS_USE_CI_SNAPSHOTS=true # to select snapshots built by beats-ci
+   export KUBERNETES_VERSION="1.18.2" # version of the cluster to be passed to kind
+   ```
+
+3. Install dependencies.
+
+   - Install Kubectl 1.18 or newer
+   - Install Kind 0.10.0 or newer
+   - Install Go: `https://golang.org/doc/install` _(The CI uses [GVM](https://github.com/andrewkroh/gvm))_
+   - Install godog (from project's root directory): `make -C e2e install-godog`
+
+4. Run the tests.
+   ```shell
+   cd e2e/_suites/kubernetes-autodiscover
+   OP_LOG_LEVEL=DEBUG godog
+   ```
+
+   Optionally, you can run only one of the feature files
+   ```shell
+   cd e2e/_suites/kubernetes-autodiscover
+   OP_LOG_LEVEL=DEBUG godog features/filebeat.feature
+   ```
+
+   The tests will take a few minutes to run, spinning up the Kubernetes cluster
+   if needed.
+
+### Problems with the environment
+
+If a Kubernetes cluster is pre-configured in kubectl, you can directly use this
+command to investigate the resources deployed in the cluster by the suite. If
+the cluster was deployed by the suite, it will have a randomized name, and will
+use a temporary configuration file for kubectl.
+
+The name of the cluster can be obtained with `kubectl get clusters`, clusters
+created by this suite will follow the pattern `kind-<random uuid>`.
+
+The temporary configuration file is logged by the suite at the info level. If a
+cluster is created by the suite, you will see something like this:
+```shell
+INFO[0000] Kubernetes cluster not available, will start one using kind 
+INFO[0000] Using kind v0.10.0 go1.15.7 linux/amd64      
+INFO[0046] Kubeconfig in /tmp/test-252418601/kubeconfig
+```
+
+Then you could use the following command to control the resources with
+`kubectl`:
+```shell
+kubectl --kubeconfig /tmp/test-252418601/kubeconfig ...
+```
+
+Each scenario creates its own namespace, you can find them with `kubectl get
+ns`, they will follow the pattern `test-<random uuid>`.
+
+Interrupting the tests with Ctrl-C will leave all resources as they were, you
+can use the previous instructions to investigate problems or access to logs of
+the deployed pods.
+
+### I cannot move on
+
+Please open an issue here: https://github.com/elastic/e2e-testing/issues/new