diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-cleanup.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-cleanup.yml deleted file mode 100644 index 7c6bc2dc9..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-cleanup.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+cluster-cleanup - display-name: elastic / helm-charts - staging - cluster cleanup - description: staging - cluster cleanup - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make destroy KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-creation.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-creation.yml deleted file mode 100644 index 0279a6cfb..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+cluster-creation.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+cluster-creation - display-name: elastic / helm-charts - staging - cluster creation - description: staging - cluster creation - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make up KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} - ./in-docker make k8s-staging-registry KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-apm-server.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-apm-server.yml deleted file mode 100644 index 0496ad456..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-apm-server.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-apm-server - display-name: elastic / helm-charts - staging - integration apm-server - description: staging - integration apm-server - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: APM_SERVER_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${APM_SERVER_SUITE} CHART=apm-server diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-elasticsearch.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-elasticsearch.yml deleted file mode 100644 index e441f2141..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-elasticsearch.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-elasticsearch - display-name: elastic / helm-charts - staging - integration elasticsearch - description: staging - integration elasticsearch - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: ES_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - - DOCKER_PASSWORD=$(retry 5 vault read -field password secret/devops-ci/docker.elastic.co/devops-ci) - retry 5 docker login -u devops-ci -p $DOCKER_PASSWORD docker.elastic.co - unset DOCKER_PASSWORD - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - - # pull private images while we have the hosts docker daemon authenticated - make pull-private-images - - # the private images will be used in here - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${ES_SUITE} CHART=elasticsearch diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-filebeat.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-filebeat.yml deleted file mode 100644 index da18266be..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-filebeat.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-filebeat - display-name: elastic / helm-charts - staging - integration filebeat - description: staging - integration filebeat - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: FILEBEAT_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${FILEBEAT_SUITE} CHART=filebeat diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-kibana.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-kibana.yml deleted file mode 100644 index 8944098a1..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-kibana.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-kibana - display-name: elastic / helm-charts - staging - integration kibana - description: staging - integration kibana - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: KIBANA_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${KIBANA_SUITE} CHART=kibana diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-logstash.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-logstash.yml deleted file mode 100644 index 7db73f065..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-logstash.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-logstash - display-name: elastic / helm-charts - staging - integration logstash - description: staging - integration logstash - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: LOGSTASH_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${LOGSTASH_SUITE} CHART=logstash diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-metricbeat.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-metricbeat.yml deleted file mode 100644 index 1046c0a83..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging+integration-metricbeat.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging+integration-metricbeat - display-name: elastic / helm-charts - staging - integration metricbeat - description: staging - integration metricbeat - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.6.1-abcdabcd)" - scm: - - git: - wipe-workspace: 'True' - axes: - - axis: - type: slave - name: label - values: - - docker&&virtual - - axis: - type: yaml - name: METRICBEAT_SUITE - filename: helpers/matrix.yml - - axis: - type: yaml - name: KUBERNETES_VERSION - filename: helpers/matrix.yml - builders: - - shell: |- - #!/usr/local/bin/runbld - set -euo pipefail - - source /usr/local/bin/bash_standard_lib.sh - - set +x - VAULT_TOKEN=$(retry 5 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") - export VAULT_TOKEN - unset VAULT_ROLE_ID VAULT_SECRET_ID - set -x - - env BUMPER_VERSION_7="$BUILD_ID" BUMPER_USE_STAGING_IMAGES="true" ./helpers/bumper.py - - cluster_name="helm-${KUBERNETES_VERSION//./}-${BUILD_ID//./-}" - - cd helpers/terraform/ - ./in-docker make integration KUBERNETES_VERSION=${KUBERNETES_VERSION} CLUSTER_NAME=${cluster_name} SUITE=${METRICBEAT_SUITE} CHART=metricbeat diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}+staging.yml b/.ci/jobs.t/elastic+helm-charts+{branch}+staging.yml deleted file mode 100644 index 223a830a9..000000000 --- a/.ci/jobs.t/elastic+helm-charts+{branch}+staging.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- -- job: - name: elastic+helm-charts+%BRANCH%+staging - display-name: elastic / helm-charts +%BRANCH%+ - staging tests - description: Staging image testing - concurrent: true - parameters: - - string: - name: BUILD_ID - description: "The buildId for the staging images. (Example: 7.7.0-abcdabcd)" - project-type: multijob - scm: - - git: - wipe-workspace: 'False' - builders: - - multijob: - name: template testing and kubernetes cluster creation - condition: SUCCESSFUL - projects: - - name: elastic+helm-charts+%BRANCH%+staging+cluster-creation - current-parameters: true - - multijob: - name: elasticsearch integration testing - condition: ALWAYS - projects: - - name: elastic+helm-charts+%BRANCH%+staging+integration-elasticsearch - current-parameters: true - - multijob: - name: integration testing - condition: ALWAYS - projects: - - name: elastic+helm-charts+%BRANCH%+staging+integration-kibana - current-parameters: true - - name: elastic+helm-charts+%BRANCH%+staging+integration-filebeat - current-parameters: true - - name: elastic+helm-charts+%BRANCH%+staging+integration-metricbeat - current-parameters: true - - name: elastic+helm-charts+%BRANCH%+staging+integration-logstash - current-parameters: true - - name: elastic+helm-charts+%BRANCH%+staging+integration-apm-server - current-parameters: true - publishers: - - trigger-parameterized-builds: - - project: elastic+helm-charts+%BRANCH%+staging+cluster-cleanup - current-parameters: true - trigger-with-no-params: false diff --git a/.ci/jobs.t/elastic+helm-charts+{branch}.yml b/.ci/jobs.t/elastic+helm-charts+{branch}.yml index f526b0db9..3b02beb85 100644 --- a/.ci/jobs.t/elastic+helm-charts+{branch}.yml +++ b/.ci/jobs.t/elastic+helm-charts+{branch}.yml @@ -46,3 +46,10 @@ - project: elastic+helm-charts+%BRANCH%+cluster-cleanup current-parameters: false trigger-with-no-params: true + - slack: + notify-back-to-normal: True + notify-every-failure: True + room: infra-release-notify + team-domain: elastic + auth-token-id: release-slack-integration-token + auth-token-credential-id: release-slack-integration-token diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89f4722a8..e04b154fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -261,12 +261,12 @@ make goss [elastic helm repository]: https://helm.elastic.co [github forking model]: https://help.github.com/articles/fork-a-repo/ [goss]: https://github.com/aelsabbahy/goss/blob/master/docs/manual.md -[integration test example]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/default/test/goss.yaml +[integration test example]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/examples/default/test/goss.yaml [integration tests section]: #integration-tests [pytest]: https://docs.pytest.org/en/latest/ [serverspec]: https://serverspec.org -[templating test example]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/tests/elasticsearch_test.py +[templating test example]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/tests/elasticsearch_test.py [templating tests section]: #templating-tests [release.md]: https://github.com/elastic/helm-charts/blob/master/helpers/release.md [releases section]: #releases -[requirements.txt]: https://github.com/elastic/helm-charts/blob/7.7/requirements.txt +[requirements.txt]: https://github.com/elastic/helm-charts/blob/6.8/requirements.txt diff --git a/README.md b/README.md index ada0377f1..1286783ea 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,82 @@ # Elastic Stack Kubernetes Helm Charts -[![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+helm-charts+7.7.svg)](https://devops-ci.elastic.co/job/elastic+helm-charts+7.7/) +[![Build Status](https://img.shields.io/jenkins/s/https/devops-ci.elastic.co/job/elastic+helm-charts+6.8.svg)](https://devops-ci.elastic.co/job/elastic+helm-charts+6.8/) + + + + + +- [Charts](#charts) +- [Supported Configurations](#supported-configurations) + - [Support Matrix](#support-matrix) + - [Kubernetes Versions](#kubernetes-versions) + - [Helm versions](#helm-versions) +- [ECK](#eck) + + -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. ## Charts -Please look in the chart directories for the documentation for each chart. These -Helm charts are designed to be a lightweight way to configure our official +These Helm charts are designed to be a lightweight way to configure our official Docker images. Links to the relevant Docker image documentation has also been added below. -| Chart | Docker documentation | -|--------------------------------------------|---------------------------------------------------------------------------------| -| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/7.7/running-on-docker.html | -| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html | -| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/7.7/running-on-docker.html | -| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/7.7/docker.html | -| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/7.7/docker.html | -| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/7.7/running-on-docker.html | +We recommend that the Helm chart version is aligned to the version of the product +you want to deploy. This will ensure that you using a chart version that has been +tested against the corresponding production version. +This will also ensure that the documentation and examples for the chart will work +with the version of the product you are installing. + +For example if you want to deploy an Elasticsearch `7.7.1` cluster, use the +corresponding `7.7.1` [tag][elasticsearch-771]. + +The `master` version of these charts are intended to support the latest pre-release +versions of our products, and therefore may or may not work with current released +versions. + +| Chart | Docker documentation | Latest 7 Version | Latest 6 Version | +|--------------------------------------------|-----------------------------------------------------------------------------|----------------------------|-----------------------------| +| [APM-Server](./apm-server/README.md) | https://www.elastic.co/guide/en/apm/server/6.8/running-on-docker.html | [`7.9.0`][apm-7] | [`6.8.13`][apm-6] | +| [Elasticsearch](./elasticsearch/README.md) | https://www.elastic.co/guide/en/elasticsearch/reference/6.8/docker.html | [`7.9.0`][elasticsearch-7] | [`6.8.13`][elasticsearch-6] | +| [Filebeat](./filebeat/README.md) | https://www.elastic.co/guide/en/beats/filebeat/6.8/running-on-docker.html | [`7.9.0`][filebeat-7] | [`6.8.13`][filebeat-6] | +| [Kibana](./kibana/README.md) | https://www.elastic.co/guide/en/kibana/6.8/docker.html | [`7.9.0`][kibana-7] | [`6.8.13`][kibana-6] | +| [Logstash](./logstash/README.md) | https://www.elastic.co/guide/en/logstash/6.8/docker.html | [`7.9.0`][logstash-7] | [`6.8.13`][logstash-6] | +| [Metricbeat](./metricbeat/README.md) | https://www.elastic.co/guide/en/beats/metricbeat/6.8/running-on-docker.html | [`7.9.0`][metricbeat-7] | [`6.8.13`][metricbeat-6] | -## Kubernetes Versions +## Supported Configurations + +Starting with the `7.7.0` release, some charts are reaching GA. + +Note that only the released charts coming from [Elastic Helm repo][] or +[GitHub releases][] are supported. + +### Support Matrix + +| | Elasticsearch | Kibana | Logstash | Filebeat | Metricbeat | APM Server | +|-----|---------------|--------|----------|----------|------------|------------| +| 6.8 | Beta | Beta | Beta | Beta | Beta | Alpha | +| 7.0 | Alpha | Alpha | / | / | / | / | +| 7.1 | Beta | Beta | / | Beta | / | / | +| 7.2 | Beta | Beta | / | Beta | Beta | / | +| 7.3 | Beta | Beta | / | Beta | Beta | / | +| 7.4 | Beta | Beta | / | Beta | Beta | / | +| 7.5 | Beta | Beta | Beta | Beta | Beta | Alpha | +| 7.6 | Beta | Beta | Beta | Beta | Beta | Alpha | +| 7.7 | GA | GA | Beta | GA | GA | Alpha | +| 7.8 | GA | GA | Beta | GA | GA | Alpha | + +### Kubernetes Versions The charts are [currently tested][] against all GKE versions available. The exact versions are defined under `KUBERNETES_VERSIONS` in [helpers/matrix.yml][]. -## Helm versions +### Helm versions While we are checking backward compatibility, the charts are only tested with -Helm version mentioned in [helm-tester Dockerfile][] (currently 2.16.6). -Note that we don't support [Helm 3][] version. +Helm version mentioned in [helm-tester Dockerfile][] (currently 3.4.1). + ## ECK @@ -43,9 +86,24 @@ Elastic recommended way to deploy Elasticsearch, Kibana and APM Server on Kubernetes. -[currently tested]: https://devops-ci.elastic.co/job/elastic+helm-charts+7.7/ +[currently tested]: https://devops-ci.elastic.co/job/elastic+helm-charts+6.8/ [elastic cloud on kubernetes]: https://github.com/elastic/cloud-on-k8s -[helm 3]: https://v3.helm.sh -[helm-tester Dockerfile]: https://github.com/elastic/helm-charts/blob/7.7/helpers/helm-tester/Dockerfile -[helpers/matrix.yml]: https://github.com/elastic/helm-charts/blob/7.7/helpers/matrix.yml +[elastic helm repo]: https://helm.elastic.co +[github releases]: https://github.com/elastic/helm-charts/releases +[helm-tester Dockerfile]: https://github.com/elastic/helm-charts/blob/6.8/helpers/helm-tester/Dockerfile +[helpers/matrix.yml]: https://github.com/elastic/helm-charts/blob/6.8/helpers/matrix.yml [operator pattern]: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ +[elasticsearch-771]: https://github.com/elastic/helm-charts/tree/7.7.1/elasticsearch/ + +[apm-7]: https://github.com/elastic/helm-charts/tree/7.9.0/apm-server/README.md +[apm-6]: https://github.com/elastic/helm-charts/tree/6.8.13/apm-server/README.md +[elasticsearch-7]: https://github.com/elastic/helm-charts/tree/7.9.0/elasticsearch/README.md +[elasticsearch-6]: https://github.com/elastic/helm-charts/tree/6.8.13/elasticsearch/README.md +[filebeat-7]: https://github.com/elastic/helm-charts/tree/7.9.0/filebeat/README.md +[filebeat-6]: https://github.com/elastic/helm-charts/tree/6.8.13/filebeat/README.md +[kibana-7]: https://github.com/elastic/helm-charts/tree/7.9.0/kibana/README.md +[kibana-6]: https://github.com/elastic/helm-charts/tree/6.8.13/kibana/README.md +[logstash-7]: https://github.com/elastic/helm-charts/tree/7.9.0/logstash/README.md +[logstash-6]: https://github.com/elastic/helm-charts/tree/6.8.13/logstash/README.md +[metricbeat-7]: https://github.com/elastic/helm-charts/tree/7.9.0/metricbeat/README.md +[metricbeat-6]: https://github.com/elastic/helm-charts/tree/6.8.13/metricbeat/README.md diff --git a/apm-server/Chart.yaml b/apm-server/Chart.yaml index 0454d9152..a722f23f5 100755 --- a/apm-server/Chart.yaml +++ b/apm-server/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: apm-server -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/apm icon: https://helm.elastic.co/icons/apm.png diff --git a/apm-server/README.md b/apm-server/README.md index 3aa695991..fefdf00b1 100644 --- a/apm-server/README.md +++ b/apm-server/README.md @@ -1,12 +1,24 @@ # APM Server Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[APM Server Docker image][]. + +**Warning**: This functionality is in alpha and is subject to change. +The design and code is less mature than official GA features and is being +provided as-is with no warranties. Alpha features are not subject to the support +SLA of official GA features (see [supported configurations][] for more details). + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -20,45 +32,36 @@ -This functionality is in alpha and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Alpha features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[APM Server Docker image][]. - - ## Requirements * Kubernetes >= 1.9 * [Helm][] >= 2.8.0 +See [supported configurations][] for more details. + ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name apm-server elastic/apm-server --version=7.7.0` +* Install it: + - with Helm 3: `helm install apm-server --version elastic/apm-server` + - with Helm 2 (deprecated): `helm install --name apm-server --version elastic/apm-server` -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name apm-server ./helm-charts/apm-server` +* Install it: + - with Helm 3: `helm install apm-server ./helm-charts/apm-server --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name apm-server ./helm-charts/apm-server --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -82,41 +85,42 @@ as a reference. They are also used in the automated testing of this chart. ## Configuration -| Parameter | Description | Default | -|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| -| `affinity` | Configurable [affinity][] | `{}` | -| `apmConfig` | Allows you to add any config files in `/usr/share/apm-server/config` such as `apm-server.yml` | see [values.yaml][] | -| `autoscaling` | Enable the [horizontal pod autoscaler][] | `enabled: false` | -| `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | -| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `extraVolumeMounts` | List of additional `volumeMounts` | `[]` | -| `extraVolumes` | List of additional `volumes` | `[]` | -| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to `.Release.Name` - `.Values.nameOverride` or `.Chart.Name` | `""` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | -| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The APM Server Docker image tag | `7.7.0-SNAPSHOT` | -| `image` | The APM Server Docker image | `docker.elastic.co/apm/apm-server` | -| `ingress` | Configurable [ingress][] to expose the APM Server service | see [values.yaml][] | -| `labels` | Configurable [labels][] applied to all APM server pods | `{}` | -| `lifecycle` | Configurable [lifecycle hooks][] | `false` | -| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | -| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | -| `nodeSelector` | Configurable [nodeSelector][] | `{}` | -| `podAnnotations` | Configurable [annotations][] applied to all APM Server pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext][] for APM Server pod execution environment | see [values.yaml][] | -| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the `PriorityClass` must be created first | `""` | -| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `replicas` | Number of APM servers to run | `1` | -| `resources` | Allows you to set the [resources][] for the `Deployment` | see [values.yaml][] | -| `secretMounts` | Allows you easily mount a secret as a file inside the `Deployment`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `serviceAccount` | Custom [serviceAccount][] that APM Server will use during execution. By default will use the `serviceAccount` created by this chart | `""` | -| `service` | Configurable [service][] to expose the APM Server service. See [values.yaml][] for an example | see [values.yaml][] | -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing APM Server pod process on pod shutdown | `30` | -| `tolerations` | Configurable [tolerations][] | `[]` | -| `updateStrategy` | Allows you to change the default [updateStrategy][] for the deployment | see [values.yaml][] | +| Parameter | Description | Default | +|-----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| `affinity` | Configurable [affinity][] | `{}` | +| `apmConfig` | Allows you to add any config files in `/usr/share/apm-server/config` such as `apm-server.yml` | see [values.yaml][] | +| `autoscaling` | Enable the [horizontal pod autoscaler][] | `enabled: false` | +| `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | +| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraVolumeMounts` | List of additional `volumeMounts` | `[]` | +| `extraVolumes` | List of additional `volumes` | `[]` | +| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to `.Release.Name` - `.Values.nameOverride` or `.Chart.Name` | `""` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | +| `imageTag` | The APM Server Docker image tag | `6.8.14-SNAPSHOT` | +| `image` | The APM Server Docker image | `docker.elastic.co/apm/apm-server` | +| `ingress` | Configurable [ingress][] to expose the APM Server service | see [values.yaml][] | +| `labels` | Configurable [labels][] applied to all APM server pods | `{}` | +| `lifecycle` | Configurable [lifecycle hooks][] | `false` | +| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | +| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | +| `nodeSelector` | Configurable [nodeSelector][] | `{}` | +| `podAnnotations` | Configurable [annotations][] applied to all APM Server pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext][] for APM Server pod execution environment | see [values.yaml][] | +| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the `PriorityClass` must be created first | `""` | +| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `replicas` | Number of APM servers to run | `1` | +| `resources` | Allows you to set the [resources][] for the `Deployment` | see [values.yaml][] | +| `secretMounts` | Allows you easily mount a secret as a file inside the `Deployment`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `serviceAccount` | Custom [serviceAccount][] that APM Server will use during execution. By default will use the `serviceAccount` created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | +| `service` | Configurable [service][] to expose the APM Server service. See [values.yaml][] for an example | see [values.yaml][] | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing APM Server pod process on pod shutdown | `30` | +| `tolerations` | Configurable [tolerations][] | `[]` | +| `updateStrategy` | Allows you to change the default [updateStrategy][] for the deployment | see [values.yaml][] | ## FAQ @@ -143,20 +147,21 @@ An example of APM Server deployment using OSS version can be found in Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/apm-server/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -[apm server docker image]: https://www.elastic.co/guide/en/apm/server/7.7/running-on-docker.html -[apm server oss docker image]: https://www.docker.elastic.co/#apm-server-7-7-0-oss -[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default +[apm server docker image]: https://www.elastic.co/guide/en/apm/server/6.8/running-on-docker.html +[apm server oss docker image]: https://www.docker.elastic.co/r/apm/apm-server-oss +[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/oss -[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security +[examples]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/security [helm]: https://helm.sh [horizontal pod autoscaler]: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images @@ -172,6 +177,7 @@ about our development and testing process. [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ [service]: https://kubernetes.io/docs/concepts/services-networking/service/ [serviceAccount]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/values.yaml diff --git a/apm-server/examples/default/Makefile b/apm-server/examples/default/Makefile index 23a7eedc6..57c4116d2 100644 --- a/apm-server/examples/default/Makefile +++ b/apm-server/examples/default/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-default install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/default/README.md b/apm-server/examples/default/README.md index 8af8a5038..d480dfbb0 100644 --- a/apm-server/examples/default/README.md +++ b/apm-server/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy APM Server 7.7.0-SNAPSHOT using [default values][]. +This example deploy APM Server 6.8.14-SNAPSHOT using [default values][]. ## Usage @@ -22,6 +22,6 @@ This example deploy APM Server 7.7.0-SNAPSHOT using [default values][]. You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/values.yaml diff --git a/apm-server/examples/default/test/goss.yaml b/apm-server/examples/default/test/goss.yaml index 2dc9067df..a9c4ab5c4 100644 --- a/apm-server/examples/default/test/goss.yaml +++ b/apm-server/examples/default/test/goss.yaml @@ -3,4 +3,4 @@ http: status: 200 timeout: 2000 body: - - '7.7.0' + - '6.8.14' diff --git a/apm-server/examples/oss/Makefile b/apm-server/examples/oss/Makefile index 175317b98..61ad62b28 100644 --- a/apm-server/examples/oss/Makefile +++ b/apm-server/examples/oss/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-oss install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/oss/README.md b/apm-server/examples/oss/README.md index e32234f8a..a31c2716c 100644 --- a/apm-server/examples/oss/README.md +++ b/apm-server/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy APM Server 7.7.0-SNAPSHOT using [APM Server OSS][] version. +This example deploy APM Server 6.8.14-SNAPSHOT using [APM Server OSS][] version. ## Usage @@ -23,5 +23,5 @@ You can also run [goss integration tests][] using `make test` [apm server oss]: https://www.elastic.co/downloads/apm-oss -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/oss/test/goss.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/oss/test/goss.yaml diff --git a/apm-server/examples/oss/test/goss.yaml b/apm-server/examples/oss/test/goss.yaml index 2dc9067df..a9c4ab5c4 100644 --- a/apm-server/examples/oss/test/goss.yaml +++ b/apm-server/examples/oss/test/goss.yaml @@ -3,4 +3,4 @@ http: status: 200 timeout: 2000 body: - - '7.7.0' + - '6.8.14' diff --git a/apm-server/examples/security/Makefile b/apm-server/examples/security/Makefile index 2e9be1f86..de704f595 100644 --- a/apm-server/examples/security/Makefile +++ b/apm-server/examples/security/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-apm-server-security install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/apm-server/examples/security/README.md b/apm-server/examples/security/README.md index 517e53f86..c6fd717c0 100644 --- a/apm-server/examples/security/README.md +++ b/apm-server/examples/security/README.md @@ -1,6 +1,6 @@ # Security -This example deploy APM Server 7.7.0-SNAPSHOT using authentication and TLS to connect to +This example deploy APM Server 6.8.14-SNAPSHOT using authentication and TLS to connect to Elasticsearch (see [values][]). @@ -23,6 +23,6 @@ Elasticsearch (see [values][]). You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/apm-server/examples/security/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/apm-server/examples/security/values.yaml diff --git a/apm-server/examples/security/test/goss.yaml b/apm-server/examples/security/test/goss.yaml index 2dc9067df..a9c4ab5c4 100644 --- a/apm-server/examples/security/test/goss.yaml +++ b/apm-server/examples/security/test/goss.yaml @@ -3,4 +3,4 @@ http: status: 200 timeout: 2000 body: - - '7.7.0' + - '6.8.14' diff --git a/apm-server/templates/_helpers.tpl b/apm-server/templates/_helpers.tpl index abc1361ea..6f64c4b38 100755 --- a/apm-server/templates/_helpers.tpl +++ b/apm-server/templates/_helpers.tpl @@ -23,14 +23,14 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Return the appropriate apiVersion for ingress. */}} {{- define "apm.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.Version -}} {{- print "extensions/v1beta1" -}} {{- else -}} {{- print "networking.k8s.io/v1beta1" -}} {{- end -}} {{- end -}} {{- define "apm.autoscaling.apiVersion" -}} -{{- if semverCompare "<1.12-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.12-0" .Capabilities.KubeVersion.Version -}} {{- print "autoscaling/v2beta1" -}} {{- else -}} {{- print "autoscaling/v2beta2" -}} diff --git a/apm-server/templates/clusterrole.yaml b/apm-server/templates/clusterrole.yaml index 6e0d2bc98..2d8e1ebf8 100644 --- a/apm-server/templates/clusterrole.yaml +++ b/apm-server/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "apm.serviceAccount" . }}-cluster-role diff --git a/apm-server/templates/clusterrolebinding.yaml b/apm-server/templates/clusterrolebinding.yaml index 3060496bf..c65b2be8d 100644 --- a/apm-server/templates/clusterrolebinding.yaml +++ b/apm-server/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "apm.serviceAccount" . }}-cluster-role-binding diff --git a/apm-server/templates/deployment.yaml b/apm-server/templates/deployment.yaml index 87a1bc085..0cf3b50d9 100644 --- a/apm-server/templates/deployment.yaml +++ b/apm-server/templates/deployment.yaml @@ -33,9 +33,7 @@ spec: {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} - {{- if .Values.serviceAccount }} - serviceAccountName: {{ .Values.serviceAccount }} - {{- end }} + serviceAccountName: {{ template "apm.serviceAccount" . }} volumes: {{- range .Values.secretMounts }} - name: {{ .name }} @@ -88,15 +86,9 @@ spec: {{ toYaml .Values.podSecurityContext | indent 10 }} {{- end }} livenessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 30 +{{ toYaml .Values.livenessProbe | indent 10 }} readinessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 30 +{{ toYaml .Values.readinessProbe | indent 10 }} ports: - containerPort: {{ .Values.service.port }} name: http @@ -125,4 +117,4 @@ spec: {{- end }} {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/apm-server/templates/serviceaccount.yaml b/apm-server/templates/serviceaccount.yaml index 683838131..c03750aa7 100644 --- a/apm-server/templates/serviceaccount.yaml +++ b/apm-server/templates/serviceaccount.yaml @@ -3,6 +3,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "apm.serviceAccount" . }} + annotations: + {{- with .Values.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} labels: app: "{{ template "apm.fullname" . }}" chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" diff --git a/apm-server/tests/apmserver_test.py b/apm-server/tests/apmserver_test.py index cb89d880e..2ce3b70b1 100644 --- a/apm-server/tests/apmserver_test.py +++ b/apm-server/tests/apmserver_test.py @@ -258,6 +258,20 @@ def test_adding_pod_labels(): ) +def test_adding_serviceaccount_annotations(): + config = """ +serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][name]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) + + def test_adding_a_node_selector(): config = """ nodeSelector: diff --git a/apm-server/values.yaml b/apm-server/values.yaml index f96ac03f7..e8ec6047b 100755 --- a/apm-server/values.yaml +++ b/apm-server/values.yaml @@ -62,7 +62,7 @@ extraVolumes: [] # emptyDir: {} image: "docker.elastic.co/apm/apm-server" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] @@ -80,14 +80,20 @@ podSecurityContext: privileged: false livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 failureThreshold: 3 - initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 failureThreshold: 3 - initialDelaySeconds: 10 periodSeconds: 10 timeoutSeconds: 5 @@ -97,11 +103,15 @@ resources: memory: "100Mi" limits: cpu: "1000m" - memory: "200Mi" + memory: "512Mi" # Custom service account override that the pod will use serviceAccount: "" +# Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. +serviceAccountAnnotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount + # A list of secrets and their paths to mount inside the pod secretMounts: [] # - name: elastic-certificate-pem diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml index 38a824234..68393b83b 100755 --- a/elasticsearch/Chart.yaml +++ b/elasticsearch/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: elasticsearch -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/elasticsearch icon: https://helm.elastic.co/icons/elasticsearch.png diff --git a/elasticsearch/README.md b/elasticsearch/README.md index ee568fa06..dadfe5745 100644 --- a/elasticsearch/README.md +++ b/elasticsearch/README.md @@ -1,12 +1,19 @@ # Elasticsearch Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[Elasticsearch Docker image][]. + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -32,49 +39,41 @@ -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[Elasticsearch Docker image][]. - - ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 (see [parent README][] for more details) +* [Helm][] >=2.8.0 and <3.0.0 * Kubernetes >=1.8 * Minimum cluster requirements include the following to run this chart with default settings. All of these settings are configurable. * Three Kubernetes nodes to respect the default "hard" affinity settings * 1GB of RAM for the JVM heap +See [supported configurations][] for more details. + ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name elasticsearch elastic/elasticsearch --version=7.7.0` +* Install it: + - with Helm 3: `helm install elasticsearch --version elastic/elasticsearch` + - with Helm 2 (deprecated): `helm install --name elasticsearch --version elastic/elasticsearch` + -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name elasticsearch ./helm-charts/elasticsearch` +* Install it: + - with Helm 3: `helm install elasticsearch ./helm-charts/elasticsearch --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name elasticsearch ./helm-charts/elasticsearch --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -116,6 +115,7 @@ support multiple versions with minimal changes. | `antiAffinity` | Setting this to hard enforces the [anti-affinity][] rules. If it is set to soft it will be done "best effort". Other values will be ignored | `hard` | | `clusterHealthCheckParams` | The [Elasticsearch cluster health status params][] that will be used by readiness [probe][] command | `wait_for_status=green&timeout=1s` | | `clusterName` | This will be used as the Elasticsearch [cluster.name][] and should be unique per cluster in the namespace | `elasticsearch` | +| `enableServiceLinks` | Set to false to disabling service links, which can cause slow pod startup times when there are many services in the current namespace. | `true` | | `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | | `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml][] for an example of the formatting | `{}` | | `esJavaOpts` | [Java options][] for Elasticsearch. This is where you should configure the [jvm heap size][] | `-Xmx1g -Xms1g` | @@ -129,7 +129,7 @@ support multiple versions with minimal changes. | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port][] in `extraEnvs` | `9200` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Elasticsearch Docker image tag | `7.7.0-SNAPSHOT` | +| `imageTag` | The Elasticsearch Docker image tag | `6.8.14-SNAPSHOT` | | `image` | The Elasticsearch Docker image | `docker.elastic.co/elasticsearch/elasticsearch` | | `ingress` | Configurable [ingress][] to expose the Elasticsearch service. See [values.yaml][] for an example | see [values.yaml][] | | `initResources` | Allows you to set the [resources][] for the `initContainer` in the StatefulSet | `{}` | @@ -161,6 +161,7 @@ support multiple versions with minimal changes. | `secretMounts` | Allows you easily mount a secret as a file inside the StatefulSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | | `securityContext` | Allows you to set the [securityContext][] for the container | see [values.yaml][] | | `service.annotations` | [LoadBalancer annotations][] that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` | `{}` | +| `service.externalTrafficPolicy` | Some cloud providers allow you to specify the [LoadBalancer externalTrafficPolicy][]. Kubernetes will use this to preserve the client source IP. This will configure load balancer if `service.type` is `LoadBalancer` | `""` | | `service.httpPortName` | The name of the http port within the service | `http` | | `service.labelsHeadless` | Labels to be added to headless service | `{}` | | `service.labels` | Labels to be added to non-headless service | `{}` | @@ -293,16 +294,16 @@ Create the secret, the key name needs to be the keystore key path. In this example we will create a secret from a file and from a literal string. ``` -kubectl create secret generic encryption_key --from-file=xpack.watcher.encryption_key=./watcher_encryption_key -kubectl create secret generic slack_hook --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +kubectl create secret generic encryption-key --from-file=xpack.watcher.encryption_key=./watcher_encryption_key +kubectl create secret generic slack-hook --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' ``` To add these secrets to the keystore: ``` keystore: - - secretName: encryption_key - - secretName: slack_hook + - secretName: encryption-key + - secretName: slack-hook ``` #### Multiple keys @@ -311,12 +312,12 @@ All keys in the secret will be added to the keystore. To create the previous example in one secret you could also do: ``` -kubectl create secret generic keystore_secrets --from-file=xpack.watcher.encryption_key=./watcher_encryption_key --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +kubectl create secret generic keystore-secrets --from-file=xpack.watcher.encryption_key=./watcher_encryption_key --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' ``` ``` keystore: - - secretName: keystore_secrets + - secretName: keystore-secrets ``` #### Custom paths and keys @@ -329,7 +330,7 @@ example we will only add the `slack_hook` key from a secret that also has other keys. Our secret looks like this: ``` -kubectl create secret generic slack_secrets --from-literal=slack_channel='#general' --from-literal=slack_hook='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +kubectl create secret generic slack-secrets --from-literal=slack_channel='#general' --from-literal=slack_hook='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' ``` We only want to add the `slack_hook` key to the keystore at path @@ -337,7 +338,7 @@ We only want to add the `slack_hook` key to the keystore at path ``` keystore: - - secretName: slack_secrets + - secretName: slack-secrets items: - key: slack_hook path: xpack.notification.slack.account.monitoring.secure_url @@ -389,75 +390,77 @@ lifecycle: Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[#63]: https://github.com/elastic/helm-charts/issues/63 +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/elasticsearch/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md -[#63]: https://github.com/elastic/helm-charts/issues/63 [alternate scheduler]: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [anti-affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity -[cluster.name]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster.name.html -[clustering and node discovery]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#clustering-and-node-discovery -[config example]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/values.yaml -[curator]: https://www.elastic.co/guide/en/elasticsearch/client/curator/7.7/snapshot.html -[custom docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html#_c_customized_image +[cluster.name]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/cluster.name.html +[clustering and node discovery]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#clustering-and-node-discovery +[config example]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/config/values.yaml +[curator]: https://www.elastic.co/guide/en/elasticsearch/client/curator/6.8/snapshot.html +[custom docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/docker.html#_c_customized_image [deploys statefulsets serially]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies -[discovery.zen.minimum_master_nodes]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/discovery-settings.html#minimum_master_nodes -[docker for mac]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/docker-for-mac -[elasticsearch cluster health status params]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster-health.html#request-params -[elasticsearch docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/docker.html -[elasticsearch oss docker image]: https://www.docker.elastic.co/#elasticsearch-7-7-0-oss +[discovery.zen.minimum_master_nodes]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/discovery-settings.html#minimum_master_nodes +[docker for mac]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/docker-for-mac +[elasticsearch cluster health status params]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/cluster-health.html#request-params +[elasticsearch docker image]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/docker.html +[elasticsearch oss docker image]: https://www.docker.elastic.co/r/elasticsearch/elasticsearch-oss [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/ -[examples/multi]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss -[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security +[examples]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/ +[examples/multi]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security [gke]: https://cloud.google.com/kubernetes-engine [helm]: https://helm.sh [helm/charts stable]: https://github.com/helm/charts/tree/master/stable/elasticsearch/ -[how to install plugins guide]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#how-to-install-plugins -[how to use the keystore]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#how-to-use-the-keystore -[http.port]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-http.html#_settings +[how to install plugins guide]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#how-to-install-plugins +[how to use the keystore]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#how-to-use-the-keystore +[http.port]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-http.html#_settings [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ -[java options]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/jvm-options.html -[jvm heap size]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/heap-size.html -[kind]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/kubernetes-kind +[java options]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/jvm-options.html +[jvm heap size]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/heap-size.html +[kind]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/kubernetes-kind [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ [loadBalancer annotations]: https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws +[loadBalancer externalTrafficPolicy]: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip [loadBalancer]: https://kubernetes.io/docs/concepts/services-networking/service/#loadbalancer [maxUnavailable]: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget -[migration guide]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/migration/README.md -[minikube]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/minikube -[microk8s]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/microk8s -[multi]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/ -[network.host elasticsearch setting]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/network.host.html +[migration guide]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/migration/README.md +[minikube]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/minikube +[microk8s]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/microk8s +[multi]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi/ +[network.host elasticsearch setting]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/network.host.html [node affinity settings]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature -[node-certificates]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/configuring-tls.html#node-certificates +[node-certificates]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/configuring-tls.html#node-certificates [nodePort]: https://kubernetes.io/docs/concepts/services-networking/service/#nodeport -[nodes types]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-node.html +[nodes types]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-node.html [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector -[openshift]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift -[parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md +[openshift]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ -[roles]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-node.html +[roles]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-node.html [secret]: https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ [service types]: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types [snapshot lifecycle management]: https://github.com/elastic/elasticsearch/issues/38461 -[snapshot plugin]: https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository.html -[snapshot repository]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-snapshots.html -[sysctl vm.max_map_count]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/vm-max-map-count.html#vm-max-map-count +[snapshot plugin]: https://www.elastic.co/guide/en/elasticsearch/plugins/6.8/repository.html +[snapshot repository]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-snapshots.html +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations +[sysctl vm.max_map_count]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/vm-max-map-count.html#vm-max-map-count [terminationGracePeriod]: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -[transport port configuration]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/modules-transport.html#_transport_settings +[transport port configuration]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-transport.html#_transport_settings [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/values.yaml [volumeClaimTemplate for statefulsets]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage diff --git a/elasticsearch/examples/config/Makefile b/elasticsearch/examples/config/Makefile index a3f96174b..9eee45df2 100644 --- a/elasticsearch/examples/config/Makefile +++ b/elasticsearch/examples/config/Makefile @@ -4,7 +4,7 @@ include ../../../helpers/examples.mk RELEASE := helm-es-config install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../ secrets: kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack elastic-config-custom-path || true @@ -16,4 +16,4 @@ secrets: test: secrets install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/config/README.md b/elasticsearch/examples/config/README.md index 98d1360f5..da145362d 100644 --- a/elasticsearch/examples/config/README.md +++ b/elasticsearch/examples/config/README.md @@ -1,6 +1,6 @@ # Config -This example deploy a single node Elasticsearch 7.7.0-SNAPSHOT with authentication and +This example deploy a single node Elasticsearch 6.8.14-SNAPSHOT with authentication and custom [values][]. @@ -23,5 +23,5 @@ custom [values][]. You can also run [goss integration tests][] using `make test` -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/config/values.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/config/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/config/values.yaml diff --git a/elasticsearch/examples/config/values.yaml b/elasticsearch/examples/config/values.yaml index ebde4f4d9..1394a8ee3 100644 --- a/elasticsearch/examples/config/values.yaml +++ b/elasticsearch/examples/config/values.yaml @@ -2,6 +2,7 @@ clusterName: "config" replicas: 1 +minimumMasterNodes: 1 extraEnvs: - name: ELASTIC_PASSWORD diff --git a/elasticsearch/examples/default/Makefile b/elasticsearch/examples/default/Makefile index 5f5215c6b..beb90461d 100644 --- a/elasticsearch/examples/default/Makefile +++ b/elasticsearch/examples/default/Makefile @@ -5,12 +5,12 @@ include ../../../helpers/examples.mk RELEASE := helm-es-default install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) ../../ restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --set terminationGracePeriod=121 --wait --timeout=900s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/default/README.md b/elasticsearch/examples/default/README.md index 495e2a1ac..299901a82 100644 --- a/elasticsearch/examples/default/README.md +++ b/elasticsearch/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster using +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster using [default values][]. @@ -21,5 +21,5 @@ This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster using You can also run [goss integration tests][] using `make test` -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/values.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/values.yaml diff --git a/elasticsearch/examples/default/test/goss.yaml b/elasticsearch/examples/default/test/goss.yaml index 272064e29..0fb8916f6 100644 --- a/elasticsearch/examples/default/test/goss.yaml +++ b/elasticsearch/examples/default/test/goss.yaml @@ -15,7 +15,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.7.0-SNAPSHOT"' + - '"number" : "6.8.14"' - '"cluster_name" : "elasticsearch"' - '"name" : "elasticsearch-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/docker-for-mac/Makefile b/elasticsearch/examples/docker-for-mac/Makefile index 398545e64..705b08f0b 100644 --- a/elasticsearch/examples/docker-for-mac/Makefile +++ b/elasticsearch/examples/docker-for-mac/Makefile @@ -3,10 +3,10 @@ default: test RELEASE := helm-es-docker-for-mac install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/docker-for-mac/README.md b/elasticsearch/examples/docker-for-mac/README.md index 569bd416d..2c0fca636 100644 --- a/elasticsearch/examples/docker-for-mac/README.md +++ b/elasticsearch/examples/docker-for-mac/README.md @@ -1,6 +1,6 @@ # Docker for Mac -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Docker for Mac][] +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster on [Docker for Mac][] using [custom values][]. Note that this configuration should be used for test only and isn't recommended @@ -19,5 +19,5 @@ for production. ``` -[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/docker-for-mac/values.yaml +[custom values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/docker-for-mac/values.yaml [docker for mac]: https://docs.docker.com/docker-for-mac/kubernetes/ diff --git a/elasticsearch/examples/kubernetes-kind/Makefile b/elasticsearch/examples/kubernetes-kind/Makefile index 9dad380be..3f6bdead9 100644 --- a/elasticsearch/examples/kubernetes-kind/Makefile +++ b/elasticsearch/examples/kubernetes-kind/Makefile @@ -3,11 +3,14 @@ default: test RELEASE := helm-es-kind install: + helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../ + +install-local-path: kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=1200s --install --values values-local-path.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/kubernetes-kind/README.md b/elasticsearch/examples/kubernetes-kind/README.md index 76ce26646..6f11d8256 100644 --- a/elasticsearch/examples/kubernetes-kind/README.md +++ b/elasticsearch/examples/kubernetes-kind/README.md @@ -1,26 +1,24 @@ # KIND -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Kind][] +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster on [Kind][] using [custom values][]. Note that this configuration should be used for test only and isn't recommended for production. +Note that Kind < 0.7.0 are affected by a [kind issue][] with mount points +created from PVCs not writable by non-root users. [kubernetes-sigs/kind#1157][] +fix it in Kind 0.7.0. -## Requirements - -There is currently an [kind issue][] with mount points created from PVCs not -writable by non-root users. [kubernetes-sigs/kind#1157][] should fix it in a -future release. - -Meanwhile, the workaround is to install manually +The workaround for Kind < 0.7.0 is to install manually [Rancher Local Path Provisioner][] and use `local-path` storage class for Elasticsearch volumes (see [Makefile][] instructions). ## Usage -* Deploy Elasticsearch chart with the default values: `make install` +* For Kind >= 0.7.0: Deploy Elasticsearch chart with the default values: `make install` +* For Kind < 0.7.0: Deploy Elasticsearch chart with `local-path` storage class: `make install-local-path` * You can now setup a port forward to query Elasticsearch API: @@ -30,9 +28,9 @@ Elasticsearch volumes (see [Makefile][] instructions). ``` -[custom values]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/kubernetes-kind/values.yaml +[custom values]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/examples/kubernetes-kind/values.yaml [kind]: https://kind.sigs.k8s.io/ [kind issue]: https://github.com/kubernetes-sigs/kind/issues/830 [kubernetes-sigs/kind#1157]: https://github.com/kubernetes-sigs/kind/pull/1157 [rancher local path provisioner]: https://github.com/rancher/local-path-provisioner -[Makefile]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/kubernetes-kind/Makefile#L5 +[Makefile]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/examples/kubernetes-kind/Makefile#L5 diff --git a/elasticsearch/examples/kubernetes-kind/values-local-path.yaml b/elasticsearch/examples/kubernetes-kind/values-local-path.yaml new file mode 100644 index 000000000..500ad4b14 --- /dev/null +++ b/elasticsearch/examples/kubernetes-kind/values-local-path.yaml @@ -0,0 +1,23 @@ +--- +# Permit co-located instances for solitary minikube virtual machines. +antiAffinity: "soft" + +# Shrink default JVM heap. +esJavaOpts: "-Xmx128m -Xms128m" + +# Allocate smaller chunks of memory per pod. +resources: + requests: + cpu: "100m" + memory: "512M" + limits: + cpu: "1000m" + memory: "512M" + +# Request smaller persistent volumes. +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "local-path" + resources: + requests: + storage: 100M diff --git a/elasticsearch/examples/microk8s/Makefile b/elasticsearch/examples/microk8s/Makefile index 2c7d3d394..3ce3ebdef 100644 --- a/elasticsearch/examples/microk8s/Makefile +++ b/elasticsearch/examples/microk8s/Makefile @@ -3,10 +3,10 @@ default: test RELEASE := helm-es-microk8s install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/microk8s/README.md b/elasticsearch/examples/microk8s/README.md index 487899d85..69df1e5ef 100644 --- a/elasticsearch/examples/microk8s/README.md +++ b/elasticsearch/examples/microk8s/README.md @@ -1,6 +1,6 @@ # MicroK8S -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [MicroK8S][] +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster on [MicroK8S][] using [custom values][]. Note that this configuration should be used for test only and isn't recommended @@ -28,5 +28,5 @@ The following MicroK8S [addons][] need to be enabled: [addons]: https://microk8s.io/docs/addons -[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/microk8s/values.yaml +[custom values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/microk8s/values.yaml [MicroK8S]: https://microk8s.io diff --git a/elasticsearch/examples/migration/Makefile b/elasticsearch/examples/migration/Makefile index 3b1dac105..efd4abfc5 100644 --- a/elasticsearch/examples/migration/Makefile +++ b/elasticsearch/examples/migration/Makefile @@ -1,10 +1,10 @@ PREFIX := helm-es-migration data: - helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ + helm upgrade --wait --timeout=900s --install --values ./data.yml $(PREFIX)-data ../../ master: - helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ + helm upgrade --wait --timeout=900s --install --values ./master.yml $(PREFIX)-master ../../ client: - helm upgrade --wait --timeout=600 --install --values ./client.yml $(PREFIX)-client ../../ + helm upgrade --wait --timeout=900s --install --values ./client.yml $(PREFIX)-client ../../ diff --git a/elasticsearch/examples/migration/README.md b/elasticsearch/examples/migration/README.md index b492d515d..f962224b5 100644 --- a/elasticsearch/examples/migration/README.md +++ b/elasticsearch/examples/migration/README.md @@ -160,8 +160,8 @@ client nodes: working correctly you can cleanup leftover resources from your old cluster. [basic license]: https://www.elastic.co/subscriptions -[data.yml]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/migration/data.yml +[data.yml]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/examples/migration/data.yml [helm/charts]: https://github.com/helm/charts/tree/master/stable/elasticsearch -[master.yml]: https://github.com/elastic/helm-charts/blob/7.7/elasticsearch/examples/migration/master.yml -[restoring to a different cluster guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/modules-snapshots.html#_restoring_to_a_different_cluster -[rolling upgrades guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.6/rolling-upgrades.html +[master.yml]: https://github.com/elastic/helm-charts/blob/6.8/elasticsearch/examples/migration/master.yml +[restoring to a different cluster guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/modules-snapshots.html#_restoring_to_a_different_cluster +[rolling upgrades guide]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/rolling-upgrades.html diff --git a/elasticsearch/examples/migration/master.yml b/elasticsearch/examples/migration/master.yml index 3e3a2f172..6e40a4d08 100644 --- a/elasticsearch/examples/migration/master.yml +++ b/elasticsearch/examples/migration/master.yml @@ -3,6 +3,7 @@ # Temporarily set to 3 so we can scale up/down the old a new cluster # one at a time whilst always keeping 3 masters running replicas: 1 +minimumMasterNodes: 1 esMajorVersion: 6 diff --git a/elasticsearch/examples/minikube/Makefile b/elasticsearch/examples/minikube/Makefile index 97109ce8c..09b61f81d 100644 --- a/elasticsearch/examples/minikube/Makefile +++ b/elasticsearch/examples/minikube/Makefile @@ -3,10 +3,10 @@ default: test RELEASE := helm-es-minikube install: - helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + helm upgrade --wait --timeout=1200s --install --values values.yaml $(RELEASE) ../../ test: install helm test $(RELEASE) purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/minikube/README.md b/elasticsearch/examples/minikube/README.md index 08f4f4fc4..2a939150c 100644 --- a/elasticsearch/examples/minikube/README.md +++ b/elasticsearch/examples/minikube/README.md @@ -1,6 +1,6 @@ # Minikube -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [Minikube][] +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster on [Minikube][] using [custom values][]. If helm or kubectl timeouts occur, you may consider creating a minikube VM with @@ -34,5 +34,5 @@ minikube addons enable storage-provisioner ``` -[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/minikube/values.yaml +[custom values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/minikube/values.yaml [minikube]: https://minikube.sigs.k8s.io/docs/ diff --git a/elasticsearch/examples/multi/Makefile b/elasticsearch/examples/multi/Makefile index 836ec2e0f..bb3da7267 100644 --- a/elasticsearch/examples/multi/Makefile +++ b/elasticsearch/examples/multi/Makefile @@ -6,11 +6,13 @@ PREFIX := helm-es-multi RELEASE := helm-es-multi-master install: - helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ - helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ + helm upgrade --wait --timeout=900s --install --values ./master.yml $(PREFIX)-master ../../ + helm upgrade --wait --timeout=900s --install --values ./data.yml $(PREFIX)-data ../../ + helm upgrade --wait --timeout=900s --install --values ./client.yml $(PREFIX)-client ../../ test: install goss purge: - helm del --purge $(PREFIX)-master - helm del --purge $(PREFIX)-data + helm del $(PREFIX)-master + helm del $(PREFIX)-data + helm del $(PREFIX)-client diff --git a/elasticsearch/examples/multi/README.md b/elasticsearch/examples/multi/README.md index 410391dbd..7368bb1be 100644 --- a/elasticsearch/examples/multi/README.md +++ b/elasticsearch/examples/multi/README.md @@ -1,14 +1,15 @@ # Multi -This example deploy an Elasticsearch 7.7.0-SNAPSHOT cluster composed of 2 different Helm +This example deploy an Elasticsearch 6.8.14-SNAPSHOT cluster composed of 3 different Helm releases: - `helm-es-multi-master` for the 3 master nodes using [master values][] - `helm-es-multi-data` for the 3 data nodes using [data values][] +- `helm-es-multi-client` for the 3 client nodes using [client values][] ## Usage -* Deploy the 2 Elasticsearch releases: `make install` +* Deploy the 3 Elasticsearch releases: `make install` * You can now setup a port forward to query Elasticsearch API: @@ -22,6 +23,7 @@ releases: You can also run [goss integration tests][] using `make test` -[data values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/data.yaml -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/test/goss.yaml -[master values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/multi/master.yaml +[client values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi/client.yml +[data values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi/data.yml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi/test/goss.yaml +[master values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/multi/master.yml diff --git a/elasticsearch/examples/multi/client.yml b/elasticsearch/examples/multi/client.yml new file mode 100644 index 000000000..e5d9f0e33 --- /dev/null +++ b/elasticsearch/examples/multi/client.yml @@ -0,0 +1,12 @@ +--- + +clusterName: "multi" +nodeGroup: "client" + +roles: + master: "false" + ingest: "false" + data: "false" + +persistence: + enabled: false diff --git a/elasticsearch/examples/multi/test/goss.yaml b/elasticsearch/examples/multi/test/goss.yaml index 18cb25063..794416b8f 100644 --- a/elasticsearch/examples/multi/test/goss.yaml +++ b/elasticsearch/examples/multi/test/goss.yaml @@ -5,5 +5,5 @@ http: body: - 'green' - '"cluster_name":"multi"' - - '"number_of_nodes":6' + - '"number_of_nodes":9' - '"number_of_data_nodes":3' diff --git a/elasticsearch/examples/openshift/Makefile b/elasticsearch/examples/openshift/Makefile index 6e495916d..653288854 100644 --- a/elasticsearch/examples/openshift/Makefile +++ b/elasticsearch/examples/openshift/Makefile @@ -7,9 +7,9 @@ template: helm template --values ./values.yaml ../../ install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/openshift/README.md b/elasticsearch/examples/openshift/README.md index 53a924a25..46455a141 100644 --- a/elasticsearch/examples/openshift/README.md +++ b/elasticsearch/examples/openshift/README.md @@ -1,6 +1,6 @@ # OpenShift -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster on [OpenShift][] +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster on [OpenShift][] using [custom values][]. ## Usage @@ -19,6 +19,6 @@ using [custom values][]. You can also run [goss integration tests][] using `make test` -[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/values.yaml -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/test/goss.yaml +[custom values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift/values.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift/test/goss.yaml [openshift]: https://www.openshift.com/ diff --git a/elasticsearch/examples/openshift/test/goss.yaml b/elasticsearch/examples/openshift/test/goss.yaml index b6141d4b9..b1853e8f7 100644 --- a/elasticsearch/examples/openshift/test/goss.yaml +++ b/elasticsearch/examples/openshift/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.7.0"' + - '"number" : "6.8.14"' - '"cluster_name" : "elasticsearch"' - '"name" : "elasticsearch-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/openshift/values.yaml b/elasticsearch/examples/openshift/values.yaml index 7f5cd8494..8a211268b 100644 --- a/elasticsearch/examples/openshift/values.yaml +++ b/elasticsearch/examples/openshift/values.yaml @@ -5,6 +5,7 @@ securityContext: podSecurityContext: fsGroup: null + runAsUser: null sysctlInitContainer: enabled: false diff --git a/elasticsearch/examples/oss/Makefile b/elasticsearch/examples/oss/Makefile index e274659c6..e57284176 100644 --- a/elasticsearch/examples/oss/Makefile +++ b/elasticsearch/examples/oss/Makefile @@ -4,9 +4,9 @@ include ../../../helpers/examples.mk RELEASE := helm-es-oss install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values ./values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/elasticsearch/examples/oss/README.md b/elasticsearch/examples/oss/README.md index 67a36b114..8c54a32d8 100644 --- a/elasticsearch/examples/oss/README.md +++ b/elasticsearch/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT cluster using +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT cluster using [Elasticsearch OSS][] version. ## Usage @@ -20,4 +20,4 @@ You can also run [goss integration tests][] using `make test` [elasticsearch oss]: https://www.elastic.co/downloads/elasticsearch-oss -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/test/goss.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss/test/goss.yaml diff --git a/elasticsearch/examples/oss/test/goss.yaml b/elasticsearch/examples/oss/test/goss.yaml index 8240c8c84..a4c6ba56b 100644 --- a/elasticsearch/examples/oss/test/goss.yaml +++ b/elasticsearch/examples/oss/test/goss.yaml @@ -11,7 +11,7 @@ http: status: 200 timeout: 2000 body: - - '"number" : "7.7.0-SNAPSHOT"' + - '"number" : "6.8.14"' - '"cluster_name" : "oss"' - '"name" : "oss-master-0"' - 'You Know, for Search' diff --git a/elasticsearch/examples/security/Makefile b/elasticsearch/examples/security/Makefile index d8c6f11c7..c92916085 100644 --- a/elasticsearch/examples/security/Makefile +++ b/elasticsearch/examples/security/Makefile @@ -6,11 +6,11 @@ RELEASE := helm-es-security ELASTICSEARCH_IMAGE := docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) install: - helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install --values ./security.yml $(RELEASE) ../../ purge: - kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true - helm del --purge $(RELEASE) + kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem elastic-certificate-crt|| true + helm del $(RELEASE) test: secrets install goss @@ -19,7 +19,7 @@ pull-elasticsearch-image: secrets: docker rm -f elastic-helm-charts-certs || true - rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12 || true password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm busybox:1.31.1 /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \ docker run --name elastic-helm-charts-certs -i -w /app \ $(ELASTICSEARCH_IMAGE) \ @@ -29,7 +29,9 @@ secrets: docker cp elastic-helm-charts-certs:/app/elastic-certificates.p12 ./ && \ docker rm -f elastic-helm-charts-certs && \ openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \ + openssl x509 -outform der -in elastic-certificate.pem -out elastic-certificate.crt && \ kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \ + kubectl create secret generic elastic-certificate-crt --from-file=elastic-certificate.crt && \ kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \ - rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-certificate.crt elastic-stack-ca.p12 diff --git a/elasticsearch/examples/security/README.md b/elasticsearch/examples/security/README.md index 1fdcf0d94..7ad74d338 100644 --- a/elasticsearch/examples/security/README.md +++ b/elasticsearch/examples/security/README.md @@ -1,6 +1,6 @@ # Security -This example deploy a 3 nodes Elasticsearch 7.7.0-SNAPSHOT with authentication and +This example deploy a 3 nodes Elasticsearch 6.8.14-SNAPSHOT with authentication and autogenerated certificates for TLS (see [values][]). Note that this configuration should be used for test only. For a production @@ -24,6 +24,6 @@ deployment you should generate SSL certificates following the [official docs][]. You can also run [goss integration tests][] using `make test` -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/test/goss.yaml -[official docs]: https://www.elastic.co/guide/en/elasticsearch/reference/7.7/configuring-tls.html#node-certificates -[values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/security.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/test/goss.yaml +[official docs]: https://www.elastic.co/guide/en/elasticsearch/reference/6.8/configuring-tls.html#node-certificates +[values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/security.yaml diff --git a/elasticsearch/examples/upgrade/Makefile b/elasticsearch/examples/upgrade/Makefile deleted file mode 100644 index 9e1e6fd5e..000000000 --- a/elasticsearch/examples/upgrade/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -default: test - -include ../../../helpers/examples.mk - -RELEASE := helm-es-upgrade - -# Right now the version is hardcoded because helm install will ignore -# anything with an alpha tag when trying to install the latest release -# This hardcoding can be removed once we drop the alpha tag -# The "--set terminationGracePeriod=121" always makes sure that a rolling -# upgrade is forced for this test -install: - helm repo add elastic https://helm.elastic.co && \ - helm upgrade --wait --timeout=600 --install $(RELEASE) elastic/elasticsearch --version 7.0.0-alpha1 --set clusterName=upgrade ; \ - kubectl rollout status sts/upgrade-master --timeout=600s - helm upgrade --wait --timeout=600 --set terminationGracePeriod=121 --install $(RELEASE) ../../ --set clusterName=upgrade ; \ - kubectl rollout status sts/upgrade-master --timeout=600s - -init: - helm init --client-only - -test: init install goss - -purge: - helm del --purge $(RELEASE) diff --git a/elasticsearch/examples/upgrade/README.md b/elasticsearch/examples/upgrade/README.md deleted file mode 100644 index 28cdb46e9..000000000 --- a/elasticsearch/examples/upgrade/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Upgrade - -This example deploy a 3 nodes Elasticsearch cluster using [7.0.0-alpha1][] chart -version, then upgrade it to 7.7.0-SNAPSHOT version. - - -## Usage - -Running `make install` command will do both 7.0.0-alpha1 install and 7.7.0-SNAPSHOT -upgrade - - -## Testing - -You can also run [goss integration tests][] using `make test` - - -[7.0.0-alpha1]: https://github.com/elastic/helm-charts/releases/tag/7.0.0-alpha1 -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/upgrade/test/goss.yaml diff --git a/elasticsearch/examples/upgrade/test/goss.yaml b/elasticsearch/examples/upgrade/test/goss.yaml deleted file mode 100644 index 4f062c242..000000000 --- a/elasticsearch/examples/upgrade/test/goss.yaml +++ /dev/null @@ -1,17 +0,0 @@ -http: - http://localhost:9200/_cluster/health: - status: 200 - timeout: 2000 - body: - - 'green' - - '"number_of_nodes":3' - - '"number_of_data_nodes":3' - - http://localhost:9200: - status: 200 - timeout: 2000 - body: - - '"number" : "7.7.0-SNAPSHOT"' - - '"cluster_name" : "upgrade"' - - '"name" : "upgrade-master-0"' - - 'You Know, for Search' diff --git a/elasticsearch/templates/NOTES.txt b/elasticsearch/templates/NOTES.txt index 3841adafc..73edf425a 100755 --- a/elasticsearch/templates/NOTES.txt +++ b/elasticsearch/templates/NOTES.txt @@ -1,4 +1,4 @@ 1. Watch all cluster members come up. $ kubectl get pods --namespace={{ .Release.Namespace }} -l app={{ template "elasticsearch.uname" . }} -w 2. Test cluster health using Helm test. - $ helm test {{ .Release.Name }} --cleanup + $ helm test {{ .Release.Name }} diff --git a/elasticsearch/templates/_helpers.tpl b/elasticsearch/templates/_helpers.tpl index f7f212893..7aaebc2ce 100755 --- a/elasticsearch/templates/_helpers.tpl +++ b/elasticsearch/templates/_helpers.tpl @@ -43,14 +43,6 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} -{{- define "elasticsearch.endpoints" -}} -{{- $replicas := int (toString (.Values.replicas)) }} -{{- $uname := printf "%s-%s" .Values.clusterName .Values.nodeGroup }} - {{- range $i, $e := untilStep 0 $replicas 1 -}} -{{ $uname }}-{{ $i }}, - {{- end -}} -{{- end -}} - {{- define "elasticsearch.esMajorVersion" -}} {{- if .Values.esMajorVersion -}} {{ .Values.esMajorVersion }} @@ -59,7 +51,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- if and (contains "docker.elastic.co/elasticsearch/elasticsearch" .Values.image) (not (eq $version 0)) -}} {{ $version }} {{- else -}} -7 +6 {{- end -}} {{- end -}} {{- end -}} @@ -68,7 +60,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Return the appropriate apiVersion for statefulset. */}} {{- define "elasticsearch.statefulset.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} {{- print "apps/v1beta2" -}} {{- else -}} {{- print "apps/v1" -}} @@ -79,7 +71,7 @@ Return the appropriate apiVersion for statefulset. Return the appropriate apiVersion for ingress. */}} {{- define "elasticsearch.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.Version -}} {{- print "extensions/v1beta1" -}} {{- else -}} {{- print "networking.k8s.io/v1beta1" -}} diff --git a/elasticsearch/templates/service.yaml b/elasticsearch/templates/service.yaml index 4572af078..ee7ba5c9c 100644 --- a/elasticsearch/templates/service.yaml +++ b/elasticsearch/templates/service.yaml @@ -20,7 +20,6 @@ metadata: spec: type: {{ .Values.service.type }} selector: - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}" app: "{{ template "elasticsearch.uname" . }}" @@ -41,6 +40,9 @@ spec: loadBalancerSourceRanges: {{ toYaml . | indent 4 }} {{- end }} +{{- if .Values.service.externalTrafficPolicy }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy }} +{{- end }} --- kind: Service apiVersion: v1 diff --git a/elasticsearch/templates/serviceaccount.yaml b/elasticsearch/templates/serviceaccount.yaml index c85e37554..801d1cf90 100644 --- a/elasticsearch/templates/serviceaccount.yaml +++ b/elasticsearch/templates/serviceaccount.yaml @@ -8,6 +8,10 @@ metadata: {{- else }} name: {{ .Values.rbac.serviceAccountName | quote }} {{- end }} + annotations: + {{- with .Values.rbac.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} labels: heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} diff --git a/elasticsearch/templates/statefulset.yaml b/elasticsearch/templates/statefulset.yaml index 5297aac25..c4ed1ebca 100644 --- a/elasticsearch/templates/statefulset.yaml +++ b/elasticsearch/templates/statefulset.yaml @@ -26,6 +26,15 @@ spec: volumeClaimTemplates: - metadata: name: {{ template "elasticsearch.uname" . }} + {{- if .Values.persistence.labels.enabled }} + labels: + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "elasticsearch.uname" . }}" + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} {{- with .Values.persistence.annotations }} annotations: {{ toYaml . | indent 8 }} @@ -37,7 +46,6 @@ spec: metadata: name: "{{ template "elasticsearch.uname" . }}" labels: - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} chart: "{{ .Chart.Name }}" app: "{{ template "elasticsearch.uname" . }}" @@ -113,6 +121,9 @@ spec: - name: {{ .name }} secret: secretName: {{ .secretName }} + {{- if .defaultMode }} + defaultMode: {{ .defaultMode }} + {{- end }} {{- end }} {{- if .Values.esConfig }} - name: esconfig @@ -140,6 +151,9 @@ spec: {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + {{- if semverCompare ">1.13-0" .Capabilities.KubeVersion.Version }} + enableServiceLinks: {{ .Values.enableServiceLinks }} {{- end }} initContainers: {{- if .Values.sysctlInitContainer.enabled }} @@ -213,22 +227,36 @@ spec: - -c - | #!/usr/bin/env bash -e - # If the node is starting up wait for the cluster to be ready (request params: '{{ .Values.clusterHealthCheckParams }}' ) + # If the node is starting up wait for the cluster to be ready (request params: "{{ .Values.clusterHealthCheckParams }}" ) # Once it has started only check that the node itself is responding START_FILE=/tmp/.es_start_file - if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then - BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" - else - BASIC_AUTH='' - fi + # Disable nss cache to avoid filling dentry cache when calling curl + # This is required with Elasticsearch Docker using nss < 3.52 + export NSS_SDB_USE_CACHE=no + + http () { + local path="${1}" + local args="${2}" + set -- -XGET -s + + if [ "$args" != "" ]; then + set -- "$@" $args + fi + + if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then + set -- "$@" -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" + fi + + curl --output /dev/null -k "$@" "{{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}${path}" + } if [ -f "${START_FILE}" ]; then echo 'Elasticsearch is already running, lets check the node is healthy' - HTTP_CODE=$(curl -XGET -s -k ${BASIC_AUTH} -o /dev/null -w '%{http_code}' {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/) + HTTP_CODE=$(http "/" "-w %{http_code}") RC=$? if [[ ${RC} -ne 0 ]]; then - echo "curl -XGET -s -k \${BASIC_AUTH} -o /dev/null -w '%{http_code}' {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/ failed with RC ${RC}" + echo "curl --output /dev/null -k -XGET -s -w '%{http_code}' \${BASIC_AUTH} {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/ failed with RC ${RC}" exit ${RC} fi # ready if HTTP code 200, 503 is tolerable if ES version is 6.x @@ -237,13 +265,13 @@ spec: elif [[ ${HTTP_CODE} == "503" && "{{ include "elasticsearch.esMajorVersion" . }}" == "6" ]]; then exit 0 else - echo "curl -XGET -s -k \${BASIC_AUTH} -o /dev/null -w '%{http_code}' {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/ failed with HTTP code ${HTTP_CODE}" + echo "curl --output /dev/null -k -XGET -s -w '%{http_code}' \${BASIC_AUTH} {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/ failed with HTTP code ${HTTP_CODE}" exit 1 fi else echo 'Waiting for elasticsearch cluster to become ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' - if curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}/_cluster/health?{{ .Values.clusterHealthCheckParams }} ; then + if http "/_cluster/health?{{ .Values.clusterHealthCheckParams }}" "--fail" ; then touch ${START_FILE} exit 0 else @@ -265,14 +293,9 @@ spec: fieldRef: fieldPath: metadata.name {{- if eq .Values.roles.master "true" }} - {{- if ge (int (include "elasticsearch.esMajorVersion" .)) 7 }} - - name: cluster.initial_master_nodes - value: "{{ template "elasticsearch.endpoints" . }}" - {{- else }} - name: discovery.zen.minimum_master_nodes value: "{{ .Values.minimumMasterNodes }}" {{- end }} - {{- end }} {{- if lt (int (include "elasticsearch.esMajorVersion" .)) 7 }} - name: discovery.zen.ping.unicast.hosts value: "{{ template "elasticsearch.masterService" . }}-headless" diff --git a/elasticsearch/templates/test/test-elasticsearch-health.yaml b/elasticsearch/templates/test/test-elasticsearch-health.yaml index 0aac728a8..a278b140a 100644 --- a/elasticsearch/templates/test/test-elasticsearch-health.yaml +++ b/elasticsearch/templates/test/test-elasticsearch-health.yaml @@ -11,6 +11,7 @@ spec: containers: - name: "{{ .Release.Name }}-{{ randAlpha 5 | lower }}-test" image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" command: - "sh" - "-c" diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py index 82e5d279b..53612b2ad 100755 --- a/elasticsearch/tests/elasticsearch_test.py +++ b/elasticsearch/tests/elasticsearch_test.py @@ -38,11 +38,7 @@ def test_defaults(): "name": "node.name", "valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}}, }, - { - "name": "cluster.initial_master_nodes", - "value": uname + "-0," + uname + "-1," + uname + "-2,", - }, - {"name": "discovery.seed_hosts", "value": uname + "-headless"}, + {"name": "discovery.zen.ping.unicast.hosts", "value": uname + "-headless"}, {"name": "network.host", "value": "0.0.0.0"}, {"name": "cluster.name", "value": clusterName}, {"name": "ES_JAVA_OPTS", "value": "-Xmx1g -Xms1g"}, @@ -84,8 +80,10 @@ def test_defaults(): assert c["volumeMounts"][0]["mountPath"] == "/usr/share/elasticsearch/data" assert c["volumeMounts"][0]["name"] == uname + # volumeClaimTemplates v = r["statefulset"][uname]["spec"]["volumeClaimTemplates"][0] assert v["metadata"]["name"] == uname + assert "labels" not in v["metadata"] assert v["spec"]["accessModes"] == ["ReadWriteOnce"] assert v["spec"]["resources"]["requests"]["storage"] == "30Gi" @@ -213,54 +211,6 @@ def test_set_master_service_to_default_nodegroup_name_with_custom_cluster_name() } in env -def test_set_initial_master_nodes_when_using_v_7(): - config = """ -esMajorVersion: 7 -roles: - master: "true" -""" - r = helm_template(config) - env = r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0]["env"] - assert { - "name": "cluster.initial_master_nodes", - "value": "elasticsearch-master-0," - + "elasticsearch-master-1," - + "elasticsearch-master-2,", - } in env - - for e in env: - assert e["name"] != "discovery.zen.minimum_master_nodes" - - -def test_dont_set_initial_master_nodes_if_not_master_when_using_es_version_7(): - config = """ -esMajorVersion: 7 -roles: - master: "false" -""" - r = helm_template(config) - env = r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0]["env"] - for e in env: - assert e["name"] != "cluster.initial_master_nodes" - - -def test_set_discovery_seed_host_when_using_v_7(): - config = """ -esMajorVersion: 7 -roles: - master: "true" -""" - r = helm_template(config) - env = r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0]["env"] - assert { - "name": "discovery.seed_hosts", - "value": "elasticsearch-master-headless", - } in env - - for e in env: - assert e["name"] != "discovery.zen.ping.unicast.hosts" - - def test_enabling_machine_learning_role(): config = """ roles: @@ -479,6 +429,23 @@ def test_adding_multiple_persistence_annotations(): assert annotations["world"] == "hello" +def test_enabling_persistence_label_in_volumeclaimtemplate(): + config = """ +persistence: + labels: + enabled: true +""" + r = helm_template(config) + volume_claim_template_labels = r["statefulset"][uname]["spec"][ + "volumeClaimTemplates" + ][0]["metadata"]["labels"] + statefulset_labels = r["statefulset"][uname]["metadata"]["labels"] + expected_labels = statefulset_labels + # heritage label shouldn't be present in volumeClaimTemplates labels + expected_labels.pop("heritage") + assert volume_claim_template_labels == expected_labels + + def test_adding_a_secret_mount(): config = """ secretMounts: @@ -514,6 +481,24 @@ def test_adding_a_secret_mount_with_subpath(): } +def test_adding_a_secret_mount_with_default_mode(): + config = """ +secretMounts: + - name: elastic-certificates + secretName: elastic-certs + path: /usr/share/elasticsearch/config/certs + subPath: cert.crt + defaultMode: 0755 +""" + r = helm_template(config) + s = r["statefulset"][uname]["spec"]["template"]["spec"] + assert s["containers"][0]["volumeMounts"][-1] == { + "mountPath": "/usr/share/elasticsearch/config/certs", + "subPath": "cert.crt", + "name": "elastic-certificates", + } + + def test_adding_image_pull_secrets(): config = """ imagePullSecrets: @@ -558,6 +543,22 @@ def test_adding_pod_annotations(): ) +def test_adding_serviceaccount_annotations(): + config = """ +rbac: + create: true + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][uname]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) + + def test_adding_a_node_selector(): config = """ nodeSelector: @@ -812,6 +813,23 @@ def test_adding_a_loadBalancerIP(): assert r["service"][uname]["spec"]["loadBalancerIP"] == "12.4.19.81" +def test_adding_an_externalTrafficPolicy(): + config = "" + + r = helm_template(config) + + assert "externalTrafficPolicy" not in r["service"][uname]["spec"] + + config = """ + service: + externalTrafficPolicy: Local + """ + + r = helm_template(config) + + assert r["service"][uname]["spec"]["externalTrafficPolicy"] == "Local" + + def test_adding_a_label_on_non_headless_service(): config = "" @@ -908,26 +926,26 @@ def test_esMajorVersion_detect_default_version(): config = "" r = helm_template(config) - assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "7" + assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "6" -def test_esMajorVersion_default_to_7_if_not_elastic_image(): +def test_esMajorVersion_default_to_6_if_not_elastic_image(): config = """ image: notElastic imageTag: 1.0.0 """ r = helm_template(config) - assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "7" + assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "6" -def test_esMajorVersion_default_to_7_if_no_version_is_found(): +def test_esMajorVersion_default_to_6_if_no_version_is_found(): config = """ imageTag: not_a_number """ r = helm_template(config) - assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "7" + assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "6" def test_esMajorVersion_set_to_6_based_on_image_tag(): @@ -939,16 +957,6 @@ def test_esMajorVersion_set_to_6_based_on_image_tag(): assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "6" -def test_esMajorVersion_always_wins(): - config = """ - esMajorVersion: 7 - imageTag: 6.0.0 - """ - - r = helm_template(config) - assert r["statefulset"][uname]["metadata"]["annotations"]["esMajorVersion"] == "7" - - def test_esMajorVersion_parse_image_tag_for_oss_image(): config = """ image: docker.elastic.co/elasticsearch/elasticsearch-oss diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index 1f7b0bab3..59a873015 100755 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -48,9 +48,10 @@ secretMounts: [] # - name: elastic-certificates # secretName: elastic-certificates # path: /usr/share/elasticsearch/config/certs +# defaultMode: 0755 image: "docker.elastic.co/elasticsearch/elasticsearch" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" podAnnotations: {} @@ -95,6 +96,7 @@ volumeClaimTemplate: rbac: create: false + serviceAccountAnnotations: {} serviceAccountName: "" podSecurityPolicy: @@ -117,6 +119,9 @@ podSecurityPolicy: persistence: enabled: true + labels: + # Add default labels for the volumeClaimTemplate fo the StatefulSet + enabled: false annotations: {} extraVolumes: [] @@ -158,6 +163,11 @@ nodeAffinity: {} # the same time when bootstrapping the cluster podManagementPolicy: "Parallel" +# The environment variables injected by service links are not used, but can lead to slow Elasticsearch boot times when +# there are many services in the current namespace. +# If you experience slow pod startups you probably want to set this to `false`. +enableServiceLinks: true + protocol: http httpPort: 9200 transportPort: 9300 @@ -172,6 +182,7 @@ service: transportPortName: transport loadBalancerIP: "" loadBalancerSourceRanges: [] + externalTrafficPolicy: "" updateStrategy: RollingUpdate @@ -204,7 +215,7 @@ readinessProbe: successThreshold: 3 timeoutSeconds: 5 -# https://www.elastic.co/guide/en/elasticsearch/reference/7.7/cluster-health.html#request-params wait_for_status +# https://www.elastic.co/guide/en/elasticsearch/reference/6.8/cluster-health.html#request-params wait_for_status clusterHealthCheckParams: "wait_for_status=green&timeout=1s" ## Use an alternate scheduler. diff --git a/filebeat/Chart.yaml b/filebeat/Chart.yaml index e5891a665..ce4859f79 100755 --- a/filebeat/Chart.yaml +++ b/filebeat/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: filebeat -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/beats.png diff --git a/filebeat/README.md b/filebeat/README.md index 11ae6a9cd..0b79b0932 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -1,12 +1,19 @@ # Filebeat Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[Filebeat Docker image][]. + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -14,6 +21,7 @@ - [How to use Filebeat with Elasticsearch with security (authentication and TLS) enabled?](#how-to-use-filebeat-with-elasticsearch-with-security-authentication-and-tls-enabled) - [How to install OSS version of Filebeat?](#how-to-install-oss-version-of-filebeat) - [Why is Filebeat host.name field set to Kubernetes pod name?](#why-is-filebeat-hostname-field-set-to-kubernetes-pod-name) + - [How to change readinessProbe for outputs which don't support testing](#how-to-change-readinessprobe-for-outputs-which-dont-support-testing) - [Contributing](#contributing) @@ -21,45 +29,37 @@ -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[Filebeat Docker image][]. - - ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 (see [parent README][] for more details) +* [Helm][] >=2.8.0 and <3.0.0 * Kubernetes >=1.9 +See [supported configurations][] for more details. + ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name filebeat elastic/filebeat --version=7.7.0` +* Install it: + - with Helm 3: `helm install filebeat --version elastic/filebeat` + - with Helm 2 (deprecated): `helm install --name filebeat --version elastic/filebeat` + -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name filebeat ./helm-charts/filebeat` +* Install it: + - with Helm 3: `helm install filebeat ./helm-charts/filebeat --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name filebeat ./helm-charts/filebeat --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -92,38 +92,40 @@ as a reference. They are also used in the automated testing of this chart. ## Configuration -| Parameter | Description | Default | -|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| -| `affinity` | Configurable [affinity][] | `{}` | -| `envFrom` | Templatable string of envFrom to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | -| `extraContainers` | List of additional init containers to be added at the DaemonSet | `""` | -| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| `extraInitContainers` | List of additional init containers to be added at the DaemonSet. It also accepts a templatable string of additional containers to be passed to the `tpl` function | `[]` | -| `extraVolumeMounts` | List of additional volumeMounts to be mounted on the DaemonSet | `[]` | -| `extraVolumes` | List of additional volumes to be mounted on the DaemonSet | `[]` | -| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` | see [values.yaml][] | -| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | -| `hostNetworking` | Use host networking in the DaemonSet so that hostname is reported correctly | `false` | -| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist Filebeat registry data | `/var/lib` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | -| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Filebeat Docker image tag | `7.7.0-SNAPSHOT` | -| `image` | The Filebeat Docker image | `docker.elastic.co/beats/filebeat` | -| `labels` | Configurable [labels][] applied to all Filebeat pods | `{}` | -| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | -| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | -| `nodeSelector` | Configurable [nodeSelector][] | `{}` | -| `podAnnotations` | Configurable [annotations][] applied to all Filebeat pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext][] for Filebeat pod execution environment | see [values.yaml][] | -| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | -| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `resources` | Allows you to set the [resources][] for the `DaemonSet` | see [values.yaml][] | -| `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `serviceAccount` | Custom [serviceAccount][] that Filebeat will use during execution. By default will use the service account created by this chart | `""` | -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Filebeat pod process on pod shutdown | `30` | -| `tolerations` | Configurable [tolerations][] | `[]` | -| `updateStrategy` | The [updateStrategy][] for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | +| Parameter | Description | Default | +|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| `affinity` | Configurable [affinity][] | `{}` | +| `envFrom` | Templatable string of envFrom to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | +| `extraContainers` | List of additional init containers to be added at the DaemonSet | `""` | +| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | +| `extraInitContainers` | List of additional init containers to be added at the DaemonSet. It also accepts a templatable string of additional containers to be passed to the `tpl` function | `[]` | +| `extraVolumeMounts` | List of additional volumeMounts to be mounted on the DaemonSet | `[]` | +| `extraVolumes` | List of additional volumes to be mounted on the DaemonSet | `[]` | +| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` | see [values.yaml][] | +| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | +| `hostNetworking` | Use host networking in the DaemonSet so that hostname is reported correctly | `false` | +| `dnsConfig` | Configurable [dnsConfig][] | `{}` | +| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist Filebeat registry data | `/var/lib` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | +| `imageTag` | The Filebeat Docker image tag | `6.8.14-SNAPSHOT` | +| `image` | The Filebeat Docker image | `docker.elastic.co/beats/filebeat` | +| `labels` | Configurable [labels][] applied to all Filebeat pods | `{}` | +| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | +| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | +| `nodeSelector` | Configurable [nodeSelector][] | `{}` | +| `podAnnotations` | Configurable [annotations][] applied to all Filebeat pods | `{}` | +| `podSecurityContext` | Configurable [podSecurityContext][] for Filebeat pod execution environment | see [values.yaml][] | +| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | +| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `resources` | Allows you to set the [resources][] for the `DaemonSet` | see [values.yaml][] | +| `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `serviceAccount` | Custom [serviceAccount][] that Filebeat will use during execution. By default will use the service account created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Filebeat pod process on pod shutdown | `30` | +| `tolerations` | Configurable [tolerations][] | `[]` | +| `updateStrategy` | The [updateStrategy][] for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | ## FAQ @@ -150,47 +152,73 @@ The default Filebeat configuration is using Filebeat pod name for `agent.hostname` and `host.name` fields. The `hostname` of the Kubernetes nodes can be find in `kubernetes.node.name` field. If you would like to have `agent.hostname` and `host.name` fields set to the hostname of the nodes, you'll -need to set `daemonset.hostNetworking` value to true. +need to set `hostNetworking` value to true. Note that enabling [hostNetwork][] make Filebeat pod use the host network namespace which gives it access to the host loopback device, services listening on localhost, could be used to snoop on network activity of other pods on the same node. +### How to change readinessProbe for outputs which don't support testing + +Some [Filebeat outputs][] like [Kafka output][] don't support testing using +`filebeat test output` command which is used by Filebeat chart readiness probe. + +This makes Filebeat pods crash before being ready with the following message: +`Readiness probe failed: kafka output doesn't support testing`. + +The workaround when using this kind of output is to override the readiness probe +command to check Filebeat API instead (same as existing liveness probe). + +``` +readinessProbe: + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + curl --fail 127.0.0.1:5066 +``` + ## Contributing Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/filebeat/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -[default Filebeat Helm chart]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/README.md#default +[default Filebeat Helm chart]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/oss -[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security -[filebeat docker image]: https://www.elastic.co/guide/en/beats/filebeat/7.7/running-on-docker.html -[filebeat oss docker image]: https://www.docker.elastic.co/#filebeat-7-7-0-oss +[examples]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/security +[filebeat docker image]: https://www.elastic.co/guide/en/beats/filebeat/6.8/running-on-docker.html +[filebeat oss docker image]: https://www.docker.elastic.co/r/beats/filebeat-oss +[filebeat outputs]: https://www.elastic.co/guide/en/beats/filebeat/6.8/configuring-output.html [helm]: https://helm.sh [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces +[dnsConfig]: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/ [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret +[kafka output]: https://www.elastic.co/guide/en/beats/filebeat/6.8/kafka-output.html [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -[parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector [podSecurityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations [serviceAccount]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [updateStrategy]: https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/values.yaml diff --git a/filebeat/examples/default/Makefile b/filebeat/examples/default/Makefile index 6cfa6a648..e27affffc 100644 --- a/filebeat/examples/default/Makefile +++ b/filebeat/examples/default/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-default install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/default/README.md b/filebeat/examples/default/README.md index aee7abde5..35e417b0e 100644 --- a/filebeat/examples/default/README.md +++ b/filebeat/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy Filebeat 7.7.0-SNAPSHOT using [default values][]. +This example deploy Filebeat 6.8.14-SNAPSHOT using [default values][]. ## Usage @@ -22,6 +22,6 @@ This example deploy Filebeat 7.7.0-SNAPSHOT using [default values][]. You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/values.yaml diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml index f0c2458e4..34bed762c 100644 --- a/filebeat/examples/default/test/goss.yaml +++ b/filebeat/examples/default/test/goss.yaml @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.7.0' + - 'filebeat-6.8.14' file: /usr/share/filebeat/filebeat.yml: @@ -44,4 +44,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/filebeat/examples/oss/Makefile b/filebeat/examples/oss/Makefile index e6b665412..455c2e63a 100644 --- a/filebeat/examples/oss/Makefile +++ b/filebeat/examples/oss/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-oss install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/oss/README.md b/filebeat/examples/oss/README.md index b964ab23a..5ef5c7e42 100644 --- a/filebeat/examples/oss/README.md +++ b/filebeat/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy Filebeat 7.7.0-SNAPSHOT using [Filebeat OSS][] version. +This example deploy Filebeat 6.8.14-SNAPSHOT using [Filebeat OSS][] version. ## Usage @@ -23,5 +23,5 @@ You can also run [goss integration tests][] using `make test` [filebeat oss]: https://www.elastic.co/downloads/beats/filebeat-oss -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/oss/test/goss.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/oss/test/goss.yaml diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml index 7f6925e5b..1d71c1e8d 100644 --- a/filebeat/examples/oss/test/goss.yaml +++ b/filebeat/examples/oss/test/goss.yaml @@ -19,4 +19,4 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.7.0' + - 'filebeat-6.8.14' diff --git a/filebeat/examples/security/Makefile b/filebeat/examples/security/Makefile index 6d79d4231..50c0b5bed 100644 --- a/filebeat/examples/security/Makefile +++ b/filebeat/examples/security/Makefile @@ -5,9 +5,9 @@ include ../../../helpers/examples.mk RELEASE := helm-filebeat-security install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/filebeat/examples/security/README.md b/filebeat/examples/security/README.md index c88e277ca..756d55697 100644 --- a/filebeat/examples/security/README.md +++ b/filebeat/examples/security/README.md @@ -1,6 +1,6 @@ # Security -This example deploy Filebeat 7.7.0-SNAPSHOT using authentication and TLS to connect to +This example deploy Filebeat 6.8.14-SNAPSHOT using authentication and TLS to connect to Elasticsearch (see [values][]). @@ -23,6 +23,6 @@ Elasticsearch (see [values][]). You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/filebeat/examples/security/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/filebeat/examples/security/values.yaml diff --git a/filebeat/examples/security/test/goss.yaml b/filebeat/examples/security/test/goss.yaml index 4f44f46b5..fa2807070 100644 --- a/filebeat/examples/security/test/goss.yaml +++ b/filebeat/examples/security/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - 'filebeat-7.7.0' + - 'filebeat-6.8.14' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' diff --git a/filebeat/examples/security/values.yaml b/filebeat/examples/security/values.yaml index 606961fa8..eac92e8fe 100644 --- a/filebeat/examples/security/values.yaml +++ b/filebeat/examples/security/values.yaml @@ -1,15 +1,12 @@ filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log + - type: docker + containers.ids: + - '*' processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" + - add_kubernetes_metadata: + in_cluster: true output.elasticsearch: username: '${ELASTICSEARCH_USERNAME}' diff --git a/filebeat/templates/clusterrole.yaml b/filebeat/templates/clusterrole.yaml index 8bec82cd1..36dc5d310 100644 --- a/filebeat/templates/clusterrole.yaml +++ b/filebeat/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "filebeat.serviceAccount" . }}-cluster-role @@ -13,6 +13,7 @@ rules: - "" resources: - namespaces + - nodes - pods verbs: - get diff --git a/filebeat/templates/clusterrolebinding.yaml b/filebeat/templates/clusterrolebinding.yaml index 45436b76b..887775c2c 100644 --- a/filebeat/templates/clusterrolebinding.yaml +++ b/filebeat/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "filebeat.serviceAccount" . }}-cluster-role-binding diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index eace44437..4b42bc226 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -56,6 +56,9 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet {{- end }} + {{- if .Values.dnsConfig }} + dnsConfig: {{ toYaml .Values.dnsConfig | nindent 8 }} + {{- end }} volumes: {{- range .Values.secretMounts }} - name: {{ .name }} @@ -159,7 +162,7 @@ spec: mountPath: /var/log readOnly: true # Necessary when using autodiscovery; avoid mounting it otherwise - # See: https://www.elastic.co/guide/en/beats/filebeat/7.7/configuration-autodiscover.html + # See: https://www.elastic.co/guide/en/beats/filebeat/6.8/configuration-autodiscover.html - name: varrundockersock mountPath: /var/run/docker.sock readOnly: true diff --git a/filebeat/templates/serviceaccount.yaml b/filebeat/templates/serviceaccount.yaml index f398a58a9..8c0fcc60c 100644 --- a/filebeat/templates/serviceaccount.yaml +++ b/filebeat/templates/serviceaccount.yaml @@ -3,6 +3,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "filebeat.serviceAccount" . }} + annotations: + {{- with .Values.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} labels: app: "{{ template "filebeat.fullname" . }}" chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index a3eaff7c4..7c8dc0ad0 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -296,6 +296,20 @@ def test_adding_pod_labels(): ) +def test_adding_serviceaccount_annotations(): + config = """ +serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][name]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) + + def test_adding_a_node_selector(): config = """ nodeSelector: diff --git a/filebeat/values.yaml b/filebeat/values.yaml index ceb1f7526..372dbef03 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -4,16 +4,12 @@ filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log + - type: docker + containers.ids: + - '*' processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" - + - add_kubernetes_metadata: + in_cluster: true output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' @@ -51,8 +47,12 @@ envFrom: [] # Root directory where Filebeat will write data to in order to persist registry data across pod restarts (file position and other metadata). hostPathRoot: /var/lib hostNetworking: false +dnsConfig: {} +# options: +# - name: ndots +# value: "2" image: "docker.elastic.co/beats/filebeat" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] @@ -110,6 +110,10 @@ resources: # Custom service account override that the pod will use serviceAccount: "" +# Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. +serviceAccountAnnotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount + # A list of secrets and their paths to mount inside the pod # This is useful for mounting certificates for security other sensitive values secretMounts: [] diff --git a/helpers/bumper.py b/helpers/bumper.py index 03dee561a..9dcf547b9 100755 --- a/helpers/bumper.py +++ b/helpers/bumper.py @@ -6,7 +6,6 @@ # # Configurable environment variables: # - BUMPER_VERSION_6 overrides the 6.x.x version. -# - BUMPER_VERSION_7 overrides the 7.x.x version. # - BUMPER_USE_STAGING_IMAGES set to "true" causes the # docker.elastic.co/staging/ docker registry namespace to be used. # @@ -20,10 +19,10 @@ os.chdir(os.path.join(os.path.dirname(__file__), "..")) versions = { - 7: os.environ.get("BUMPER_VERSION_7", "7.7.0-SNAPSHOT"), + 6: os.environ.get("BUMPER_VERSION_6", "6.8.13-SNAPSHOT"), } -chart_version = versions[7] +chart_version = versions[6] file_patterns = [ "*/examples/*/*.y*ml", @@ -41,7 +40,7 @@ # This was happening because strings like 127.0.0.1 match for 7.0.0 # "7.0.0-alpha1" is also used in elasticsearch upgrade test and so shouldn't # been bump -blacklist = re.compile(r".*127.0.0.1.*|.*7.0.0-alpha1.*") +blacklist = re.compile(r".*127.0.0.1.*") print("Updating versions...") @@ -62,8 +61,8 @@ for f in glob.glob(pattern): print(f) for line in fileinput.input([f], inplace=True): - # If we have a version with a build id, like 7.6.2-abcdabcd, - # strip off the latter part and only use the 7.6.2 in the goss + # If we have a version with a build id, like 6.8.11-abcdabcd, + # strip off the latter part and only use the 6.8.11 in the goss # tests version_without_build_id = re.sub(r"-.*", "", version) if re.match(blacklist, line): diff --git a/helpers/common.mk b/helpers/common.mk index d98b0c61f..f7debead3 100644 --- a/helpers/common.mk +++ b/helpers/common.mk @@ -13,18 +13,10 @@ build: ## Build helm-tester docker image .PHONY: deps deps: ## Update helm charts dependencies - sed --in-place '/charts\//d' ./.helmignore helm dependency update -.PHONY: helm -helm: ## Deploy helm on k8s cluster - kubectl get cs - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default || true - helm init --wait --upgrade - .PHONY: lint lint: ## Lint helm templates - grep 'charts/' ./.helmignore || echo 'charts/' >> ./.helmignore helm lint --strict ./ .PHONY: lint-python @@ -44,4 +36,4 @@ test: build ## Run all tests in a docker container docker run --rm -i --user "$$(id -u):$$(id -g)" -v $$(pwd)/../:/app -w /app/$$(basename $$(pwd)) helm-tester make test-all .PHONY: test-all ## Run all tests -test-all: lint deps template pytest +test-all: deps lint template pytest diff --git a/helpers/examples.mk b/helpers/examples.mk index 26bb0b2a7..46c4e000b 100644 --- a/helpers/examples.mk +++ b/helpers/examples.mk @@ -1,7 +1,7 @@ GOSS_VERSION := v0.3.6 GOSS_FILE ?= goss.yaml GOSS_SELECTOR ?= release=$(RELEASE) -STACK_VERSION := 7.7.0-SNAPSHOT +STACK_VERSION := 6.8.14-SNAPSHOT .PHONY: help help: ## Display this help @@ -13,4 +13,3 @@ goss: ## Run goss tests echo Testing with pod: $$GOSS_CONTAINER && \ kubectl cp test/$(GOSS_FILE) $$GOSS_CONTAINER:/tmp/$(GOSS_FILE) && \ kubectl exec $$GOSS_CONTAINER -- sh -c "cd /tmp/ && curl -s -L https://github.com/aelsabbahy/goss/releases/download/$(GOSS_VERSION)/goss-linux-amd64 -o goss && chmod +rx ./goss && ./goss --gossfile $(GOSS_FILE) validate --retry-timeout 300s --sleep 5s --color --format documentation" - diff --git a/helpers/helm-tester/Dockerfile b/helpers/helm-tester/Dockerfile index 66d7dab91..96ad4c483 100644 --- a/helpers/helm-tester/Dockerfile +++ b/helpers/helm-tester/Dockerfile @@ -1,13 +1,13 @@ FROM python:3.7 -ENV HELM_VERSION=2.16.6 +ENV HELM_VERSION=3.4.1 -RUN wget --no-verbose https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ +RUN wget --no-verbose https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf linux-amd64 && \ - HOME=/ helm init --client-only && \ - chmod 777 -R /.helm + mkdir --parents --mode=777 /.config/helm && \ + HOME=/ helm repo add stable https://charts.helm.sh/stable && \ + rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 COPY requirements.txt /usr/src/app/ RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt diff --git a/helpers/helpers.py b/helpers/helpers.py index 19349b1f9..4cc96756e 100644 --- a/helpers/helpers.py +++ b/helpers/helpers.py @@ -9,7 +9,7 @@ def helm_template(config): with tempfile.NamedTemporaryFile() as temp: with open(temp.name, "w") as values: values.write(config) - helm_cmd = "helm template -f {0} ./".format(temp.name) + helm_cmd = "helm template release-name -f {0} ./".format(temp.name) result = yaml.load_all(check_output(helm_cmd.split())) results = {} diff --git a/helpers/matrix.yml b/helpers/matrix.yml index 3a5871aad..7dc71f9d3 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -4,13 +4,13 @@ CHART: - filebeat - metricbeat - logstash + - apm-server ES_SUITE: - default - config - multi - oss - security - - upgrade KIBANA_SUITE: - default - oss @@ -27,10 +27,12 @@ LOGSTASH_SUITE: - default - oss - elasticsearch + - security APM_SERVER_SUITE: - default - oss - security KUBERNETES_VERSION: - - '1.14' - '1.15' + - '1.16' + - '1.17' diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile index 621f9f40e..3540dfb60 100644 --- a/helpers/terraform/Dockerfile +++ b/helpers/terraform/Dockerfile @@ -2,9 +2,10 @@ FROM centos:7 ENV VAULT_VERSION 0.9.3 ENV TERRAFORM_VERSION=0.11.7 -ENV KUBECTL_VERSION=1.15.4 -ENV HELM_VERSION=2.16.6 +ENV KUBECTL_VERSION=1.16.10 +ENV HELM_VERSION=3.4.1 ENV DOCKER_VERSION=18.09.7 +ENV JQ_VERSION=1.6 RUN yum -y install \ make \ @@ -21,6 +22,7 @@ RUN yum -y install \ RUN curl -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \ unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \ chmod +x /usr/local/bin/vault && \ + rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \ vault version RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ @@ -33,14 +35,19 @@ RUN curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL chmod a+x /usr/local/bin/kubectl && \ kubectl version --client -RUN curl -O https://storage.googleapis.com/kubernetes-helm/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ +RUN curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \ mv linux-amd64/helm /usr/local/bin/ && \ - rm -rf linux-amd64 && \ + rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \ helm version --client RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \ tar xfv docker* && \ mv docker/docker /usr/local/bin && \ - rm -rf docker/ && \ - docker + rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \ + docker -v + +RUN curl -O -L https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 && \ + mv jq-linux64 /usr/local/bin/jq && \ + chmod a+x /usr/local/bin/jq && \ + jq --version diff --git a/helpers/terraform/Makefile b/helpers/terraform/Makefile index a6dcff24b..8d448c3e3 100644 --- a/helpers/terraform/Makefile +++ b/helpers/terraform/Makefile @@ -65,8 +65,8 @@ creds: credentials.json ## Get gke credentials kubectl create namespace $(NAMESPACE) || true kubectl config set-context $$(kubectl config current-context) --namespace=$(NAMESPACE) -.PHONY: k8s -k8s: apply creds ## Configure gke cluster +.PHONY: up +up: apply creds ## Configure gke cluster kubectl get cs .PHONY: k8s-staging-registry @@ -77,15 +77,10 @@ k8s-staging-registry: creds ## Create the staging registry auth secret in k8s --docker-username="devops-ci" \ --docker-password="$$DOCKER_PASSWORD" -.PHONY: up -up: k8s ## Install helm on gke cluster - kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default || true - for i in 1 2 3 4 5; do helm init --wait --upgrade && break || sleep 5; done - .PHONY: integration integration: creds ## Deploy helm chart and run integration tests cd ../../$(CHART)/ && \ - helm init --client-only && \ + helm repo add stable https://charts.helm.sh/stable && \ helm dependency update && \ cd ./examples/$(SUITE) && \ make @@ -98,4 +93,3 @@ build: ## Build helm-charts docker image pull-private-images: ## Pull private images used in testing cd ../../elasticsearch/examples/security/ && \ make pull-elasticsearch-image - diff --git a/helpers/terraform/main.tf b/helpers/terraform/main.tf index 292c881fa..0d1ece129 100644 --- a/helpers/terraform/main.tf +++ b/helpers/terraform/main.tf @@ -16,6 +16,8 @@ resource "google_container_cluster" "cluster" { additional_zones = "${var.additional_zones}" min_master_version = "${var.kubernetes_version}" node_version = "${var.kubernetes_version}" + logging_service = "none" + monitoring_service = "none" network = "${var.network}" subnetwork = "${var.subnetwork}" diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml index ee49723c3..0259124aa 100755 --- a/kibana/Chart.yaml +++ b/kibana/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: kibana -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/kibana icon: https://helm.elastic.co/icons/kibana.png diff --git a/kibana/README.md b/kibana/README.md index 8eaca9a39..8fc5c232e 100644 --- a/kibana/README.md +++ b/kibana/README.md @@ -1,12 +1,19 @@ # Kibana Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[Kibana Docker image][]. + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -24,45 +31,35 @@ -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[Kibana Docker image][]. - - ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 (see [parent README][] for more details) +* [Helm][] >=2.8.0 and <3.0.0 * Kubernetes >=1.9 +See [supported configurations][] for more details. ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name kibana elastic/kibana --version=7.7.0` +* Install it: + - with Helm 3: `helm install kibana --version elastic/kibana` + - with Helm 2 (deprecated): `helm install --name kibana --version elastic/kibana` -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name kibana ./helm-charts/kibana` +* Install it: + - with Helm 3: `helm install kibana ./helm-charts/kibana --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name kibana ./helm-charts/kibana --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -95,7 +92,7 @@ as a reference. They are also used in the automated testing of this chart. | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service | `5601` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][]value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Kibana Docker image tag | `7.7.0-SNAPSHOT` | +| `imageTag` | The Kibana Docker image tag | `6.8.14-SNAPSHOT` | | `image` | The Kibana Docker image | `docker.elastic.co/kibana/kibana` | | `ingress` | Configurable [ingress][] to expose the Kibana service. | see [values.yaml][] | | `kibanaConfig` | Allows you to add any config files in `/usr/share/kibana/config/` such as `kibana.yml` See [values.yaml][] for an example of the formatting | `{}` | @@ -149,7 +146,7 @@ An example can be found in [examples/security][]. Deploying OSS version of Elasticsearch can be done by setting `image` value to [kibana OSS Docker image][] -An example of APM Server deployment using OSS version can be found in +An example of Kibana deployment using OSS version can be found in [examples/oss][]. ### How to install plugins? @@ -206,39 +203,41 @@ lifecycle: Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/kibana/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default +[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#default [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/oss -[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security +[examples]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/security +[gke]: https://cloud.google.com/kubernetes-engine [helm]: https://helm.sh [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret [ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ -[kibana docker image]: https://www.elastic.co/guide/en/kibana/7.7/docker.html -[kibana oss docker image]: https://www.docker.elastic.co/#kibana-7-7-0-oss +[kibana docker image]: https://www.elastic.co/guide/en/kibana/6.8/docker.html +[kibana oss docker image]: https://www.docker.elastic.co/r/kibana/kibana-oss [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ [lifecycle hooks]: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector -[openshift]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/openshift -[parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md +[openshift]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/openshift [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ -[security enabled elasticsearch cluster]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#security +[security enabled elasticsearch cluster]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#security [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod -[server.host]: https://www.elastic.co/guide/en/kibana/7.7/settings.html +[server.host]: https://www.elastic.co/guide/en/kibana/6.8/settings.html [service]: https://kubernetes.io/docs/concepts/services-networking/service/ [serviceAccount]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ -[standard upgrade]: https://www.elastic.co/guide/en/kibana/7.7/upgrade-standard.html +[standard upgrade]: https://www.elastic.co/guide/en/kibana/6.8/upgrade-standard.html +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/kibana/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/kibana/values.yaml diff --git a/kibana/examples/default/Makefile b/kibana/examples/default/Makefile index 39caa3072..4c6a6e6d8 100644 --- a/kibana/examples/default/Makefile +++ b/kibana/examples/default/Makefile @@ -5,9 +5,9 @@ RELEASE := helm-kibana-default install: echo "Goss container: $(GOSS_CONTAINER)" - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/default/README.md b/kibana/examples/default/README.md index b06e0188e..eb4fdf9e4 100644 --- a/kibana/examples/default/README.md +++ b/kibana/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy Kibana 7.7.0-SNAPSHOT using [default values][]. +This example deploy Kibana 6.8.14-SNAPSHOT using [default values][]. ## Usage @@ -22,6 +22,6 @@ This example deploy Kibana 7.7.0-SNAPSHOT using [default values][]. You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/kibana/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/kibana/values.yaml diff --git a/kibana/examples/default/test/goss.yaml b/kibana/examples/default/test/goss.yaml index 0fe62169f..f32643bfa 100644 --- a/kibana/examples/default/test/goss.yaml +++ b/kibana/examples/default/test/goss.yaml @@ -3,7 +3,7 @@ http: status: 200 timeout: 2000 body: - - '"number":"7.7.0"' + - '"number":"6.8.14"' http://localhost:5601/app/kibana: status: 200 diff --git a/kibana/examples/openshift/Makefile b/kibana/examples/openshift/Makefile index 9dccc65ed..8435ba2fe 100644 --- a/kibana/examples/openshift/Makefile +++ b/kibana/examples/openshift/Makefile @@ -6,10 +6,10 @@ RELEASE := kibana template: helm template --values ./values.yml ../../ -install: - helm upgrade --wait --timeout=600 --install --values ./values.yml $(RELEASE) ../../ +install: + helm upgrade --wait --timeout=900s --install --values ./values.yml $(RELEASE) ../../ test: install goss - + purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/openshift/README.md b/kibana/examples/openshift/README.md index d8eefbb54..5ee0e9309 100644 --- a/kibana/examples/openshift/README.md +++ b/kibana/examples/openshift/README.md @@ -1,6 +1,6 @@ # OpenShift -This example deploy Kibana 7.7.0-SNAPSHOT on [OpenShift][] using [custom values][]. +This example deploy Kibana 6.8.14-SNAPSHOT on [OpenShift][] using [custom values][]. ## Usage @@ -20,7 +20,7 @@ This example deploy Kibana 7.7.0-SNAPSHOT on [OpenShift][] using [custom values] You can also run [goss integration tests][] using `make test` -[custom values]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/values.yaml -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/openshift/test/goss.yaml +[custom values]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/openshift/test/goss.yaml [openshift]: https://www.openshift.com/ diff --git a/kibana/examples/oss/Makefile b/kibana/examples/oss/Makefile index 77c6412db..cbda8764d 100644 --- a/kibana/examples/oss/Makefile +++ b/kibana/examples/oss/Makefile @@ -4,9 +4,9 @@ include ../../../helpers/examples.mk RELEASE := helm-kibana-oss install: - helm upgrade --wait --timeout=600 --install --values ./values.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install --values ./values.yml $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/kibana/examples/oss/README.md b/kibana/examples/oss/README.md index a4d5896ff..54c9974fc 100644 --- a/kibana/examples/oss/README.md +++ b/kibana/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy Kibana 7.7.0-SNAPSHOT using [Kibana OSS][] version. +This example deploy Kibana 6.8.14-SNAPSHOT using [Kibana OSS][] version. ## Usage @@ -23,5 +23,5 @@ You can also run [goss integration tests][] using `make test` [kibana oss]: https://www.elastic.co/downloads/kibana-oss -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/oss/test/goss.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/oss/test/goss.yaml diff --git a/kibana/examples/security/Makefile b/kibana/examples/security/Makefile index d3a365fd7..2fa8607e6 100644 --- a/kibana/examples/security/Makefile +++ b/kibana/examples/security/Makefile @@ -4,13 +4,13 @@ include ../../../helpers/examples.mk RELEASE := helm-kibana-security install: - helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install --values ./security.yml $(RELEASE) ../../ test: secrets install goss purge: kubectl delete secret kibana || true - helm del --purge $(RELEASE) + helm del $(RELEASE) secrets: encryptionkey=$$(docker run --rm busybox:1.31.1 /bin/sh -c "< /dev/urandom tr -dc _A-Za-z0-9 | head -c50") && \ diff --git a/kibana/examples/security/README.md b/kibana/examples/security/README.md index e54c091e6..ff34e9b1b 100644 --- a/kibana/examples/security/README.md +++ b/kibana/examples/security/README.md @@ -1,6 +1,6 @@ # Security -This example deploy Kibana 7.7.0-SNAPSHOT using authentication and TLS to connect to +This example deploy Kibana 6.8.14-SNAPSHOT using authentication and TLS to connect to Elasticsearch (see [values][]). @@ -23,6 +23,6 @@ Elasticsearch (see [values][]). You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/kibana/examples/security/security.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/kibana/examples/security/security.yml diff --git a/kibana/templates/_helpers.tpl b/kibana/templates/_helpers.tpl index 2fe259ebd..d03cc29cf 100755 --- a/kibana/templates/_helpers.tpl +++ b/kibana/templates/_helpers.tpl @@ -23,9 +23,21 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Return the appropriate apiVersion for ingress. */}} {{- define "kibana.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.Version -}} {{- print "extensions/v1beta1" -}} {{- else -}} {{- print "networking.k8s.io/v1beta1" -}} {{- end -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "kibana.labels" -}} +app: {{ .Chart.Name }} +release: {{ .Release.Name | quote }} +heritage: {{ .Release.Service }} +{{- if .Values.labels }} +{{ toYaml .Values.labels }} +{{- end }} +{{- end -}} diff --git a/kibana/templates/configmap.yaml b/kibana/templates/configmap.yaml index 88927597a..98977a840 100644 --- a/kibana/templates/configmap.yaml +++ b/kibana/templates/configmap.yaml @@ -4,12 +4,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ template "kibana.fullname" . }}-config - labels: - app: {{ .Chart.Name }} - release: {{ .Release.Name | quote }} + labels: {{ include "kibana.labels" . | nindent 4 }} data: {{- range $path, $config := .Values.kibanaConfig }} {{ $path }}: | -{{ $config | indent 4 -}} +{{ tpl $config $ | indent 4 -}} {{- end -}} {{- end -}} diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml index aca2ee153..44920abb9 100644 --- a/kibana/templates/deployment.yaml +++ b/kibana/templates/deployment.yaml @@ -2,12 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ template "kibana.fullname" . }} - labels: - app: {{ .Chart.Name }} - release: {{ .Release.Name | quote }} - {{- range $key, $value := .Values.labels }} - {{ $key }}: {{ $value | quote }} - {{- end }} + labels: {{ include "kibana.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicas }} strategy: @@ -69,8 +64,15 @@ spec: {{ toYaml .Values.imagePullSecrets | indent 8 }} {{- end }} {{- if .Values.extraInitContainers }} + # Currently some extra blocks accept strings + # to continue with backwards compatibility this is being kept + # whilst also allowing for yaml to be specified too. initContainers: + {{- if eq "string" (printf "%T" .Values.extraInitContainers) }} {{ tpl .Values.extraInitContainers . | indent 6 }} + {{- else }} +{{ toYaml .Values.extraInitContainers | indent 6 }} + {{- end }} {{- end }} containers: - name: kibana @@ -103,6 +105,11 @@ spec: - -c - | #!/usr/bin/env bash -e + + # Disable nss cache to avoid filling dentry cache when calling curl + # This is required with Kibana Docker using nss < 3.52 + export NSS_SDB_USE_CACHE=no + http () { local path="${1}" set -- -XGET -s --fail -L @@ -143,5 +150,12 @@ spec: subPath: {{ $path }} {{- end -}} {{- if .Values.extraContainers }} + # Currently some extra blocks accept strings + # to continue with backwards compatibility this is being kept + # whilst also allowing for yaml to be specified too. + {{- if eq "string" (printf "%T" .Values.extraContainers) }} {{ tpl .Values.extraContainers . | indent 6 }} - {{- end }} \ No newline at end of file + {{- else }} +{{ toYaml .Values.extraContainers | indent 6 }} + {{- end }} + {{- end }} diff --git a/kibana/templates/ingress.yaml b/kibana/templates/ingress.yaml index e9aafcb65..e62977495 100644 --- a/kibana/templates/ingress.yaml +++ b/kibana/templates/ingress.yaml @@ -6,10 +6,7 @@ apiVersion: {{ template "kibana.ingress.apiVersion" . }} kind: Ingress metadata: name: {{ $fullName }} - labels: - app: {{ .Chart.Name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} + labels: {{ include "kibana.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} annotations: {{ toYaml . | indent 4 }} diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml index 5734580bf..dfb254c0c 100644 --- a/kibana/templates/service.yaml +++ b/kibana/templates/service.yaml @@ -3,10 +3,7 @@ apiVersion: v1 kind: Service metadata: name: {{ template "kibana.fullname" . }} - labels: - app: {{ .Chart.Name }} - release: {{ .Release.Name | quote }} - heritage: {{ .Release.Service }} + labels: {{ include "kibana.labels" . | nindent 4 }} {{- if .Values.service.labels }} {{ toYaml .Values.service.labels | indent 4}} {{- end }} @@ -16,6 +13,9 @@ metadata: {{- end }} spec: type: {{ .Values.service.type }} +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} +{{- end }} {{- with .Values.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{ toYaml . | indent 4 }} diff --git a/kibana/tests/kibana_test.py b/kibana/tests/kibana_test.py index 1341c6146..38a9fb154 100644 --- a/kibana/tests/kibana_test.py +++ b/kibana/tests/kibana_test.py @@ -51,6 +51,9 @@ def test_defaults(): # Make sure that the default 'loadBalancerSourceRanges' list is empty assert "loadBalancerSourceRanges" not in r["service"][name]["spec"] + # Make sure that the default 'loadBalancerIP' string is empty + assert "loadBalancerIP" not in r["service"][name]["spec"] + def test_overriding_the_elasticsearch_hosts(): config = """ @@ -615,3 +618,14 @@ def test_setting_fullnameOverride(): ] == "kibana" ) + + +def test_adding_loadBalancerIP(): + config = """ + service: + loadBalancerIP: 12.5.11.79 + """ + + r = helm_template(config) + + assert r["service"][name]["spec"]["loadBalancerIP"] == "12.5.11.79" diff --git a/kibana/values.yaml b/kibana/values.yaml index 8af17fbc7..53ba34e8b 100755 --- a/kibana/values.yaml +++ b/kibana/values.yaml @@ -29,7 +29,7 @@ secretMounts: [] # subPath: kibana.keystore # optional image: "docker.elastic.co/kibana/kibana" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" # additionals labels @@ -95,6 +95,7 @@ updateStrategy: service: type: ClusterIP + loadBalancerIP: "" port: 5601 nodePort: "" labels: {} diff --git a/logstash/Chart.yaml b/logstash/Chart.yaml index a2466fd6f..ed8c4a4c9 100755 --- a/logstash/Chart.yaml +++ b/logstash/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: logstash -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/logstash icon: https://helm.elastic.co/icons/logstash.png diff --git a/logstash/README.md b/logstash/README.md index d2f6686fb..2947358af 100644 --- a/logstash/README.md +++ b/logstash/README.md @@ -1,12 +1,24 @@ # Logstash Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[Logstash Docker image][]. + +**Warning**: This functionality is in beta and is subject to change. +The design and code is less mature than official GA features and is being +provided as-is with no warranties. Alpha features are not subject to the support +SLA of official GA features (see [supported configurations][] for more details). + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -20,45 +32,36 @@ -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[Logstash Docker image][]. - - ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 (see [parent README][] for more details) +* [Helm][] >=2.8.0 and <3.0.0 * Kubernetes >=1.8 +See [supported configurations][] for more details. ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name logstash elastic/logstash --version=7.7.0` +* Install it: + - with Helm 3: `helm install logstash --version elastic/logstash` + - with Helm 2 (deprecated): `helm install --name logstash --version elastic/logstash` + -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name logstash ./helm-charts/logstash` +* Install it: + - with Helm 3: `helm install logstash ./helm-charts/logstash --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name logstash ./helm-charts/logstash --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -93,6 +96,8 @@ in this [note][]). to make default probes work. If restricting HTTP API to 127.0.0.1 is required by using `http.host: 127.0.0.1`, default probes should be disabled or overrided (see [values.yaml][] for the good syntax). +* An ingress is provided that can be used to expose the HTTP port. This can be +useful for the [http input plugin][], for instance. ## Configuration @@ -112,9 +117,10 @@ using `http.host: 127.0.0.1`, default probes should be disabled or overrided | `httpPort` | The http port that Kubernetes will use for the healthchecks and the service | `9600` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Logstash Docker image tag | `7.7.0-SNAPSHOT` | +| `imageTag` | The Logstash Docker image tag | `6.8.14-SNAPSHOT` | | `image` | The Logstash Docker image | `docker.elastic.co/logstash/logstash` | | `labels` | Configurable [labels][] applied to all Logstash pods | `{}` | +| `ingress` | Configurable [ingress][] for external access to Logstash HTTP port. | see [values.yaml][] | | `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml][] for an example of the formatting | `{}` | | `livenessProbe` | Configuration fields for the liveness [probe][] | see [values.yaml][] | | `logstashConfig` | Allows you to add any config files in `/usr/share/logstash/config/` such as `logstash.yml` and `log4j2.properties` See [values.yaml][] for an example of the formatting | `{}` | @@ -135,6 +141,7 @@ using `http.host: 127.0.0.1`, default probes should be disabled or overrided | `replicas` | Kubernetes replica count for the StatefulSet (i.e. how many pods) | `1` | | `resources` | Allows you to set the [resources][] for the StatefulSet | see [values.yaml][] | | `schedulerName` | Name of the [alternate scheduler][] | `""` | +| `secrets` | Allows you easily create a secret from as variables or file. For add secrets from file, add suffix `.filepath` to the key of secret key. The value will be encoded to base64. Useful for store certificates and other secrets. | See [values.yaml][] | | `secretMounts` | Allows you easily mount a secret as a file inside the StatefulSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | | `securityContext` | Allows you to set the [securityContext][] for the container | see [values.yaml][] | | `service` | Configurable [service][] to expose the Logstash service. | see [values.yaml][] | @@ -151,7 +158,7 @@ using `http.host: 127.0.0.1`, default probes should be disabled or overrided Deploying OSS version of Elasticsearch can be done by setting `image` value to [Logstash OSS Docker image][] -An example of APM Server deployment using OSS version can be found in +An example of Logstash deployment using OSS version can be found in [examples/oss][]. ### How to install plugins? @@ -184,7 +191,8 @@ against best practices of containers and immutable infrastructure. Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/logstash/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md @@ -192,30 +200,32 @@ about our development and testing process. [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ [anti-affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [deploys statefulsets serially]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies -[custom docker image]: https://www.elastic.co/guide/en/logstash/7.7/docker-config.html#_custom_images +[custom docker image]: https://www.elastic.co/guide/en/logstash/6.8/docker-config.html#_custom_images [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/oss +[examples]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples/oss [helm]: https://helm.sh +[http input plugin]: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-http.html [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret +[ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -[logstash docker image]: https://www.elastic.co/guide/en/logstash/7.7/docker.html -[logstash oss docker image]: https://www.docker.elastic.co/#logstash-7-7-0-oss +[logstash docker image]: https://www.elastic.co/guide/en/logstash/6.8/docker.html +[logstash oss docker image]: https://www.docker.elastic.co/r/logstash/logstash-oss [maxUnavailable]: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget [node affinity settings]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector -[note]: https://www.elastic.co/guide/en/logstash/7.7/docker-config.html#docker-env-config -[parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md +[note]: https://www.elastic.co/guide/en/logstash/6.8/docker-config.html#docker-env-config [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ [updateStrategy]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/ [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod [service]: https://kubernetes.io/docs/concepts/services-networking/service/ +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations [terminationGracePeriod]: https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/logstash/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/logstash/values.yaml [volumeClaimTemplate for statefulsets]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage diff --git a/logstash/examples/default/Makefile b/logstash/examples/default/Makefile index 03d9592f1..4c5920df5 100644 --- a/logstash/examples/default/Makefile +++ b/logstash/examples/default/Makefile @@ -5,12 +5,12 @@ include ../../../helpers/examples.mk RELEASE := helm-logstash-default install: - helm upgrade --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=1200s --install $(RELEASE) ../../ restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --set terminationGracePeriod=121 --wait --timeout=1200s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/logstash/examples/default/README.md b/logstash/examples/default/README.md index ba2fa62cf..a62deb3fd 100644 --- a/logstash/examples/default/README.md +++ b/logstash/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy Logstash 7.7.0-SNAPSHOT using [default values][]. +This example deploy Logstash 6.8.14-SNAPSHOT using [default values][]. ## Usage @@ -13,5 +13,5 @@ This example deploy Logstash 7.7.0-SNAPSHOT using [default values][]. You can also run [goss integration tests][] using `make test` -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/logstash/values.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/logstash/values.yaml diff --git a/logstash/examples/default/test/goss.yaml b/logstash/examples/default/test/goss.yaml index 4a883cf36..d06e6e34d 100644 --- a/logstash/examples/default/test/goss.yaml +++ b/logstash/examples/default/test/goss.yaml @@ -9,14 +9,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-default-logstash-0"' - - '"version" : "7.7.0"' + - '"version" : "6.8.14"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-default-logstash-0"' - - '"status" : "green"' - - '"workers" : 1' - - '"batch_size" : 125' - - '"batch_delay" : 50' file: /usr/share/logstash/config/logstash.yml: diff --git a/logstash/examples/elasticsearch/Makefile b/logstash/examples/elasticsearch/Makefile index f78c42f4e..b11f714e3 100644 --- a/logstash/examples/elasticsearch/Makefile +++ b/logstash/examples/elasticsearch/Makefile @@ -5,13 +5,13 @@ include ../../../helpers/examples.mk RELEASE := helm-logstash-elasticsearch install: - helm upgrade --wait --timeout=900 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=1200s --install $(RELEASE) --values ./values.yaml ../../ restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --set terminationGracePeriod=121 --wait --timeout=1200s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) kubectl delete $$(kubectl get pvc -l release=$(RELEASE) -o name) diff --git a/logstash/examples/elasticsearch/README.md b/logstash/examples/elasticsearch/README.md index 58d9850e9..aad50192d 100644 --- a/logstash/examples/elasticsearch/README.md +++ b/logstash/examples/elasticsearch/README.md @@ -1,6 +1,6 @@ # Elasticsearch -This example deploy Logstash 7.7.0-SNAPSHOT which connects to Elasticsearch (see +This example deploy Logstash 6.8.14-SNAPSHOT which connects to Elasticsearch (see [values][]). @@ -23,6 +23,6 @@ This example deploy Logstash 7.7.0-SNAPSHOT which connects to Elasticsearch (see You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/elasticsearch/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/elasticsearch/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples/elasticsearch/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples/elasticsearch/values.yaml diff --git a/logstash/examples/elasticsearch/test/goss.yaml b/logstash/examples/elasticsearch/test/goss.yaml index 4e850758c..09f22bb34 100644 --- a/logstash/examples/elasticsearch/test/goss.yaml +++ b/logstash/examples/elasticsearch/test/goss.yaml @@ -21,14 +21,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-elasticsearch-logstash-0"' - - '"version" : "7.7.0"' + - '"version" : "6.8.14"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-elasticsearch-logstash-0"' - - '"status" : "green"' - - '"workers" : 1' - - '"batch_size" : 125' - - '"batch_delay" : 50' http://elasticsearch-master:9200/_cat/indices: status: 200 timeout: 2000 diff --git a/logstash/examples/oss/Makefile b/logstash/examples/oss/Makefile index 856234431..81c83f702 100644 --- a/logstash/examples/oss/Makefile +++ b/logstash/examples/oss/Makefile @@ -5,12 +5,12 @@ include ../../../helpers/examples.mk RELEASE := helm-logstash-oss install: - helm upgrade --wait --timeout=900 --install $(RELEASE) --values ./values.yaml ../../ + helm upgrade --wait --timeout=1200s --install $(RELEASE) --values ./values.yaml ../../ restart: - helm upgrade --set terminationGracePeriod=121 --wait --timeout=900 --install $(RELEASE) ../../ + helm upgrade --set terminationGracePeriod=121 --wait --timeout=1200s --install $(RELEASE) ../../ test: install goss purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/logstash/examples/oss/README.md b/logstash/examples/oss/README.md index 8350847e6..d0af5a2bd 100644 --- a/logstash/examples/oss/README.md +++ b/logstash/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy Logstash 7.7.0-SNAPSHOT using [Logstash OSS][] version. +This example deploy Logstash 6.8.14-SNAPSHOT using [Logstash OSS][] version. ## Usage @@ -14,4 +14,4 @@ You can also run [goss integration tests][] using `make test` [logstash oss]: https://www.elastic.co/downloads/logstash-oss -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/logstash/examples/oss/test/goss.yaml +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/logstash/examples/oss/test/goss.yaml diff --git a/logstash/examples/oss/test/goss.yaml b/logstash/examples/oss/test/goss.yaml index e1d11d999..42cbf538c 100644 --- a/logstash/examples/oss/test/goss.yaml +++ b/logstash/examples/oss/test/goss.yaml @@ -9,14 +9,8 @@ http: status: 200 timeout: 2000 body: - - '"host" : "helm-logstash-oss-logstash-0"' - - '"version" : "7.7.0"' + - '"version" : "6.8.14"' - '"http_address" : "0.0.0.0:9600"' - - '"name" : "helm-logstash-oss-logstash-0"' - - '"status" : "green"' - - '"workers" : 1' - - '"batch_size" : 125' - - '"batch_delay" : 50' file: /usr/share/logstash/config/logstash.yml: diff --git a/logstash/examples/security/Makefile b/logstash/examples/security/Makefile new file mode 100644 index 000000000..ef8ebf13e --- /dev/null +++ b/logstash/examples/security/Makefile @@ -0,0 +1,14 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-logstash-security + +install: + helm upgrade --wait --timeout=1200s --install $(RELEASE) --values values.yaml ../../ + +test: install goss + +purge: + helm del $(RELEASE) + kubectl delete $$(kubectl get pvc -l release=$(RELEASE) -o name) diff --git a/logstash/examples/security/README.md b/logstash/examples/security/README.md new file mode 100644 index 000000000..0f9af83dd --- /dev/null +++ b/logstash/examples/security/README.md @@ -0,0 +1,28 @@ +# Security + +This example deploy Logstash 7.7.1 which connects to Elasticsearch using TLS +(see [values][]). + + +## Usage + +* Deploy [Elasticsearch Helm chart with security][]. + +* Deploy Logstash chart: `make install` + +* You can now setup a port forward to query Logstash indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart with security]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/master/logstash/examples/security/values.yaml diff --git a/logstash/examples/security/test/goss.yaml b/logstash/examples/security/test/goss.yaml new file mode 100644 index 000000000..5cd5dffeb --- /dev/null +++ b/logstash/examples/security/test/goss.yaml @@ -0,0 +1,58 @@ +mount: + /usr/share/logstash/data: + exists: true + /usr/share/logstash/config/logstash.yml: + exists: true + opts: + - ro + /usr/share/logstash/pipeline/uptime.conf: + exists: true + opts: + - ro + +user: + logstash: + exists: true + uid: 1000 + gid: 1000 + +http: + http://localhost:9600?pretty: + status: 200 + timeout: 2000 + body: + - '"version" : "6.8.14"' + - '"http_address" : "0.0.0.0:9600"' + https://security-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'logstash' + allow-insecure: true + username: '{{ .Env.ELASTICSEARCH_USERNAME }}' + password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' + +file: + /usr/share/logstash/config/logstash.yml: + exists: true + mode: "0644" + owner: root + group: logstash + filetype: file + contains: + - 'http.host: 0.0.0.0' + - 'xpack.monitoring.enabled: true' + - 'xpack.monitoring.elasticsearch.hosts: ["https://security-master:9200"]' + - 'xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt' + /usr/share/logstash/pipeline/uptime.conf: + exists: true + mode: "0644" + owner: root + group: logstash + filetype: file + contains: + - 'input { exec { command => "uptime" interval => 30 } }' + - 'output { elasticsearch {' + - 'hosts => ["https://security-master:9200"]' + - 'cacert => "/usr/share/logstash/config/certs/elastic-certificate.crt"' + - 'index => "logstash"' diff --git a/logstash/examples/security/values.yaml b/logstash/examples/security/values.yaml new file mode 100644 index 000000000..1457d7865 --- /dev/null +++ b/logstash/examples/security/values.yaml @@ -0,0 +1,40 @@ +persistence: + enabled: true + +logstashConfig: + logstash.yml: | + http.host: 0.0.0.0 + xpack.monitoring.enabled: true + xpack.monitoring.elasticsearch.username: '${ELASTICSEARCH_USERNAME}' + xpack.monitoring.elasticsearch.password: '${ELASTICSEARCH_PASSWORD}' + xpack.monitoring.elasticsearch.hosts: ["https://security-master:9200"] + xpack.monitoring.elasticsearch.ssl.certificate_authority: /usr/share/logstash/config/certs/elastic-certificate.crt + +logstashPipeline: + uptime.conf: | + input { exec { command => "uptime" interval => 30 } } + output { elasticsearch { + hosts => ["https://security-master:9200"] + cacert => "/usr/share/logstash/config/certs/elastic-certificate.crt" + user => '${ELASTICSEARCH_USERNAME}' + password => '${ELASTICSEARCH_PASSWORD}' + index => "logstash" + } + } + +secretMounts: + - name: elastic-certificate-crt + secretName: elastic-certificate-crt + path: /usr/share/logstash/config/certs + +extraEnvs: + - name: 'ELASTICSEARCH_USERNAME' + valueFrom: + secretKeyRef: + name: elastic-credentials + key: username + - name: 'ELASTICSEARCH_PASSWORD' + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password diff --git a/logstash/templates/_helpers.tpl b/logstash/templates/_helpers.tpl index 162a3ee08..13c76fde6 100755 --- a/logstash/templates/_helpers.tpl +++ b/logstash/templates/_helpers.tpl @@ -23,9 +23,20 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Return the appropriate apiVersion for statefulset. */}} {{- define "logstash.statefulset.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} +{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} {{- print "apps/v1beta2" -}} {{- else -}} {{- print "apps/v1" -}} {{- end -}} {{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "logstash.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.Version -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/logstash/templates/configmap-config.yaml b/logstash/templates/configmap-config.yaml index e69d524d0..638038435 100644 --- a/logstash/templates/configmap-config.yaml +++ b/logstash/templates/configmap-config.yaml @@ -12,6 +12,6 @@ metadata: data: {{- range $path, $config := .Values.logstashConfig }} {{ $path }}: | -{{ $config | indent 4 -}} +{{ tpl $config $ | indent 4 -}} {{- end -}} {{- end -}} diff --git a/logstash/templates/configmap-pipeline.yaml b/logstash/templates/configmap-pipeline.yaml index 5ce180ff7..2a92bd4a1 100644 --- a/logstash/templates/configmap-pipeline.yaml +++ b/logstash/templates/configmap-pipeline.yaml @@ -12,6 +12,6 @@ metadata: data: {{- range $path, $config := .Values.logstashPipeline }} {{ $path }}: | -{{ $config | indent 4 -}} +{{ tpl $config $ | indent 4 -}} {{- end -}} {{- end -}} diff --git a/logstash/templates/ingress.yaml b/logstash/templates/ingress.yaml new file mode 100644 index 000000000..cef5666b9 --- /dev/null +++ b/logstash/templates/ingress.yaml @@ -0,0 +1,33 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "logstash.fullname" . -}} +apiVersion: {{ template "logstash.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ $fullName | quote}} + chart: "{{ .Chart.Name }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} +{{- end }} + rules: + {{- range $.Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ .servicePort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/logstash/templates/secret.yaml b/logstash/templates/secret.yaml new file mode 100644 index 000000000..0abf78650 --- /dev/null +++ b/logstash/templates/secret.yaml @@ -0,0 +1,27 @@ +{{- if .Values.secrets }} +{{- $fullName := include "logstash.fullname" . -}} +{{- range .Values.secrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $fullName .name | quote }} + labels: + app: {{ $fullName | quote }} + chart: {{ $.Chart.Name | quote }} + heritage: {{ $.Release.Service | quote }} + release: {{ $.Release.Name | quote }} + {{- range $key, $value := $.Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} +data: +{{- range $key, $val := .value }} + {{- if hasSuffix "filepath" $key }} + {{ $key | replace ".filepath" "" }}: {{ $.Files.Get $val | b64enc | quote }} + {{ else }} + {{ $key }}: {{ $val | b64enc | quote }} + {{- end }} +{{- end }} +type: Opaque +{{- end }} +{{- end }} \ No newline at end of file diff --git a/logstash/templates/service-headless.yaml b/logstash/templates/service-headless.yaml new file mode 100644 index 000000000..47148dfb0 --- /dev/null +++ b/logstash/templates/service-headless.yaml @@ -0,0 +1,20 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: "{{ template "logstash.fullname" . }}-headless" + labels: + app: "{{ template "logstash.fullname" . }}" + chart: "{{ .Chart.Name }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + app: "{{ template "logstash.fullname" . }}" + ports: + - name: http + port: {{ .Values.httpPort }} diff --git a/logstash/templates/service.yaml b/logstash/templates/service.yaml index 6cbca1ee4..6540c8c61 100644 --- a/logstash/templates/service.yaml +++ b/logstash/templates/service.yaml @@ -16,7 +16,6 @@ spec: selector: app: "{{ template "logstash.fullname" . }}" chart: "{{ .Chart.Name }}" - heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} ports: {{ toYaml .Values.service.ports | indent 4 }} diff --git a/logstash/templates/serviceaccount.yaml b/logstash/templates/serviceaccount.yaml index 8302d1403..4508878b8 100644 --- a/logstash/templates/serviceaccount.yaml +++ b/logstash/templates/serviceaccount.yaml @@ -8,6 +8,10 @@ metadata: {{- else }} name: {{ .Values.rbac.serviceAccountName | quote }} {{- end }} + annotations: + {{- with .Values.rbac.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} labels: app: "{{ template "logstash.fullname" . }}" chart: "{{ .Chart.Name }}" diff --git a/logstash/templates/statefulset.yaml b/logstash/templates/statefulset.yaml index 30dbb062f..9200cfc0c 100644 --- a/logstash/templates/statefulset.yaml +++ b/logstash/templates/statefulset.yaml @@ -12,9 +12,7 @@ metadata: {{ $key }}: {{ $value | quote }} {{- end }} spec: - {{- if .Values.service }} - serviceName: {{ template "logstash.fullname" . }} - {{- end }} + serviceName: {{ template "logstash.fullname" . }}-headless selector: matchLabels: app: "{{ template "logstash.fullname" . }}" @@ -49,14 +47,17 @@ spec: {{- range $key, $value := .Values.podAnnotations }} {{ $key }}: {{ $value | quote }} {{- end }} - {{/* This forces a restart if the configmap has changed */}} + {{- /* This forces a restart if the configmap has changed */}} {{- if .Values.logstashConfig }} configchecksum: {{ include (print .Template.BasePath "/configmap-config.yaml") . | sha256sum | trunc 63 }} {{- end }} - {{/* This forces a restart if the configmap has changed */}} + {{- /* This forces a restart if the configmap has changed */}} {{- if .Values.logstashPipeline }} pipelinechecksum: {{ include (print .Template.BasePath "/configmap-pipeline.yaml") . | sha256sum | trunc 63 }} {{- end }} + {{- if .Values.secrets }} + secretschecksum: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum | trunc 63 }} + {{- end }} spec: {{- if .Values.schedulerName }} schedulerName: "{{ .Values.schedulerName }}" @@ -151,7 +152,7 @@ spec: - name: http containerPort: {{ .Values.httpPort }} {{- if .Values.extraPorts }} - {{- toYaml .Values.extraPorts | nindent 8 }} + {{- toYaml .Values.extraPorts | nindent 8 }} {{- end }} resources: {{ toYaml .Values.resources | indent 10 }} diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py index 55d838c1b..caee32c29 100755 --- a/logstash/tests/logstash_test.py +++ b/logstash/tests/logstash_test.py @@ -1,10 +1,9 @@ +import base64 import os import sys sys.path.insert(1, os.path.join(sys.path[0], "../../helpers")) from helpers import helm_template -import yaml - name = "release-name-logstash" @@ -78,8 +77,9 @@ def test_defaults(): ) # Service - assert "serviceName" not in r["statefulset"][name]["spec"] - assert "service" not in r + assert r["statefulset"][name]["spec"]["serviceName"] == name + "-headless" + assert name + "-headless" in r["service"] + assert r["service"][name + "-headless"]["spec"]["ports"][0]["port"] == 9600 # Other assert r["statefulset"][name]["spec"]["template"]["spec"]["securityContext"] == { @@ -234,7 +234,7 @@ def test_adding_persistence(): assert c["volumeMounts"][0]["mountPath"] == "/usr/share/logstash/data" assert c["volumeMounts"][0]["name"] == name - v = r["statefulset"]["release-name-logstash"]["spec"]["volumeClaimTemplates"][0] + v = r["statefulset"][name]["spec"]["volumeClaimTemplates"][0] assert v["metadata"]["name"] == name assert v["spec"]["accessModes"] == ["ReadWriteOnce"] assert v["spec"]["resources"]["requests"]["storage"] == "1Gi" @@ -306,6 +306,155 @@ def test_adding_a_secret_mount_with_subpath(): } +def test_adding_a_secret(): + content = "LS1CRUdJTiBgUFJJVkFURSB" + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} +""".format( + elk_pass=content + ) + content_b64 = base64.b64encode(content.encode("ascii")).decode("ascii") + + r = helm_template(config) + secret_name = name + "-env" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 1 + assert s["data"] == {"ELASTICSEARCH_PASSWORD": content_b64} + assert ( + "secretschecksum" + in r["statefulset"][name]["spec"]["template"]["metadata"]["annotations"] + ) + + +def test_adding_secret_from_file(): + content = """ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEApCt3ychnqZHsS +DylPFZn55xDaDcWco1oNFdBGzFjw+ +zkuMFMOv7ab+yOFwHeEeAAEkEgy1u +Da1vIscBs1K0kbEFRSqySLuNHWiJp +wK2cI/gJc+S9Qd9Qsn0XGjmjQ6P2p +ot2hvCOtnei998OmDSYORKBq2jiv/ +-----END RSA PRIVATE KEY----- +""" + config = """ +secrets: + - name: "tls" + value: + cert.key.filepath: "secrets/private.key" +""" + content_b64 = base64.b64encode(content.encode("ascii")).decode("ascii") + work_dir = os.path.join(os.path.abspath(os.getcwd()), "secrets") + filename = os.path.join(work_dir, "private.key") + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + f.write(content) + + with open(filename, "r") as f: + data = f.read() + assert data == content + + r = helm_template(config) + secret_name = name + "-tls" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 1 + assert s["data"] == { + "cert.key": content_b64, + } + + os.remove(filename) + os.rmdir(work_dir) + + +def test_adding_multiple_data_secret(): + content = { + "elk_pass": "LS1CRUdJTiBgUFJJVkFURSB", + "api_key": "ui2CsdUadTiBasRJRkl9tvNnw", + } + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} + api_key: {api_key} +""".format( + elk_pass=content["elk_pass"], api_key=content["api_key"] + ) + content_b64 = { + "elk_pass": base64.b64encode(content["elk_pass"].encode("ascii")).decode( + "ascii" + ), + "api_key": base64.b64encode(content["api_key"].encode("ascii")).decode("ascii"), + } + + r = helm_template(config) + secret_name = name + "-env" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 2 + assert s["data"] == { + "ELASTICSEARCH_PASSWORD": content_b64["elk_pass"], + "api_key": content_b64["api_key"], + } + + +def test_adding_multiple_secrets(): + content = { + "elk_pass": "LS1CRUdJTiBgUFJJVkFURSB", + "cert_crt": "LS0tLS1CRUdJTiBlRJRALKJDDQVRFLS0tLS0K", + "cert_key": "LS0tLS1CRUdJTiBgUFJJVkFURSBLRVktLS0tLQo", + } + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} + - name: "tls" + value: + cert.crt: {cert_crt} + cert.key: {cert_key} + +""".format( + elk_pass=content["elk_pass"], + cert_crt=content["cert_crt"], + cert_key=content["cert_key"], + ) + content_b64 = { + "elk_pass": base64.b64encode(content["elk_pass"].encode("ascii")).decode( + "ascii" + ), + "cert_crt": base64.b64encode(content["cert_crt"].encode("ascii")).decode( + "ascii" + ), + "cert_key": base64.b64encode(content["cert_key"].encode("ascii")).decode( + "ascii" + ), + } + + r = helm_template(config) + secret_names = {"env": name + "-env", "tls": name + "-tls"} + s_env = r["secret"][secret_names["env"]] + s_tls = r["secret"][secret_names["tls"]] + assert len(r["secret"]) == 2 + assert len(s_env["data"]) == 1 + assert s_env["data"] == { + "ELASTICSEARCH_PASSWORD": content_b64["elk_pass"], + } + assert len(s_tls["data"]) == 2 + assert s_tls["data"] == { + "cert.crt": content_b64["cert_crt"], + "cert.key": content_b64["cert_key"], + } + + def test_adding_image_pull_secrets(): config = """ imagePullSecrets: @@ -350,6 +499,22 @@ def test_adding_pod_annotations(): ) +def test_adding_serviceaccount_annotations(): + config = """ +rbac: + create: true + serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][name]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) + + def test_adding_a_node_selector(): config = """ nodeSelector: @@ -375,9 +540,9 @@ def test_adding_a_node_affinity(): - myvalue """ r = helm_template(config) - assert r["statefulset"]["release-name-logstash"]["spec"]["template"]["spec"][ - "affinity" - ]["nodeAffinity"] == { + assert r["statefulset"][name]["spec"]["template"]["spec"]["affinity"][ + "nodeAffinity" + ] == { "preferredDuringSchedulingIgnoredDuringExecution": [ { "weight": 100, @@ -415,10 +580,9 @@ def test_adding_in_logstash_config(): s = r["statefulset"][name]["spec"]["template"]["spec"] - assert { - "configMap": {"name": "release-name-logstash-config"}, - "name": "logstashconfig", - } in s["volumes"] + assert {"configMap": {"name": name + "-config"}, "name": "logstashconfig",} in s[ + "volumes" + ] assert { "mountPath": "/usr/share/logstash/config/logstash.yml", "name": "logstashconfig", @@ -711,3 +875,28 @@ def test_setting_fullnameOverride(): ] == "logstash" ) + + +def test_adding_an_ingress(): + config = """ +ingress: + enabled: true + annotations: {} + hosts: + - host: logstash.local + paths: + - path: /logs + servicePort: 8080 +""" + r = helm_template(config) + s = r["ingress"][name] + assert s["metadata"]["name"] == name + assert len(s["spec"]["rules"]) == 1 + assert s["spec"]["rules"][0] == { + "host": "logstash.local", + "http": { + "paths": [ + {"path": "/logs", "backend": {"serviceName": name, "servicePort": 8080}} + ] + }, + } diff --git a/logstash/values.yaml b/logstash/values.yaml index ba65eaf13..d5166588b 100755 --- a/logstash/values.yaml +++ b/logstash/values.yaml @@ -3,7 +3,7 @@ replicas: 1 # Allows you to add any config files in /usr/share/logstash/config/ # such as logstash.yml and log4j2.properties -# +# # Note that when overriding logstash.yml, `http.host: 0.0.0.0` should always be included # to make default probes work. logstashConfig: {} @@ -39,11 +39,28 @@ envFrom: [] # - configMapRef: # name: config-map +# Add sensitive data to k8s secrets +secrets: [] +# - name: "env" +# value: +# ELASTICSEARCH_PASSWORD: "LS1CRUdJTiBgUFJJVkFURSB" +# api_key: ui2CsdUadTiBasRJRkl9tvNnw +# - name: "tls" +# value: +# ca.crt: | +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# cert.crt: "LS0tLS1CRUdJTiBlRJRklDQVRFLS0tLS0K" +# cert.key.filepath: "secrets.crt" # The path to file should be relative to the `values.yaml` file. + + # A list of secrets and their paths to mount inside the pod secretMounts: [] image: "docker.elastic.co/logstash/logstash" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] @@ -70,6 +87,7 @@ volumeClaimTemplate: rbac: create: false + serviceAccountAnnotations: {} serviceAccountName: "" podSecurityPolicy: @@ -163,9 +181,9 @@ securityContext: terminationGracePeriod: 120 # Probes -# Default probes are using `httpGet` which requires that `http.host: 0.0.0.0` is part of +# Default probes are using `httpGet` which requires that `http.host: 0.0.0.0` is part of # `logstash.yml`. If needed probes can be disabled or overrided using the following syntaxes: -# +# # disable livenessProbe # livenessProbe: null # @@ -228,3 +246,13 @@ service: {} # port: 8080 # protocol: TCP # targetPort: 8080 + +ingress: + enabled: false +# annotations: {} +# hosts: +# - host: logstash.local +# paths: +# - path: /logs +# servicePort: 8080 +# tls: [] diff --git a/metricbeat/Chart.yaml b/metricbeat/Chart.yaml index 457bd5f64..7011ae763 100755 --- a/metricbeat/Chart.yaml +++ b/metricbeat/Chart.yaml @@ -5,8 +5,8 @@ maintainers: - email: helm-charts@elastic.co name: Elastic name: metricbeat -version: 7.7.0-SNAPSHOT -appVersion: 7.7.0-SNAPSHOT +version: 6.8.14-SNAPSHOT +appVersion: 6.8.14-SNAPSHOT sources: - https://github.com/elastic/beats icon: https://helm.elastic.co/icons/beats.png diff --git a/metricbeat/README.md b/metricbeat/README.md index bc305a4e1..1015b526f 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -1,12 +1,19 @@ # Metricbeat Helm Chart + +This Helm chart is a lightweight way to configure and run our official +[Metricbeat Docker image][]. + + +**Warning**: This branch is used for development, please use the latest [6.x][] release for released version. + - [Requirements](#requirements) - [Installing](#installing) - - [Using Helm repository](#using-helm-repository) - - [Using the 7.7 branch](#using-the-77-branch) + - [Install released version using Helm repository](#install-released-version-using-helm-repository) + - [Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions](#install-development-version-using-68-branch-and-6813-snapshot-versions) - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) @@ -23,45 +30,35 @@ -This functionality is in beta and is subject to change. The design and code is -less mature than official GA features and is being provided as-is with no -warranties. Beta features are not subject to the support SLA of official GA -features. - -This Helm chart is a lightweight way to configure and run our official -[Metricbeat Docker image][]. - - ## Requirements -* [Helm][] >=2.8.0 and <3.0.0 (see [parent README][] for more details) +* [Helm][] >=2.8.0 and <3.0.0 * Kubernetes >=1.9 +See [supported configurations][] for more details. ## Installing -This chart is tested with the latest 7.7.0-SNAPSHOT versions. +This chart is tested with the latest 6.8.14-SNAPSHOT version. -### Using Helm repository +### Install released version using Helm repository * Add the Elastic Helm charts repo: `helm repo add elastic https://helm.elastic.co` -* Install the latest 7.7 release: -`helm install --name metricbeat elastic/metricbeat --version=7.7.0` +* Install it: + - with Helm 3: `helm install metricbeat --version elastic/metricbeat` + - with Helm 2 (deprecated): `helm install --name metricbeat --version elastic/metricbeat` -### Using the 7.7 branch +### Install development version using 6.8 branch and 6.8.14-SNAPSHOT versions -* Clone the git repo and checkout the right branch: +* Clone the git repo: `git clone git@github.com:elastic/helm-charts.git` - ```shell - git clone git@github.com:elastic/helm-charts.git - cd helm-charts - git checkout -b 7.7 origin/7.7 - ```` +* Checkout the branch : git checkout 6.8 -* Install the latest 7.7.0-SNAPSHOT: -`helm install --name metricbeat ./helm-charts/metricbeat` +* Install it: + - with Helm 3: `helm install metricbeat ./helm-charts/metricbeat --set imageTag=6.8.14-SNAPSHOT` + - with Helm 2 (deprecated): `helm install --name metricbeat ./helm-charts/metricbeat --set imageTag=6.8.14-SNAPSHOT` ## Upgrading @@ -94,7 +91,10 @@ as a reference. They are also used in the automated testing of this chart. | Parameter | Description | Default | |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------| | `clusterRoleRules` | Configurable [cluster role rules][] that Metricbeat uses to access Kubernetes resources | see [values.yaml][] | +| `daemonset.annotations` | Configurable [annotations][] for Metricbeat daemonset | `{}` | +| `daemonset.labels` | Configurable [labels][] applied to all Metricbeat DaemonSet pods | `{}` | | `daemonset.affinity` | Configurable [affinity][] for Metricbeat daemonset | `{}` | +| `daemonset.enabled` | If true, enable daemonset | `true` | | `daemonset.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to Metricbeat container for DaemonSet | `[]` | | `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to Metricbeat container for DaemonSet | `[]` | | `daemonset.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | @@ -106,7 +106,10 @@ as a reference. They are also used in the automated testing of this chart. | `daemonset.secretMounts` | Allows you easily mount a secret as a file inside the DaemonSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | | `daemonset.securityContext` | Configurable [securityContext][] for Metricbeat DaemonSet pod execution environment | see [values.yaml][] | | `daemonset.tolerations` | Configurable [tolerations][] for Metricbeat DaemonSet | `[]` | +| `deployment.annotations` | Configurable [annotations][] for Metricbeat Deployment | `{}` | +| `deployment.labels` | Configurable [labels][] applied to all Metricbeat Deployment pods | `{}` | | `deployment.affinity` | Configurable [affinity][] for Metricbeat Deployment | `{}` | +| `deployment.enabled` | If true, enable deployment | `true` | | `deployment.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to Metricbeat container for Deployment | `[]` | | `deployment.extraEnvs` | Extra [environment variables][] which will be appended to Metricbeat container for Deployment | `[]` | | `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | @@ -123,9 +126,10 @@ as a reference. They are also used in the automated testing of this chart. | `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist Metricbeat registry data | `/var/lib` | | `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Metricbeat Docker image tag | `7.7.0-SNAPSHOT` | +| `imageTag` | The Metricbeat Docker image tag | `6.8.14-SNAPSHOT` | | `image` | The Metricbeat Docker image | `docker.elastic.co/beats/metricbeat` | -| `labels` | Configurable [labels][] applied to all Metricbeat pods | `{}` | +| `kube_state_metrics.enabled` | Install [kube-state-metrics](https://github.com/helm/charts/tree/master/stable/kube-state-metrics) as a dependency | `true` | +| `kube_state_metrics.host` | Define kube-state-metrics endpoint for an existing deployment. Works only if `kube_state_metrics.enabled: false` | `""` | | `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | | `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | @@ -133,7 +137,9 @@ as a reference. They are also used in the automated testing of this chart. | `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | | `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | | `replicas` | The replica count for the Metricbeat deployment talking to kube-state-metrics | `1` | +| `secrets` | Allows creating a secret from variables or a file. To add secrets from file, add suffix `.filepath` to the key of the secret key. The value will be encoded to base64. | See [values.yaml][] | | `serviceAccount` | Custom [serviceAccount][] that Metricbeat will use during execution. By default will use the service account created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | | `terminationGracePeriod` | Termination period (in seconds) to wait before killing Metricbeat pod process on pod shutdown | `30` | | `updateStrategy` | The [updateStrategy][] for the DaemonSet By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | @@ -152,6 +158,7 @@ as a reference. They are also used in the automated testing of this chart. | `resources` | Allows you to set the [resources][] for both Metricbeat DaemonSet and Deployment | `{}` | | `secretMounts` | Allows you easily mount a secret as a file inside DaemonSet and Deployment Useful for mounting certificates and other secrets | `[]` | | `tolerations` | Configurable [tolerations][] for both Metricbeat DaemonSet and Deployment | `[]` | +| `labels` | Configurable [labels][] applied to all Metricbeat pods | `[]` | ## FAQ @@ -203,35 +210,38 @@ same node. Please check [CONTRIBUTING.md][] before any contribution or for any questions about our development and testing process. - +[6.x]: https://github.com/elastic/helm-charts/releases +[#471]: https://github.com/elastic/helm-charts/pull/471 +[6.8.14-SNAPSHOT]: https://github.com/elastic/helm-charts/blob/6.8.14-SNAPSHOT/metricbeat/README.md [BREAKING_CHANGES.md]: https://github.com/elastic/helm-charts/blob/master/BREAKING_CHANGES.md [CHANGELOG.md]: https://github.com/elastic/helm-charts/blob/master/CHANGELOG.md [CONTRIBUTING.md]: https://github.com/elastic/helm-charts/blob/master/CONTRIBUTING.md [affinity]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity [annotations]: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ -[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/README.md#default +[default elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/README.md#default [cluster role rules]: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#role-and-clusterrole [environment variables]: https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config [environment from variables]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables -[examples]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples -[examples/oss]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/oss -[examples/security]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security +[examples]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples +[examples/oss]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/oss +[examples/security]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/security [helm]: https://helm.sh [hostPath]: https://kubernetes.io/docs/concepts/storage/volumes/#hostpath [hostNetwork]: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#host-namespaces [imagePullPolicy]: https://kubernetes.io/docs/concepts/containers/images/#updating-images [imagePullSecrets]: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret -[kube-state-metrics]: https://github.com/helm/charts/tree/7.7/stable/kube-state-metrics +[kube-state-metrics]: https://github.com/helm/charts/tree/6.8/stable/kube-state-metrics [kubernetes secrets]: https://kubernetes.io/docs/concepts/configuration/secret/ [labels]: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ -[metricbeat docker image]: https://www.elastic.co/guide/en/beats/metricbeat/7.7/running-on-docker.html +[metricbeat docker image]: https://www.elastic.co/guide/en/beats/metricbeat/6.8/running-on-docker.html +[metricbeat oss docker image]: https://www.docker.elastic.co/r/beats/metricbeat-oss [priorityClass]: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass [nodeSelector]: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector -[parent readme]: https://github.com/elastic/helm-charts/tree/7.7/README.md [probe]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes [resources]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ [securityContext]: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ [serviceAccount]: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +[supported configurations]: https://github.com/elastic/helm-charts/tree/6.8/README.md#supported-configurations [tolerations]: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ [updateStrategy]: https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy -[values.yaml]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/values.yaml +[values.yaml]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/values.yaml diff --git a/metricbeat/examples/default/Makefile b/metricbeat/examples/default/Makefile index 833f62973..6ff5a30e7 100644 --- a/metricbeat/examples/default/Makefile +++ b/metricbeat/examples/default/Makefile @@ -6,7 +6,7 @@ RELEASE = helm-metricbeat-default GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-default-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) ../../ test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-default-metricbeat-metrics @@ -14,4 +14,4 @@ test-metrics: test: install goss test-metrics purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) diff --git a/metricbeat/examples/default/README.md b/metricbeat/examples/default/README.md index b6913179b..decbb7a40 100644 --- a/metricbeat/examples/default/README.md +++ b/metricbeat/examples/default/README.md @@ -1,6 +1,6 @@ # Default -This example deploy Metricbeat 7.7.0-SNAPSHOT using [default values][]. +This example deploy Metricbeat 6.8.14-SNAPSHOT using [default values][]. ## Usage @@ -22,6 +22,6 @@ This example deploy Metricbeat 7.7.0-SNAPSHOT using [default values][]. You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/default/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/default/test/goss.yaml -[default values]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/default/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/values.yaml diff --git a/metricbeat/examples/default/test/goss-metrics.yaml b/metricbeat/examples/default/test/goss-metrics.yaml index 7fca303f9..a2aff34c5 100644 --- a/metricbeat/examples/default/test/goss-metrics.yaml +++ b/metricbeat/examples/default/test/goss-metrics.yaml @@ -21,13 +21,13 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' 'http://elasticsearch-master:9200/_search?q=metricset.name:state_container%20AND%20kubernetes.container.name:metricbeat': status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' file: /usr/share/metricbeat/metricbeat.yml: @@ -41,4 +41,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/examples/default/test/goss.yaml b/metricbeat/examples/default/test/goss.yaml index d353cba76..e8d6b93c6 100644 --- a/metricbeat/examples/default/test/goss.yaml +++ b/metricbeat/examples/default/test/goss.yaml @@ -25,12 +25,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' 'http://elasticsearch-master:9200/_search?q=metricset.name:container%20AND%20kubernetes.container.name:metricbeat': status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' file: /usr/share/metricbeat/metricbeat.yml: @@ -45,4 +45,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://elasticsearch-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/examples/oss/Makefile b/metricbeat/examples/oss/Makefile index 0e4828ed0..acb1124bc 100644 --- a/metricbeat/examples/oss/Makefile +++ b/metricbeat/examples/oss/Makefile @@ -6,10 +6,10 @@ RELEASE := helm-metricbeat-oss GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-oss-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-oss-metricbeat-metrics diff --git a/metricbeat/examples/oss/README.md b/metricbeat/examples/oss/README.md index c8b8ee916..0e0f35889 100644 --- a/metricbeat/examples/oss/README.md +++ b/metricbeat/examples/oss/README.md @@ -1,6 +1,6 @@ # OSS -This example deploy Metricbeat 7.7.0-SNAPSHOT using [Metricbeat OSS][] version. +This example deploy Metricbeat 6.8.14-SNAPSHOT using [Metricbeat OSS][] version. ## Usage @@ -23,5 +23,5 @@ You can also run [goss integration tests][] using `make test` [metricbeat oss]: https://www.elastic.co/downloads/beats/metricbeat-oss -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/oss/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/oss/test/goss.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/oss/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/oss/test/goss.yaml diff --git a/metricbeat/examples/oss/test/goss-metrics.yaml b/metricbeat/examples/oss/test/goss-metrics.yaml index c78aa2faa..894a7f1c0 100644 --- a/metricbeat/examples/oss/test/goss-metrics.yaml +++ b/metricbeat/examples/oss/test/goss-metrics.yaml @@ -21,12 +21,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' http://oss-master:9200/_search?q=metricset.name:state_deployment: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' file: /usr/share/metricbeat/metricbeat.yml: @@ -39,4 +39,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/examples/oss/test/goss.yaml b/metricbeat/examples/oss/test/goss.yaml index af2f0826b..6a47573b0 100644 --- a/metricbeat/examples/oss/test/goss.yaml +++ b/metricbeat/examples/oss/test/goss.yaml @@ -25,12 +25,12 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' http://oss-master:9200/_search?q=metricset.name:container: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' file: /usr/share/metricbeat/metricbeat.yml: @@ -44,4 +44,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: http://oss-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/examples/security/Makefile b/metricbeat/examples/security/Makefile index 3f92e7fe2..d57cb05be 100644 --- a/metricbeat/examples/security/Makefile +++ b/metricbeat/examples/security/Makefile @@ -6,10 +6,10 @@ RELEASE := helm-metricbeat-security GOSS_SELECTOR = release=$(RELEASE),app=helm-metricbeat-security-metricbeat install: - helm upgrade --wait --timeout=600 --install $(RELEASE) --values values.yaml ../../ + helm upgrade --wait --timeout=900s --install $(RELEASE) --values values.yaml ../../ purge: - helm del --purge $(RELEASE) + helm del $(RELEASE) test-metrics: GOSS_FILE=goss-metrics.yaml make goss GOSS_SELECTOR=release=$(RELEASE),app=helm-metricbeat-security-metricbeat-metrics diff --git a/metricbeat/examples/security/README.md b/metricbeat/examples/security/README.md index 8268e1e1f..a908d5dab 100644 --- a/metricbeat/examples/security/README.md +++ b/metricbeat/examples/security/README.md @@ -1,6 +1,6 @@ # Security -This example deploy Metricbeat 7.7.0-SNAPSHOT using authentication and TLS to connect to +This example deploy Metricbeat 6.8.14-SNAPSHOT using authentication and TLS to connect to Elasticsearch (see [values][]). @@ -23,6 +23,6 @@ Elasticsearch (see [values][]). You can also run [goss integration tests][] using `make test` -[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.7/elasticsearch/examples/security/ -[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security/test/goss.yaml -[values]: https://github.com/elastic/helm-charts/tree/7.7/metricbeat/examples/security/values.yaml +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/6.8/elasticsearch/examples/security/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/security/test/goss.yaml +[values]: https://github.com/elastic/helm-charts/tree/6.8/metricbeat/examples/security/values.yaml diff --git a/metricbeat/examples/security/test/goss-metrics.yaml b/metricbeat/examples/security/test/goss-metrics.yaml index 22ed216ca..78647169d 100644 --- a/metricbeat/examples/security/test/goss-metrics.yaml +++ b/metricbeat/examples/security/test/goss-metrics.yaml @@ -21,7 +21,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -29,7 +29,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -45,4 +45,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/examples/security/test/goss.yaml b/metricbeat/examples/security/test/goss.yaml index 8ee1ea5b9..fb4b0f843 100644 --- a/metricbeat/examples/security/test/goss.yaml +++ b/metricbeat/examples/security/test/goss.yaml @@ -25,7 +25,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -33,7 +33,7 @@ http: status: 200 timeout: 2000 body: - - 'metricbeat-7.7.0' + - 'metricbeat-6.8.14' allow-insecure: true username: '{{ .Env.ELASTICSEARCH_USERNAME }}' password: '{{ .Env.ELASTICSEARCH_PASSWORD }}' @@ -50,4 +50,4 @@ command: exit-status: 0 stdout: - 'elasticsearch: https://security-master:9200' - - 'version: 7.7.0' + - 'version: 6.8.14' diff --git a/metricbeat/requirements.lock b/metricbeat/requirements.lock index 9b027f8cc..3cbe95eaf 100644 --- a/metricbeat/requirements.lock +++ b/metricbeat/requirements.lock @@ -1,6 +1,6 @@ dependencies: - name: kube-state-metrics - repository: https://kubernetes-charts.storage.googleapis.com + repository: https://charts.helm.sh/stable version: 2.4.1 -digest: sha256:89fdea6b5f048652fc2d562ff59338a8cbf25f9053dc28976a1271b4387692b1 -generated: "2019-11-01T10:31:40.002896+01:00" +digest: sha256:948dca129bc7c16b138ed8bcbdf666c324d812e43af59d475b8bb74a53e99778 +generated: "2020-10-30T18:58:57.381827+01:00" diff --git a/metricbeat/requirements.yaml b/metricbeat/requirements.yaml index 0f8c03d9c..2d78b382a 100644 --- a/metricbeat/requirements.yaml +++ b/metricbeat/requirements.yaml @@ -2,3 +2,4 @@ dependencies: - name: 'kube-state-metrics' version: '2.4.1' repository: '@stable' + condition: kube_state_metrics.enabled diff --git a/metricbeat/templates/clusterrole.yaml b/metricbeat/templates/clusterrole.yaml index bbc209db7..851153f8e 100644 --- a/metricbeat/templates/clusterrole.yaml +++ b/metricbeat/templates/clusterrole.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ template "metricbeat.serviceAccount" . }}-cluster-role diff --git a/metricbeat/templates/clusterrolebinding.yaml b/metricbeat/templates/clusterrolebinding.yaml index dc785b2e0..e95a98db8 100644 --- a/metricbeat/templates/clusterrolebinding.yaml +++ b/metricbeat/templates/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.managedServiceAccount }} -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ template "metricbeat.serviceAccount" . }}-cluster-role-binding diff --git a/metricbeat/templates/configmap.yaml b/metricbeat/templates/configmap.yaml index 54183db1b..1272402b5 100644 --- a/metricbeat/templates/configmap.yaml +++ b/metricbeat/templates/configmap.yaml @@ -16,7 +16,7 @@ data: {{- end -}} {{- end -}} -{{- if .Values.daemonset.metricbeatConfig }} +{{- if and .Values.daemonset.enabled .Values.daemonset.metricbeatConfig }} --- apiVersion: v1 kind: ConfigMap @@ -34,7 +34,7 @@ data: {{- end -}} {{- end -}} -{{- if .Values.deployment.metricbeatConfig }} +{{- if and .Values.deployment.enabled .Values.deployment.metricbeatConfig }} --- apiVersion: v1 kind: ConfigMap diff --git a/metricbeat/templates/daemonset.yaml b/metricbeat/templates/daemonset.yaml index cbaca7558..7bcf9f8f9 100644 --- a/metricbeat/templates/daemonset.yaml +++ b/metricbeat/templates/daemonset.yaml @@ -1,3 +1,4 @@ +{{- if .Values.daemonset.enabled }} --- apiVersion: apps/v1 kind: DaemonSet @@ -8,9 +9,21 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- if .Values.daemonset.labels }} + {{- range $key, $value := .Values.daemonset.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} + {{- if .Values.daemonset.annotations}} + annotations: + {{- range $key, $value := .Values.daemonset.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} spec: selector: matchLabels: @@ -25,7 +38,7 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} {{/* This forces a restart if the configmap has changed */}} - {{- if .Values.metricbeatConfig }} + {{- if or .Values.metricbeatConfig .Values.daemonset.metricbeatConfig }} configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} {{- end }} name: "{{ template "metricbeat.fullname" . }}" @@ -34,9 +47,15 @@ spec: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- if .Values.daemonset.labels }} + {{- range $key, $value := .Values.daemonset.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} spec: affinity: {{ toYaml ( .Values.affinity | default .Values.daemonset.affinity ) | nindent 8 }} nodeSelector: {{ toYaml ( .Values.nodeSelector | default .Values.daemonset.nodeSelector ) | nindent 8 }} @@ -143,7 +162,7 @@ spec: - name: data mountPath: /usr/share/metricbeat/data # Necessary when using autodiscovery; avoid mounting it otherwise - # See: https://www.elastic.co/guide/en/beats/metricbeat/7.7/configuration-autodiscover.html + # See: https://www.elastic.co/guide/en/beats/metricbeat/6.8/configuration-autodiscover.html - name: varrundockersock mountPath: /var/run/docker.sock readOnly: true @@ -158,4 +177,5 @@ spec: {{- end }} {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/metricbeat/templates/deployment.yaml b/metricbeat/templates/deployment.yaml index ca056d42b..cc97f20e5 100644 --- a/metricbeat/templates/deployment.yaml +++ b/metricbeat/templates/deployment.yaml @@ -1,3 +1,5 @@ +# Deploy singleton instance in the whole cluster for some unique data sources, like kube-state-metrics +{{- if .Values.deployment.enabled }} --- apiVersion: apps/v1 kind: Deployment @@ -8,13 +10,26 @@ metadata: chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' + {{- if .Values.deployment.labels }} + {{- range $key, $value := .Values.deployment.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + {{- if .Values.deployment.annotations}} + annotations: + {{- range $key, $value := .Values.deployment.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} spec: replicas: {{ .Values.replicas }} selector: matchLabels: app: '{{ template "metricbeat.fullname" . }}-metrics' - chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' - heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' template: metadata: @@ -23,14 +38,22 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} {{/* This forces a restart if the configmap has changed */}} - {{- if .Values.metricbeatConfig }} + {{- if or .Values.metricbeatConfig .Values.deployment.metricbeatConfig }} configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} {{- end }} labels: app: '{{ template "metricbeat.fullname" . }}-metrics' chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' - heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' + {{- if .Values.deployment.labels }} + {{- range $key, $value := .Values.deployment.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} spec: affinity: {{ toYaml .Values.deployment.affinity | nindent 8 }} nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }} @@ -73,6 +96,10 @@ spec: image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" args: + {{- if index .Values "metricbeatConfig" "kube-state-metrics-metricbeat.yml" }} + - "-c" + - "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml" + {{- end }} - "-e" - "-E" - "http.enabled=true" @@ -87,7 +114,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: KUBE_STATE_METRICS_HOSTS + {{- if .Values.kube_state_metrics.enabled }} value: "$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_HOST):$({{ .Release.Name | replace "-" "_" | upper }}_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" + {{- else }} + value: {{ .Values.kube_state_metrics.host | default "kube-state-metrics:8080"}} + {{- end }} {{- if .Values.extraEnvs | default .Values.deployment.extraEnvs }} {{ toYaml ( .Values.extraEnvs | default .Values.deployment.extraEnvs ) | indent 8 }} {{- end }} @@ -119,4 +150,5 @@ spec: {{- end }} {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/metricbeat/templates/secret.yaml b/metricbeat/templates/secret.yaml new file mode 100644 index 000000000..115034f8c --- /dev/null +++ b/metricbeat/templates/secret.yaml @@ -0,0 +1,27 @@ +{{- if .Values.secrets }} +{{- $fullName := include "metricbeat.fullname" . -}} +{{- range .Values.secrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $fullName .name | quote }} + labels: + app: {{ $fullName | quote }} + chart: {{ $.Chart.Name | quote }} + heritage: {{ $.Release.Service | quote }} + release: {{ $.Release.Name | quote }} + {{- range $key, $value := $.Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} +data: +{{- range $key, $val := .value }} + {{- if hasSuffix "filepath" $key }} + {{ $key | replace ".filepath" "" }}: {{ $.Files.Get $val | b64enc | quote }} + {{ else }} + {{ $key }}: {{ $val | b64enc | quote }} + {{- end }} +{{- end }} +type: Opaque +{{- end }} +{{- end }} diff --git a/metricbeat/templates/serviceaccount.yaml b/metricbeat/templates/serviceaccount.yaml index 233064669..227534fa2 100644 --- a/metricbeat/templates/serviceaccount.yaml +++ b/metricbeat/templates/serviceaccount.yaml @@ -3,6 +3,10 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ template "metricbeat.serviceAccount" . }} + annotations: + {{- with .Values.serviceAccountAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} labels: app: "{{ template "metricbeat.fullname" . }}" chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" diff --git a/metricbeat/tests/metricbeat_test.py b/metricbeat/tests/metricbeat_test.py index 83265498d..557fd9cdb 100644 --- a/metricbeat/tests/metricbeat_test.py +++ b/metricbeat/tests/metricbeat_test.py @@ -1,11 +1,13 @@ import os import sys +import base64 sys.path.insert(1, os.path.join(sys.path[0], "../../helpers")) from helpers import helm_template project = "metricbeat" name = "release-name-" + project +kube_state_metric_name = "release-name-kube-state-metrics" def test_defaults(): @@ -15,6 +17,15 @@ def test_defaults(): r = helm_template(config) assert name in r["daemonset"] + assert name + "-metrics" in r["deployment"] + + assert kube_state_metric_name in r["deployment"] + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][0][ + "env" + ][1]["value"] + == "$(RELEASE_NAME_KUBE_STATE_METRICS_SERVICE_HOST):$(RELEASE_NAME_KUBE_STATE_METRICS_SERVICE_PORT_HTTP)" + ) c = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0] assert c["name"] == project @@ -537,48 +548,77 @@ def test_adding_in_deprecated_metricbeat_config(): nestedkey: value dot.notation: test - other-config.yml: | + kube-state-metrics-metricbeat.yml: | hello = world """ r = helm_template(config) c = r["configmap"][name + "-config"]["data"] assert "metricbeat.yml" in c - assert "other-config.yml" in c + assert "kube-state-metrics-metricbeat.yml" in c assert "nestedkey: value" in c["metricbeat.yml"] assert "dot.notation: test" in c["metricbeat.yml"] - assert "hello = world" in c["other-config.yml"] + assert "hello = world" in c["kube-state-metrics-metricbeat.yml"] - d = r["daemonset"][name]["spec"]["template"]["spec"] + daemonset = r["daemonset"][name]["spec"]["template"]["spec"] assert { "configMap": {"name": name + "-config", "defaultMode": 0o600}, "name": project + "-config", - } in d["volumes"] + } in daemonset["volumes"] assert { "mountPath": "/usr/share/metricbeat/metricbeat.yml", "name": project + "-config", "subPath": "metricbeat.yml", "readOnly": True, - } in d["containers"][0]["volumeMounts"] + } in daemonset["containers"][0]["volumeMounts"] assert { - "mountPath": "/usr/share/metricbeat/other-config.yml", + "mountPath": "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml", "name": project + "-config", - "subPath": "other-config.yml", + "subPath": "kube-state-metrics-metricbeat.yml", "readOnly": True, - } in d["containers"][0]["volumeMounts"] + } in daemonset["containers"][0]["volumeMounts"] assert ( "configChecksum" in r["daemonset"][name]["spec"]["template"]["metadata"]["annotations"] ) + deployment = r["deployment"][name + "-metrics"]["spec"]["template"]["spec"] + + assert { + "configMap": {"name": name + "-config", "defaultMode": 0o600}, + "name": project + "-config", + } in deployment["volumes"] + assert { + "mountPath": "/usr/share/metricbeat/metricbeat.yml", + "name": project + "-config", + "subPath": "metricbeat.yml", + "readOnly": True, + } in deployment["containers"][0]["volumeMounts"] + assert { + "mountPath": "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml", + "name": project + "-config", + "subPath": "kube-state-metrics-metricbeat.yml", + "readOnly": True, + } in deployment["containers"][0]["volumeMounts"] + assert ("/usr/share/metricbeat/kube-state-metrics-metricbeat.yml") in deployment[ + "containers" + ][0]["args"] + + assert ( + "configChecksum" + in r["deployment"][name + "-metrics"]["spec"]["template"]["metadata"][ + "annotations" + ] + ) + def test_adding_a_secret_mount(): config = """ -daemonset: +daemonset: secretMounts: - name: elastic-certificates secretName: elastic-certificates-name @@ -614,7 +654,7 @@ def test_adding_a_secret_mount(): } not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["volumes"] config = """ -deployment: +deployment: secretMounts: - name: elastic-certificates secretName: elastic-certificates-name @@ -928,24 +968,115 @@ def test_cluster_role_rules(): assert rules["resources"][0] == "something" -def test_adding_pod_labels(): +def test_adding_deprecated_labels(): config = """ labels: - app.kubernetes.io/name: metricbeat + app-test: metricbeat """ r = helm_template(config) + assert r["daemonset"][name]["metadata"]["labels"]["app-test"] == "metricbeat" assert ( - r["daemonset"][name]["metadata"]["labels"]["app.kubernetes.io/name"] + r["deployment"][name + "-metrics"]["metadata"]["labels"]["app-test"] == "metricbeat" ) assert ( - r["daemonset"][name]["spec"]["template"]["metadata"]["labels"][ - "app.kubernetes.io/name" + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "metricbeat" + ) + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["metadata"]["labels"][ + "app-test" ] == "metricbeat" ) +def test_adding_daemonset_labels(): + config = """ +daemonset: + labels: + app-test: metricbeat +""" + r = helm_template(config) + assert r["daemonset"][name]["metadata"]["labels"]["app-test"] == "metricbeat" + assert ( + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "metricbeat" + ) + + +def test_adding_daemonset_labels_surpasses_root_labels(): + config = """ +labels: + app-test: root-metricbeat +daemonset: + labels: + app-test: daemonset-metricbeat +""" + r = helm_template(config) + assert ( + r["daemonset"][name]["metadata"]["labels"]["app-test"] == "daemonset-metricbeat" + ) + assert ( + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "daemonset-metricbeat" + ) + + +def test_adding_deployment_labels(): + config = """ +deployment: + labels: + app-test: metricbeat +""" + r = helm_template(config) + assert ( + r["deployment"][name + "-metrics"]["metadata"]["labels"]["app-test"] + == "metricbeat" + ) + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["metadata"]["labels"][ + "app-test" + ] + == "metricbeat" + ) + + +def test_adding_deployment_labels_surpasses_root_labels(): + config = """ +labels: + app-test: root-metricbeat +deployment: + labels: + app-test: deployment-metricbeat +""" + r = helm_template(config) + assert ( + r["deployment"][name + "-metrics"]["metadata"]["labels"]["app-test"] + == "deployment-metricbeat" + ) + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["metadata"]["labels"][ + "app-test" + ] + == "deployment-metricbeat" + ) + + +def test_adding_serviceaccount_annotations(): + config = """ +serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][name]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) + + def test_adding_env_from(): config = """ daemonset: @@ -1094,3 +1225,236 @@ def test_setting_fullnameOverride(): "type": "DirectoryOrCreate", }, } in volumes + + +def test_adding_annotations(): + config = """ +daemonset: + annotations: + foo: "bar" +""" + r = helm_template(config) + assert "foo" in r["daemonset"][name]["metadata"]["annotations"] + assert r["daemonset"][name]["metadata"]["annotations"]["foo"] == "bar" + assert "annotations" not in r["deployment"][name + "-metrics"]["metadata"] + config = """ +deployment: + annotations: + grault: "waldo" +""" + r = helm_template(config) + assert "grault" in r["deployment"][name + "-metrics"]["metadata"]["annotations"] + assert ( + r["deployment"][name + "-metrics"]["metadata"]["annotations"]["grault"] + == "waldo" + ) + assert "annotations" not in r["daemonset"][name]["metadata"] + + +def test_disable_daemonset(): + config = """ +daemonset: + enabled: false +""" + r = helm_template(config) + cfg = r["configmap"] + + assert name not in r.get("daemonset", {}) + assert name + "-daemonset-config" not in cfg + assert name + "-deployment-config" in cfg + + +def test_disable_deployment(): + config = """ +deployment: + enabled: false +""" + r = helm_template(config) + cfg = r["configmap"] + + assert name + "-metrics" not in r.get("deployment", {}) + assert name + "-daemonset-config" in cfg + assert name + "-deployment-config" not in cfg + + +def test_do_not_install_kube_stat_metrics(): + config = """ +kube_state_metrics: + enabled: false +""" + r = helm_template(config) + + assert kube_state_metric_name not in r["deployment"] + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][0][ + "env" + ][1]["name"] + == "KUBE_STATE_METRICS_HOSTS" + ) + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][0][ + "env" + ][1]["value"] + == "kube-state-metrics:8080" + ) + + +def test_custom_kube_stat_metrics_host(): + config = """ +kube_state_metrics: + enabled: false + host: "kube-state-metrics.kube-system:9999" +""" + r = helm_template(config) + + assert ( + r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["containers"][0][ + "env" + ][1]["value"] + == "kube-state-metrics.kube-system:9999" + ) + + +def test_adding_a_secret(): + content = "LS1CRUdJTiBgUFJJVkFURSB" + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} +""".format( + elk_pass=content + ) + content_b64 = base64.b64encode(content.encode("ascii")).decode("ascii") + + r = helm_template(config) + secret_name = name + "-env" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 1 + assert s["data"] == {"ELASTICSEARCH_PASSWORD": content_b64} + + +def test_adding_secret_from_file(): + content = """ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEApCt3ychnqZHsS +DylPFZn55xDaDcWco1oNFdBGzFjw+ +zkuMFMOv7ab+yOFwHeEeAAEkEgy1u +Da1vIscBs1K0kbEFRSqySLuNHWiJp +wK2cI/gJc+S9Qd9Qsn0XGjmjQ6P2p +ot2hvCOtnei998OmDSYORKBq2jiv/ +-----END RSA PRIVATE KEY----- +""" + config = """ +secrets: + - name: "tls" + value: + cert.key.filepath: "secrets/private.key" +""" + content_b64 = base64.b64encode(content.encode("ascii")).decode("ascii") + work_dir = os.path.join(os.path.abspath(os.getcwd()), "secrets") + filename = os.path.join(work_dir, "private.key") + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + f.write(content) + + with open(filename, "r") as f: + data = f.read() + assert data == content + + r = helm_template(config) + secret_name = name + "-tls" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 1 + assert s["data"] == { + "cert.key": content_b64, + } + + os.remove(filename) + os.rmdir(work_dir) + + +def test_adding_multiple_data_secret(): + content = { + "elk_pass": "LS1CRUdJTiBgUFJJVkFURSB", + "api_key": "ui2CsdUadTiBasRJRkl9tvNnw", + } + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} + api_key: {api_key} +""".format( + elk_pass=content["elk_pass"], api_key=content["api_key"] + ) + content_b64 = { + "elk_pass": base64.b64encode(content["elk_pass"].encode("ascii")).decode( + "ascii" + ), + "api_key": base64.b64encode(content["api_key"].encode("ascii")).decode("ascii"), + } + + r = helm_template(config) + secret_name = name + "-env" + s = r["secret"][secret_name] + assert s["metadata"]["labels"]["app"] == name + assert len(r["secret"]) == 1 + assert len(s["data"]) == 2 + assert s["data"] == { + "ELASTICSEARCH_PASSWORD": content_b64["elk_pass"], + "api_key": content_b64["api_key"], + } + + +def test_adding_multiple_secrets(): + content = { + "elk_pass": "LS1CRUdJTiBgUFJJVkFURSB", + "cert_crt": "LS0tLS1CRUdJTiBlRJRALKJDDQVRFLS0tLS0K", + "cert_key": "LS0tLS1CRUdJTiBgUFJJVkFURSBLRVktLS0tLQo", + } + config = """ +secrets: + - name: "env" + value: + ELASTICSEARCH_PASSWORD: {elk_pass} + - name: "tls" + value: + cert.crt: {cert_crt} + cert.key: {cert_key} + +""".format( + elk_pass=content["elk_pass"], + cert_crt=content["cert_crt"], + cert_key=content["cert_key"], + ) + content_b64 = { + "elk_pass": base64.b64encode(content["elk_pass"].encode("ascii")).decode( + "ascii" + ), + "cert_crt": base64.b64encode(content["cert_crt"].encode("ascii")).decode( + "ascii" + ), + "cert_key": base64.b64encode(content["cert_key"].encode("ascii")).decode( + "ascii" + ), + } + + r = helm_template(config) + secret_names = {"env": name + "-env", "tls": name + "-tls"} + s_env = r["secret"][secret_names["env"]] + s_tls = r["secret"][secret_names["tls"]] + assert len(r["secret"]) == 2 + assert len(s_env["data"]) == 1 + assert s_env["data"] == { + "ELASTICSEARCH_PASSWORD": content_b64["elk_pass"], + } + assert len(s_tls["data"]) == 2 + assert s_tls["data"] == { + "cert.crt": content_b64["cert_crt"], + "cert.key": content_b64["cert_key"], + } diff --git a/metricbeat/values.yaml b/metricbeat/values.yaml index 7ef153adb..01616bddb 100755 --- a/metricbeat/values.yaml +++ b/metricbeat/values.yaml @@ -1,7 +1,13 @@ --- daemonset: + # Annotations to apply to the daemonset + annotations: {} + # additionals labels + labels: {} affinity: {} + # Include the daemonset + enabled: true # Extra environment variables for Metricbeat container. envFrom: [] # - configMapRef: @@ -91,7 +97,13 @@ daemonset: tolerations: [] deployment: + # Annotations to apply to the deployment + annotations: {} + # additionals labels + labels: {} affinity: {} + # Include the deployment + enabled: true # Extra environment variables for Metricbeat container. envFrom: [] # - configMapRef: @@ -159,7 +171,7 @@ extraInitContainers: "" hostPathRoot: /var/lib image: "docker.elastic.co/beats/metricbeat" -imageTag: "7.7.0-SNAPSHOT" +imageTag: "6.8.14-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] @@ -189,9 +201,6 @@ readinessProbe: periodSeconds: 10 timeoutSeconds: 5 -# additionals labels -labels: {} - # Whether this chart should self-manage its service account, role, and associated role binding. managedServiceAccount: true @@ -224,6 +233,10 @@ podAnnotations: {} # Custom service account override that the pod will use serviceAccount: "" +# Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. +serviceAccountAnnotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount + # How long to wait for metricbeat pods to stop gracefully terminationGracePeriod: 30 @@ -238,6 +251,27 @@ updateStrategy: RollingUpdate nameOverride: "" fullnameOverride: "" +kube_state_metrics: + enabled: true + # host is used only when kube_state_metrics.enabled: false + host: "" + +# Add sensitive data to k8s secrets +secrets: [] +# - name: "env" +# value: +# ELASTICSEARCH_PASSWORD: "LS1CRUdJTiBgUFJJVkFURSB" +# api_key: ui2CsdUadTiBasRJRkl9tvNnw +# - name: "tls" +# value: +# ca.crt: | +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# LS0tLS1CRUdJT0K +# cert.crt: "LS0tLS1CRUdJTiBlRJRklDQVRFLS0tLS0K" +# cert.key.filepath: "secrets.crt" # The path to file should be relative to the `values.yaml` file. + # DEPRECATED affinity: {} envFrom: [] @@ -252,3 +286,4 @@ podSecurityContext: {} resources: {} secretMounts: [] tolerations: [] +labels: {}