From ed1fbf60036c810028f573fd81da6fd7c08c7ecb Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 24 Sep 2020 12:53:55 +0300 Subject: [PATCH 01/36] Support for Deployment Kubernetes resource This commit indtroduces the feature of deploying a Kubernetes deployment instead of a Daemonset using Filebeat, using a `values.yaml` syntax as below: `values.yaml` --- ```yaml [...] deploymentType: [daemonset|deployment] [...] ``` Specifically, this is used for creation of Filebeat instances not bound to each Worker, conducting non-Worker-related work, such as collection of AWS CloudTrail logs as described in [1]. [1]:https://github.com/elastic/helm-charts/issues/821 --- filebeat/README.md | 2 + filebeat/templates/daemonset.yaml | 2 + filebeat/templates/deployment.yaml | 172 +++++++++++++++++++++++++++++ filebeat/values.yaml | 3 + 4 files changed, 179 insertions(+) create mode 100644 filebeat/templates/deployment.yaml diff --git a/filebeat/README.md b/filebeat/README.md index dea1c5b9d..fe9155e1d 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -105,6 +105,8 @@ as a reference. They are also used in the automated testing of this chart. | `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | | `imageTag` | The Filebeat Docker image tag | `8.0.0-SNAPSHOT` | | `image` | The Filebeat Docker image | `docker.elastic.co/beats/filebeat` | +| `deploymentType` | Whether Filebeat will be deployed as `DaemonSet` running on all Worker nodes (default) or `Deployment` | `daemonset` | + | `labels` | Configurable [labels][] applied to all Filebeat pods | `{}` | | `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | | `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index dbd446748..753943087 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -1,4 +1,5 @@ --- +{{- if eq (printf "%s" (.Values.deploymentType | lower | default "daemonset")) "daemonset" }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -169,3 +170,4 @@ spec: {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} {{- end }} +{{- end }} diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml new file mode 100644 index 000000000..6439ad0f2 --- /dev/null +++ b/filebeat/templates/deployment.yaml @@ -0,0 +1,172 @@ +--- +{{- if eq (printf "%s" .Values.deploymentType | lower) "deployment" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "filebeat.fullname" . }} + labels: + app: "{{ template "filebeat.fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + selector: + matchLabels: + app: "{{ template "filebeat.fullname" . }}" + release: {{ .Release.Name | quote }} + template: + metadata: + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.filebeatConfig }} + configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} + {{- end }} + name: "{{ template "filebeat.fullname" . }}" + labels: + app: "{{ template "filebeat.fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + spec: + {{- with .Values.tolerations }} + tolerations: {{ toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{ toYaml . | nindent 8 -}} + {{- end }} + serviceAccountName: {{ template "filebeat.serviceAccount" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} + {{- if .Values.hostNetworking }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + {{- end }} + volumes: + {{- range .Values.secretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} + {{- if .Values.filebeatConfig }} + - name: filebeat-config + configMap: + defaultMode: 0600 + name: {{ template "filebeat.fullname" . }}-config + {{- end }} + - name: data + hostPath: + path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-{{ .Release.Namespace }}-data + type: DirectoryOrCreate + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log + - name: varrundockersock + hostPath: + path: /var/run/docker.sock + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | indent 6 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + # All the other beats accept a string here while + # filebeat accepts a valid yaml array. We're keeping + # this as a backwards compatible change, while adding + # also a way to pass a string as other templates to + # make these implementations consistent. + # https://github.com/elastic/helm-charts/issues/490 + {{- if eq "string" (printf "%T" .Values.extraInitContainers) }} +{{ tpl .Values.extraInitContainers . | indent 8 }} + {{- else }} +{{ toYaml .Values.extraInitContainers | indent 8 }} + {{- end }} + {{- end }} + containers: + - name: "filebeat" + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + args: + - "-e" + - "-E" + - "http.enabled=true" + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName +{{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 8 }} +{{- end }} +{{- if .Values.envFrom }} + envFrom: +{{ toYaml .Values.envFrom | indent 10 }} +{{- end }} +{{- if .Values.podSecurityContext }} + securityContext: +{{ toYaml .Values.podSecurityContext | indent 10 }} +{{- end }} + volumeMounts: + {{- range .Values.secretMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} + {{- end }} + {{- range $path, $config := .Values.filebeatConfig }} + - name: filebeat-config + mountPath: /usr/share/filebeat/{{ $path }} + readOnly: true + subPath: {{ $path }} + {{- end }} + - name: data + mountPath: /usr/share/filebeat/data + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + # Necessary when using autodiscovery; avoid mounting it otherwise + # See: https://www.elastic.co/guide/en/beats/filebeat/master/configuration-autodiscover.html + - name: varrundockersock + mountPath: /var/run/docker.sock + readOnly: true + {{- if .Values.extraVolumeMounts }} +{{ toYaml .Values.extraVolumeMounts | indent 8 }} + {{- end }} + {{- if .Values.extraContainers }} +{{ tpl .Values.extraContainers . | indent 6 }} + {{- end }} + +{{- end }} diff --git a/filebeat/values.yaml b/filebeat/values.yaml index e8c4ce22d..d7c5c3b4d 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -56,6 +56,9 @@ imageTag: "8.0.0-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] +# Choice between DaemonSet (default) and Deployment +deploymentType: daemonset + livenessProbe: exec: command: From 2e1a6d62d4a648ddf8fea62774ce02e1695ae7bf Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 24 Sep 2020 13:18:24 +0300 Subject: [PATCH 02/36] Tests on 'deploymentType' YAML directive This commit adds a default value test for `deploymentType`. Additionally, * `test_deployment_type_deployment` Checks if a `Deployment` is created but NOT a `DaemonSet` * `test_deployment_type_daemonset` Checks if a `DaemonSet` is created but NOT a `Deployment` * `test_deployment_type_case_insensitive` Checks if `deploymentType` value is accepted in a case-insensitive way. --- filebeat/tests/filebeat_test.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 7c8dc0ad0..1c798430f 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -20,6 +20,8 @@ def test_defaults(): assert c["name"] == project assert c["image"].startswith("docker.elastic.co/beats/" + project + ":") + assert c["deploymentType"] == "daemonset" + assert c["env"][0]["name"] == "POD_NAMESPACE" assert c["env"][0]["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.namespace" @@ -400,3 +402,33 @@ def test_setting_fullnameOverride(): "type": "DirectoryOrCreate", }, } in volumes + + +def test_deployment_type_deployment(): + config = """ +deploymentType: 'deployment' +""" + r = helm_template(config) + + assert "daemonset" not in r + assert r["deployment"] + + +def test_deployment_type_daemonset(): + config = """ +deploymentType: 'daemonset' +""" + r = helm_template(config) + + assert "deployment" not in r + assert r["daemonset"] + + +def test_deployment_type_case_insensitive(): + config = """ +deploymentType: 'DePloYmEnT' +""" + r = helm_template(config) + + assert "daemonset" not in r + assert r["deployment"] From 959abbffe8308b00df5f82820e2b82a3b164f81b Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 21 Oct 2020 13:51:59 +0300 Subject: [PATCH 03/36] Similar to Metricbeat Templating This commit uses the MetricBeat Helm chart to create a Daemonset/Deployment Helm chart for Filebeat. Uses the ```yaml daemonset: [...] deployment: [...] ``` structure falling back to root key defaults. --- filebeat/templates/configmap.yaml | 36 ++++++ filebeat/templates/daemonset.yaml | 47 +++---- filebeat/templates/deployment.yaml | 132 +++++++------------ filebeat/tests/filebeat_test.py | 32 ----- filebeat/values.yaml | 201 +++++++++++++++++++---------- 5 files changed, 238 insertions(+), 210 deletions(-) diff --git a/filebeat/templates/configmap.yaml b/filebeat/templates/configmap.yaml index 32df8d87c..008de825b 100644 --- a/filebeat/templates/configmap.yaml +++ b/filebeat/templates/configmap.yaml @@ -15,3 +15,39 @@ data: {{ $config | indent 4 -}} {{- end -}} {{- end -}} + +{{- if .Values.daemonset.filebeatConfig }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "filebeat.fullname" . }}-daemonset-config + labels: + app: "{{ template "filebeat.fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +data: +{{- range $path, $config := .Values.daemonset.filebeatConfig }} + {{ $path }}: | +{{ $config | indent 4 -}} +{{- end -}} +{{- end -}} + +{{- if .Values.deployment.filebeatConfig }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "filebeat.fullname" . }}-deployment-config + labels: + app: "{{ template "filebeat.fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +data: +{{- range $path, $config := .Values.deployment.filebeatConfig }} + {{ $path }}: | +{{ $config | indent 4 -}} +{{- end -}} +{{- end -}} diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 753943087..05cd57c51 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -1,5 +1,5 @@ +{{- if .Values.daemonset.enabled }} --- -{{- if eq (printf "%s" (.Values.deploymentType | lower | default "daemonset")) "daemonset" }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -39,18 +39,14 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} spec: - {{- with .Values.tolerations }} - tolerations: {{ toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} + {{- with ( .Values.tolerations | default .Values.daemonset.tolerations ) }} + tolerations: {{ toYaml . | nindent 8 }} {{- end }} + nodeSelector: {{ toYaml ( .Values.nodeSelector | default .Values.daemonset.nodeSelector ) | nindent 8 }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 -}} - {{- end }} + affinity: {{ toYaml ( .Values.affinity | default .Values.daemonset.affinity ) | nindent 8 }} serviceAccountName: {{ template "filebeat.serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} {{- if .Values.hostNetworking }} @@ -58,7 +54,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet {{- end }} volumes: - {{- range .Values.secretMounts }} + {{- range .Values.secretMounts | default .Values.daemonset.secretMounts }} - name: {{ .name }} secret: secretName: {{ .secretName }} @@ -68,6 +64,11 @@ spec: configMap: defaultMode: 0600 name: {{ template "filebeat.fullname" . }}-config + {{- else if .Values.daemonset.filebeatConfig }} + - name: filebeat-config + configMap: + defaultMode: 0600 + name: {{ template "filebeat.fullname" . }}-daemonset-config {{- end }} - name: data hostPath: @@ -82,8 +83,8 @@ spec: - name: varrundockersock hostPath: path: /var/run/docker.sock - {{- if .Values.extraVolumes }} -{{ toYaml .Values.extraVolumes | indent 6 }} + {{- if .Values.extraVolumes | default .Values.daemonset.extraVolumes }} +{{ toYaml ( .Values.extraVolumes | default .Values.daemonset.extraVolumes ) | indent 6 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -116,7 +117,7 @@ spec: readinessProbe: {{ toYaml .Values.readinessProbe | indent 10 }} resources: -{{ toYaml .Values.resources | indent 10 }} +{{ toYaml ( .Values.resources | default .Values.daemonset.resources ) | indent 10 }} env: - name: POD_NAMESPACE valueFrom: @@ -126,19 +127,13 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName -{{- if .Values.extraEnvs }} -{{ toYaml .Values.extraEnvs | indent 8 }} -{{- end }} -{{- if .Values.envFrom }} - envFrom: -{{ toYaml .Values.envFrom | indent 10 }} -{{- end }} -{{- if .Values.podSecurityContext }} - securityContext: -{{ toYaml .Values.podSecurityContext | indent 10 }} +{{- if .Values.extraEnvs | default .Values.daemonset.extraEnvs }} +{{ toYaml ( .Values.extraEnvs | default .Values.daemonset.extraEnvs ) | indent 8 }} {{- end }} + envFrom: {{ toYaml ( .Values.envFrom | default .Values.daemonset.envFrom ) | nindent 10 }} + securityContext: {{ toYaml ( .Values.podSecurityContext | default .Values.daemonset.securityContext ) | nindent 10 }} volumeMounts: - {{- range .Values.secretMounts }} + {{- range .Values.secretMounts | default .Values.daemonset.secretMounts }} - name: {{ .name }} mountPath: {{ .path }} {{- if .subPath }} @@ -164,8 +159,8 @@ spec: - name: varrundockersock mountPath: /var/run/docker.sock readOnly: true - {{- if .Values.extraVolumeMounts }} -{{ toYaml .Values.extraVolumeMounts | indent 8 }} + {{- if .Values.extraVolumeMounts | default .Values.daemonset.extraVolumeMounts }} +{{ toYaml (.Values.extraVolumeMounts | default .Values.daemonset.extraVolumeMounts ) | indent 8 }} {{- end }} {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index 6439ad0f2..08c218250 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -1,22 +1,28 @@ +# Deploy singleton instance in the whole cluster for some unique data sources, like aws input +{{- if .Values.deployment.enabled }} --- -{{- if eq (printf "%s" .Values.deploymentType | lower) "deployment" }} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ template "filebeat.fullname" . }} + name: '{{ template "filebeat.fullname" . }}' labels: - app: "{{ template "filebeat.fullname" . }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} - {{- range $key, $value := .Values.labels }} + app: '{{ template "filebeat.fullname" . }}' + chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' + {{- if .Values.deployment.annotations}} + annotations: + {{- range $key, $value := .Values.deployment.annotations }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} spec: + replicas: {{ .Values.replicas }} selector: matchLabels: - app: "{{ template "filebeat.fullname" . }}" - release: {{ .Release.Name | quote }} + app: '{{ template "filebeat.fullname" . }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' template: metadata: annotations: @@ -24,39 +30,28 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} {{/* This forces a restart if the configmap has changed */}} - {{- if .Values.filebeatConfig }} + {{- if or .Values.filebeatConfig .Values.deployment.filebeatConfig }} configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} {{- end }} - name: "{{ template "filebeat.fullname" . }}" labels: - app: "{{ template "filebeat.fullname" . }}" - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - heritage: {{ .Release.Service | quote }} - release: {{ .Release.Name | quote }} + app: '{{ template "filebeat.fullname" . }}' + chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' + heritage: '{{ .Release.Service }}' + release: '{{ .Release.Name }}' {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} spec: - {{- with .Values.tolerations }} - tolerations: {{ toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: {{ toYaml . | nindent 8 }} - {{- end }} + affinity: {{ toYaml .Values.deployment.affinity | nindent 8 }} + nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }} + tolerations: {{ toYaml ( .Values.tolerations | default .Values.deployment.tolerations ) | nindent 8 }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} - {{- with .Values.affinity }} - affinity: {{ toYaml . | nindent 8 -}} - {{- end }} serviceAccountName: {{ template "filebeat.serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} - {{- if .Values.hostNetworking }} - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - {{- end }} volumes: - {{- range .Values.secretMounts }} + {{- range .Values.secretMounts | default .Values.deployment.secretMounts }} - name: {{ .name }} secret: secretName: {{ .secretName }} @@ -66,22 +61,14 @@ spec: configMap: defaultMode: 0600 name: {{ template "filebeat.fullname" . }}-config + {{- else if .Values.deployment.filebeatConfig }} + - name: filebeat-config + configMap: + defaultMode: 0600 + name: {{ template "filebeat.fullname" . }}-deployment-config {{- end }} - - name: data - hostPath: - path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-{{ .Release.Namespace }}-data - type: DirectoryOrCreate - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers - - name: varlog - hostPath: - path: /var/log - - name: varrundockersock - hostPath: - path: /var/run/docker.sock - {{- if .Values.extraVolumes }} -{{ toYaml .Values.extraVolumes | indent 6 }} + {{- if .Values.extraVolumes | default .Values.deployment.extraVolumes }} +{{ toYaml ( .Values.extraVolumes | default .Values.deployment.extraVolumes ) | indent 6 }} {{- end }} {{- if .Values.imagePullSecrets }} imagePullSecrets: @@ -96,9 +83,9 @@ spec: # make these implementations consistent. # https://github.com/elastic/helm-charts/issues/490 {{- if eq "string" (printf "%T" .Values.extraInitContainers) }} -{{ tpl .Values.extraInitContainers . | indent 8 }} +{{ tpl .Values.extraInitContainers . | indent 6 }} {{- else }} -{{ toYaml .Values.extraInitContainers | indent 8 }} +{{ toYaml .Values.extraInitContainers | indent 6 }} {{- end }} {{- end }} containers: @@ -106,37 +93,25 @@ spec: image: "{{ .Values.image }}:{{ .Values.imageTag }}" imagePullPolicy: "{{ .Values.imagePullPolicy }}" args: - - "-e" - - "-E" - - "http.enabled=true" + - "-e" + - "-E" livenessProbe: {{ toYaml .Values.livenessProbe | indent 10 }} readinessProbe: {{ toYaml .Values.readinessProbe | indent 10 }} - resources: -{{ toYaml .Values.resources | indent 10 }} + resources: {{ toYaml ( .Values.resources | default .Values.deployment.resources ) | nindent 10 }} env: - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName -{{- if .Values.extraEnvs }} -{{ toYaml .Values.extraEnvs | indent 8 }} -{{- end }} -{{- if .Values.envFrom }} - envFrom: -{{ toYaml .Values.envFrom | indent 10 }} -{{- end }} -{{- if .Values.podSecurityContext }} - securityContext: -{{ toYaml .Values.podSecurityContext | indent 10 }} +{{- if .Values.extraEnvs | default .Values.deployment.extraEnvs }} +{{ toYaml ( .Values.extraEnvs | default .Values.deployment.extraEnvs ) | indent 8 }} {{- end }} + envFrom: {{ toYaml ( .Values.envFrom | default .Values.deployment.envFrom ) | nindent 10 }} + securityContext: {{ toYaml ( .Values.podSecurityContext | default .Values.deployment.securityContext ) | nindent 10 }} volumeMounts: - {{- range .Values.secretMounts }} + {{- range .Values.secretMounts | default .Values.deployment.secretMounts }} - name: {{ .name }} mountPath: {{ .path }} {{- if .subPath }} @@ -148,25 +123,18 @@ spec: mountPath: /usr/share/filebeat/{{ $path }} readOnly: true subPath: {{ $path }} - {{- end }} - - name: data - mountPath: /usr/share/filebeat/data - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers - readOnly: true - - name: varlog - mountPath: /var/log - readOnly: true - # Necessary when using autodiscovery; avoid mounting it otherwise - # See: https://www.elastic.co/guide/en/beats/filebeat/master/configuration-autodiscover.html - - name: varrundockersock - mountPath: /var/run/docker.sock + {{ else }} + {{- range $path, $config := .Values.deployment.filebeatConfig }} + - name: filebeat-config + mountPath: /usr/share/filebeat/{{ $path }} readOnly: true - {{- if .Values.extraVolumeMounts }} -{{ toYaml .Values.extraVolumeMounts | indent 8 }} + subPath: {{ $path }} + {{- end }} + {{- end }} + {{- if .Values.extraVolumeMounts | default .Values.deployment.extraVolumeMounts }} +{{ toYaml ( .Values.extraVolumeMounts | default .Values.deployment.extraVolumeMounts ) | indent 8 }} {{- end }} {{- if .Values.extraContainers }} {{ tpl .Values.extraContainers . | indent 6 }} {{- end }} - {{- end }} diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 1c798430f..7c8dc0ad0 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -20,8 +20,6 @@ def test_defaults(): assert c["name"] == project assert c["image"].startswith("docker.elastic.co/beats/" + project + ":") - assert c["deploymentType"] == "daemonset" - assert c["env"][0]["name"] == "POD_NAMESPACE" assert c["env"][0]["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.namespace" @@ -402,33 +400,3 @@ def test_setting_fullnameOverride(): "type": "DirectoryOrCreate", }, } in volumes - - -def test_deployment_type_deployment(): - config = """ -deploymentType: 'deployment' -""" - r = helm_template(config) - - assert "daemonset" not in r - assert r["deployment"] - - -def test_deployment_type_daemonset(): - config = """ -deploymentType: 'daemonset' -""" - r = helm_template(config) - - assert "deployment" not in r - assert r["daemonset"] - - -def test_deployment_type_case_insensitive(): - config = """ -deploymentType: 'DePloYmEnT' -""" - r = helm_template(config) - - assert "daemonset" not in r - assert r["deployment"] diff --git a/filebeat/values.yaml b/filebeat/values.yaml index d7c5c3b4d..e6cd2f277 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -1,38 +1,121 @@ --- -# Allows you to add any config files in /usr/share/filebeat -# such as filebeat.yml -filebeatConfig: - filebeat.yml: | - filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" - - output.elasticsearch: - host: '${NODE_NAME}' - hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' - -# Extra environment variables to append to the DaemonSet pod spec. -# This will be appended to the current 'env:' key. You can use any of the kubernetes env -# syntax here -extraEnvs: [] -# - name: MY_ENVIRONMENT_VAR -# value: the_value_goes_here - -extraVolumeMounts: [] +daemonset: + # Annotations to apply to the daemonset + annotations: {} + affinity: {} + # Include the daemonset + enabled: true + # Extra environment variables for Filebeat container. + envFrom: [] + # - configMapRef: + # name: config-secret + extraEnvs: [] + # - name: MY_ENVIRONMENT_VAR + # value: the_value_goes_here + extraVolumes: [] + # - name: extras + # emptyDir: {} + extraVolumeMounts: [] + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + hostNetworking: false + # Allows you to add any config files in /usr/share/filebeat + # such as filebeat.yml for daemonset + filebeatConfig: + filebeat.yml: | + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + + output.elasticsearch: + host: '${NODE_NAME}' + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + nodeSelector: {} + # A list of secrets and their paths to mount inside the pod + # This is useful for mounting certificates for security other sensitive values + secretMounts: [] + # - name: filebeat-certificates + # secretName: filebeat-certificates + # path: /usr/share/filebeat/certs + # Various pod security context settings. Bear in mind that many of these have an impact on Filebeat functioning properly. + # + # - User that the container will execute as. Typically necessary to run as root (0) in order to properly collect host container logs. + # - Whether to execute the Filebeat containers as privileged containers. Typically not necessarily unless running within environments such as OpenShift. + podSecurityContext: + runAsUser: 0 + privileged: false + resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "1000m" + memory: "200Mi" + +deployment: + # Annotations to apply to the deployment + annotations: {} + affinity: {} + # Include the deployment + enabled: true + # Extra environment variables for Filebeat container. + envFrom: [] + # - configMapRef: + # name: config-secret + extraEnvs: [] + # - name: MY_ENVIRONMENT_VAR + # value: the_value_goes_here + # Allows you to add any config files in /usr/share/filebeat + extraVolumes: [] + # - name: extras + # emptyDir: {} + extraVolumeMounts: [] # - name: extras # mountPath: /usr/share/extras # readOnly: true - -extraVolumes: [] - # - name: extras - # emptyDir: {} + # such as filebeat.yml for deployment + filebeatConfig: + filebeat.yml: | + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" + + output.elasticsearch: + host: '${NODE_NAME}' + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + nodeSelector: {} + # A list of secrets and their paths to mount inside the pod + # This is useful for mounting certificates for security other sensitive values + secretMounts: [] + # - name: filebeat-certificates + # secretName: filebeat-certificates + # path: /usr/share/filebeat/certs + securityContext: + runAsUser: 0 + privileged: false + resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "1000m" + memory: "200Mi" + tolerations: [] extraContainers: "" # - name: dummy-init @@ -41,24 +124,15 @@ extraContainers: "" extraInitContainers: [] # - name: dummy-init -# image: busybox -# command: ['echo', 'hey'] - -envFrom: [] -# - configMapRef: -# name: configmap-name # Root directory where Filebeat will write data to in order to persist registry data across pod restarts (file position and other metadata). hostPathRoot: /var/lib -hostNetworking: false + image: "docker.elastic.co/beats/filebeat" imageTag: "8.0.0-SNAPSHOT" imagePullPolicy: "IfNotPresent" imagePullSecrets: [] -# Choice between DaemonSet (default) and Deployment -deploymentType: daemonset - livenessProbe: exec: command: @@ -85,59 +159,46 @@ readinessProbe: periodSeconds: 10 timeoutSeconds: 5 -# Whether this chart should self-manage its service account, role, and associated role binding. -managedServiceAccount: true - # additionals labels labels: {} +# Whether this chart should self-manage its service account, role, and associated role binding. +managedServiceAccount: true + podAnnotations: {} # iam.amazonaws.com/role: es-cluster -# Various pod security context settings. Bear in mind that many of these have an impact on Filebeat functioning properly. -# -# - User that the container will execute as. Typically necessary to run as root (0) in order to properly collect host container logs. -# - Whether to execute the Filebeat containers as privileged containers. Typically not necessarily unless running within environments such as OpenShift. -podSecurityContext: - runAsUser: 0 - privileged: false - -resources: - requests: - cpu: "100m" - memory: "100Mi" - limits: - cpu: "1000m" - memory: "200Mi" +# Custom service account override that the pod will use +serviceAccount: "" # Custom service account override that the pod will use serviceAccount: "" # Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. serviceAccountAnnotations: {} + # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount -# A list of secrets and their paths to mount inside the pod -# This is useful for mounting certificates for security other sensitive values -secretMounts: [] -# - name: filebeat-certificates -# secretName: filebeat-certificates -# path: /usr/share/filebeat/certs # How long to wait for Filebeat pods to stop gracefully terminationGracePeriod: 30 +# This is the PriorityClass settings as defined in +# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass +priorityClassName: "" + +updateStrategy: RollingUpdate + +# Override various naming aspects of this chart +# Only edit these if you know what you're doing +nameOverride: "" +fullnameOverride: "" tolerations: [] -nodeSelector: {} affinity: {} -# This is the PriorityClass settings as defined in -# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass -priorityClassName: "" -updateStrategy: RollingUpdate # Override various naming aspects of this chart # Only edit these if you know what you're doing From 481104eef7273f026708738d2f80451a7e74b87a Mon Sep 17 00:00:00 2001 From: John Torakis Date: Mon, 2 Nov 2020 12:29:57 +0200 Subject: [PATCH 04/36] Fix double value in `filebeat/values.yaml` The value: ```yaml serviceAccount: "" ``` was existing twice in the `filebeat/values.yaml` file. --- filebeat/values.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/filebeat/values.yaml b/filebeat/values.yaml index e6cd2f277..4c41a0296 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -171,15 +171,11 @@ podAnnotations: {} # Custom service account override that the pod will use serviceAccount: "" -# Custom service account override that the pod will use -serviceAccount: "" - # Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. serviceAccountAnnotations: {} # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount - # How long to wait for Filebeat pods to stop gracefully terminationGracePeriod: 30 # This is the PriorityClass settings as defined in @@ -198,8 +194,6 @@ tolerations: [] affinity: {} - - # Override various naming aspects of this chart # Only edit these if you know what you're doing nameOverride: "" From eaa12c66c6b805d8ff57b4832e890d08c39462da Mon Sep 17 00:00:00 2001 From: John Torakis Date: Mon, 2 Nov 2020 12:31:49 +0200 Subject: [PATCH 05/36] Fix missing parameter in Filebeat Deployment template --- filebeat/templates/deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index 08c218250..cb0141fd7 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -95,6 +95,7 @@ spec: args: - "-e" - "-E" + - "http.enabled=true" livenessProbe: {{ toYaml .Values.livenessProbe | indent 10 }} readinessProbe: From fe60db0a9434323d053d06592386a5a940a5a490 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 19 Nov 2020 14:49:54 +0000 Subject: [PATCH 06/36] Update filebeat/templates/configmap.yaml Co-authored-by: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> --- filebeat/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/templates/configmap.yaml b/filebeat/templates/configmap.yaml index 008de825b..6d7f34092 100644 --- a/filebeat/templates/configmap.yaml +++ b/filebeat/templates/configmap.yaml @@ -16,7 +16,7 @@ data: {{- end -}} {{- end -}} -{{- if .Values.daemonset.filebeatConfig }} +{{- if and .Values.daemonset.enabled .Values.daemonset.filebeatConfig }} --- apiVersion: v1 kind: ConfigMap From f4f07347647457c53c037c8e9990fb75bcafbf0c Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 19 Nov 2020 14:50:17 +0000 Subject: [PATCH 07/36] Update filebeat/templates/configmap.yaml Co-authored-by: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> --- filebeat/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/templates/configmap.yaml b/filebeat/templates/configmap.yaml index 6d7f34092..559abe1ed 100644 --- a/filebeat/templates/configmap.yaml +++ b/filebeat/templates/configmap.yaml @@ -34,7 +34,7 @@ data: {{- end -}} {{- end -}} -{{- if .Values.deployment.filebeatConfig }} +{{- if and .Values.deployment.enabled .Values.deployment.filebeatConfig }} --- apiVersion: v1 kind: ConfigMap From 8e3a2a530a94963033b2c9a98632ce3ef27653cb Mon Sep 17 00:00:00 2001 From: John Torakis Date: Tue, 24 Nov 2020 08:51:14 +0000 Subject: [PATCH 08/36] Update filebeat/templates/daemonset.yaml Co-authored-by: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> --- filebeat/templates/daemonset.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index bd1b58432..4c4fd9efc 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -39,9 +39,7 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} spec: - {{- with ( .Values.tolerations | default .Values.daemonset.tolerations ) }} - tolerations: {{ toYaml . | nindent 8 }} - {{- end }} + tolerations: {{ toYaml ( .Values.tolerations | default .Values.daemonset.tolerations ) | nindent 8 }} nodeSelector: {{ toYaml ( .Values.nodeSelector | default .Values.daemonset.nodeSelector ) | nindent 8 }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} From 717cbe5c2909c03e1f9938ee47100c71e6fba058 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 26 Nov 2020 17:04:37 +0200 Subject: [PATCH 09/36] Resolving comments for 8e3a2a530a94963033b2c9a98632ce3ef27653cb --- filebeat/templates/daemonset.yaml | 8 +++++- filebeat/templates/deployment.yaml | 18 ++++++------ filebeat/values.yaml | 46 ++++++++++++++---------------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 4c4fd9efc..9396a3c8a 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -12,6 +12,12 @@ metadata: {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- if .Values.deployment.annotations }} + annotations: # comment 2 + {{- range $key, $value := .Values.deployment.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} spec: selector: matchLabels: @@ -26,7 +32,7 @@ spec: {{ $key }}: {{ $value | quote }} {{- end }} {{/* This forces a restart if the configmap has changed */}} - {{- if .Values.filebeatConfig }} + {{- if or .Values.filebeatConfig .Values.daemonset.filebeatConfig }} configChecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} {{- end }} name: "{{ template "filebeat.fullname" . }}" diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index cb0141fd7..c02e030b7 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -4,13 +4,15 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: '{{ template "filebeat.fullname" . }}' + name: {{ template "filebeat.fullname" . }} labels: - app: '{{ template "filebeat.fullname" . }}' - chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' - heritage: '{{ .Release.Service }}' - release: '{{ .Release.Name }}' - {{- if .Values.deployment.annotations}} + app: "{{ template "filebeat.fullname" . }}" + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + {{- range $key, $value := .Values.labels }} # Comment 3 + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- if .Values.deployment.annotations }} annotations: {{- range $key, $value := .Values.deployment.annotations }} {{ $key }}: {{ $value | quote }} @@ -20,9 +22,9 @@ spec: replicas: {{ .Values.replicas }} selector: matchLabels: - app: '{{ template "filebeat.fullname" . }}' + app: "{{ template "filebeat.fullname" . }}" heritage: '{{ .Release.Service }}' - release: '{{ .Release.Name }}' + release: {{ .Release.Name | quote }} template: metadata: annotations: diff --git a/filebeat/values.yaml b/filebeat/values.yaml index e434fe89a..3ca8962c3 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -2,6 +2,8 @@ daemonset: # Annotations to apply to the daemonset annotations: {} + # additionals labels + labels: {} affinity: {} # Include the daemonset enabled: true @@ -19,7 +21,7 @@ daemonset: # - name: extras # mountPath: /usr/share/extras # readOnly: true - + hostNetworking: false # Allows you to add any config files in /usr/share/filebeat # such as filebeat.yml for daemonset filebeatConfig: @@ -59,13 +61,16 @@ daemonset: limits: cpu: "1000m" memory: "200Mi" + tolerations: [] deployment: # Annotations to apply to the deployment annotations: {} + # additionals labels + labels: {} affinity: {} # Include the deployment - enabled: true + enabled: false # Extra environment variables for Filebeat container. envFrom: [] # - configMapRef: @@ -84,17 +89,6 @@ deployment: # such as filebeat.yml for deployment filebeatConfig: filebeat.yml: | - filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" - output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' @@ -128,7 +122,6 @@ extraInitContainers: [] # Root directory where Filebeat will write data to in order to persist registry data across pod restarts (file position and other metadata). hostPathRoot: /var/lib -hostNetworking: false dnsConfig: {} # options: # - name: ndots @@ -164,9 +157,6 @@ readinessProbe: periodSeconds: 10 timeoutSeconds: 5 -# additionals labels -labels: {} - # Whether this chart should self-manage its service account, role, and associated role binding. managedServiceAccount: true @@ -194,12 +184,18 @@ updateStrategy: RollingUpdate nameOverride: "" fullnameOverride: "" -tolerations: [] - - +# DEPRECATED affinity: {} - -# Override various naming aspects of this chart -# Only edit these if you know what you're doing -nameOverride: "" -fullnameOverride: "" +envFrom: [] +extraEnvs: [] +extraVolumes: [] +extraVolumeMounts: [] +# Allows you to add any config files in /usr/share/filebeat +# such as filebeat.yml for both daemonset and deployment +filebeatConfig: {} +nodeSelector: {} +podSecurityContext: {} +resources: {} +secretMounts: [] +tolerations: [] +labels: {} From dca397405de231e21f3eb96aa99947394869850f Mon Sep 17 00:00:00 2001 From: John Torakis Date: Thu, 26 Nov 2020 19:53:59 +0200 Subject: [PATCH 10/36] Add explanation for deployment/daemonset values scheme --- filebeat/README.md | 105 ++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/filebeat/README.md b/filebeat/README.md index b6251b34e..97832778a 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -84,45 +84,72 @@ activate it by setting `hostNetworking: true` in [values.yaml][]. as a reference. They are also used in the automated testing of this chart. -## Configuration - -| Parameter | Description | Default | -|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| -| `affinity` | Configurable [affinity][] | `{}` | -| `envFrom` | Templatable string of envFrom to be passed to the [environment from variables][] which will be appended to the `envFrom:` definition for the container | `[]` | -| `extraContainers` | List of additional init containers to be added at the DaemonSet | `""` | -| `extraEnvs` | Extra [environment variables][] which will be appended to the `env:` definition for the container | `[]` | -| `extraInitContainers` | List of additional init containers to be added at the DaemonSet. It also accepts a templatable string of additional containers to be passed to the `tpl` function | `[]` | -| `extraVolumeMounts` | List of additional volumeMounts to be mounted on the DaemonSet | `[]` | -| `extraVolumes` | List of additional volumes to be mounted on the DaemonSet | `[]` | -| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` | see [values.yaml][] | -| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | -| `hostNetworking` | Use host networking in the DaemonSet so that hostname is reported correctly | `false` | -| `dnsConfig` | Configurable [dnsConfig][] | `{}` | -| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist Filebeat registry data | `/var/lib` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | -| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The Filebeat Docker image tag | `8.0.0-SNAPSHOT` | -| `image` | The Filebeat Docker image | `docker.elastic.co/beats/filebeat` | -| `deploymentType` | Whether Filebeat will be deployed as `DaemonSet` running on all Worker nodes (default) or `Deployment` | `daemonset` | - -| `labels` | Configurable [labels][] applied to all Filebeat pods | `{}` | -| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this Helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | -| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | -| `nodeSelector` | Configurable [nodeSelector][] | `{}` | -| `podAnnotations` | Configurable [annotations][] applied to all Filebeat pods | `{}` | -| `podSecurityContext` | Configurable [podSecurityContext][] for Filebeat pod execution environment | see [values.yaml][] | -| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | -| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `resources` | Allows you to set the [resources][] for the `DaemonSet` | see [values.yaml][] | -| `secretMounts` | Allows you easily mount a secret as a file inside the `DaemonSet`. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `serviceAccount` | Custom [serviceAccount][] that Filebeat will use during execution. By default will use the service account created by this chart | `""` | -| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing Filebeat pod process on pod shutdown | `30` | -| `tolerations` | Configurable [tolerations][] | `[]` | -| `updateStrategy` | The [updateStrategy][] for the `DaemonSet`. By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | - +| Parameter | Description | Default | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------| +| `daemonset.annotations` | Configurable [annotations][] for filebeat daemonset | `{}` | +| `daemonset.labels` | Configurable [labels][] applied to all filebeat DaemonSet pods | `{}` | +| `daemonset.affinity` | Configurable [affinity][] for filebeat daemonset | `{}` | +| `daemonset.enabled` | If true, enable daemonset | `true` | +| `daemonset.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for DaemonSet | `[]` | +| `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for DaemonSet | `[]` | +| `daemonset.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | +| `daemonset.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function or DaemonSet | `[]` | +| `daemonset.hostNetworking` | Enable filebeat DaemonSet to use `hostNetwork` | `false` | +| `daemonset.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat DaemonSet | see [values.yaml][] | +| `daemonset.nodeSelector` | Configurable [nodeSelector][] for filebeat DaemonSet | `{}` | +| `daemonset.secretMounts` | Allows you easily mount a secret as a file inside the DaemonSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `daemonset.podSecurityContext` | Configurable [podSecurityContext][] for filebeat DaemonSet pod execution environment | see [values.yaml][] | +| `daemonset.resources` | Allows you to set the [resources][] for filebeat DaemonSet | see [values.yaml][] | +| `daemonset.tolerations` | Configurable [tolerations][] for filebeat DaemonSet | `[]` | +| `deployment.annotations` | Configurable [annotations][] for filebeat Deployment | `{}` | +| `deployment.labels` | Configurable [labels][] applied to all filebeat Deployment pods | `{}` | +| `deployment.affinity` | Configurable [affinity][] for filebeat Deployment | `{}` | +| `deployment.enabled` | If true, enable deployment | `false` | +| `deployment.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for Deployment | `[]` | +| `deployment.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for Deployment | `[]` | +| `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | +| `deployment.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function or Deployment | `[]` | +| `deployment.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat Deployment | see [values.yaml][] | +| `deployment.nodeSelector` | Configurable [nodeSelector][] for filebeat Deployment | `{}` | +| `deployment.secretMounts` | Allows you easily mount a secret as a file inside the Deployment Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `deployment.resources` | Allows you to set the [resources][] for filebeat Deployment | see [values.yaml][] | +| `deployment.securityContext` | Configurable [securityContext][] for filebeat Deployment pod execution environment | see [values.yaml][] | +| `deployment.tolerations` | Configurable [tolerations][] for filebeat Deployment | `[]` | +| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | +| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist filebeat registry data | `/var/lib` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | +| `imageTag` | The filebeat Docker image tag | `8.0.0-SNAPSHOT` | +| `image` | The filebeat Docker image | `docker.elastic.co/beats/filebeat` | +| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | +| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | +| `podAnnotations` | Configurable [annotations][] applied to all filebeat pods | `{}` | +| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | +| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `serviceAccount` | Custom [serviceAccount][] that filebeat will use during execution. By default will use the service account created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing filebeat pod process on pod shutdown | `30` | +| `updateStrategy` | The [updateStrategy][] for the DaemonSet By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | + +### Deprecated + +| Parameter | Description | Default | +|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| `affinity` | Configurable [affinity][] for filebeat DaemonSet | `{}` | +| `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to filebeat container for both DaemonSet and Deployment | `[]` | +| `extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for both DaemonSet and Deployment | `[]` | +| `extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | +| `extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | +| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for both filebeat DaemonSet and Deployment | `{}` | +| `nodeSelector` | Configurable [nodeSelector][] for filebeat DaemonSet | `{}` | +| `podSecurityContext` | Configurable [securityContext][] for filebeat DaemonSet and Deployment pod execution environment | `{}` | +| `resources` | Allows you to set the [resources][] for both filebeat DaemonSet and Deployment | `{}` | +| `secretMounts` | Allows you easily mount a secret as a file inside DaemonSet and Deployment Useful for mounting certificates and other secrets | `[]` | +| `tolerations` | Configurable [tolerations][] for both filebeat DaemonSet and Deployment | `[]` | +| `labels` | Configurable [labels][] applied to all filebeat pods ## FAQ From 332188a476b18ec2f0aee4ba82ae37702e59a250 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 2 Dec 2020 19:26:32 +0200 Subject: [PATCH 11/36] Set toleration defaults to empty list --- filebeat/tests/filebeat_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 7c8dc0ad0..7ad39304a 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -29,8 +29,10 @@ def test_defaults(): # Empty customizable defaults assert "imagePullSecrets" not in r["daemonset"][name]["spec"]["template"]["spec"] - assert "tolerations" not in r["daemonset"][name]["spec"]["template"]["spec"] - + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] + == [] + ) assert r["daemonset"][name]["spec"]["updateStrategy"]["type"] == "RollingUpdate" assert ( From 0935040c2b337a73083282f21db096e4f4d2bdb1 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sat, 5 Dec 2020 21:37:56 +0200 Subject: [PATCH 12/36] Added replicas Value for deployment --- filebeat/README.md | 1 + filebeat/values.yaml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/filebeat/README.md b/filebeat/README.md index 97832778a..e65594d14 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -115,6 +115,7 @@ as a reference. They are also used in the automated testing of this chart. | `deployment.resources` | Allows you to set the [resources][] for filebeat Deployment | see [values.yaml][] | | `deployment.securityContext` | Configurable [securityContext][] for filebeat Deployment pod execution environment | see [values.yaml][] | | `deployment.tolerations` | Configurable [tolerations][] for filebeat Deployment | `[]` | +| `replicas` | The replica count for the Filebeat deployment | `1` | | `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | | `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | | `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 3ca8962c3..5d74d3cc3 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -111,6 +111,9 @@ deployment: memory: "200Mi" tolerations: [] +# Replicas being used for the filebeat deployment +replicas: 1 + extraContainers: "" # - name: dummy-init # image: busybox From a8af38bc521229e5e15e44c15881216590f00411 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 01:15:52 +0200 Subject: [PATCH 13/36] Fixed all suggestions --- filebeat/templates/daemonset.yaml | 27 +++++++++++++++++++++++---- filebeat/templates/deployment.yaml | 10 ++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 9396a3c8a..8f97c828b 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -9,12 +9,18 @@ metadata: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- if .Values.daemonset.labels }} + {{- range $key, $value := .Values.daemonset.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} - {{- if .Values.deployment.annotations }} - annotations: # comment 2 - {{- range $key, $value := .Values.deployment.annotations }} + {{- end }} + {{- if .Values.daemonset.annotations }} + annotations: + {{- range $key, $value := .Values.daemonset.annotations }} {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} @@ -41,9 +47,15 @@ spec: chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" heritage: {{ .Release.Service | quote }} release: {{ .Release.Name | quote }} + {{- if .Values.daemonset.labels }} + {{- range $key, $value := .Values.daemonset.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} spec: tolerations: {{ toYaml ( .Values.tolerations | default .Values.daemonset.tolerations ) | nindent 8 }} nodeSelector: {{ toYaml ( .Values.nodeSelector | default .Values.daemonset.nodeSelector ) | nindent 8 }} @@ -53,7 +65,7 @@ spec: affinity: {{ toYaml ( .Values.affinity | default .Values.daemonset.affinity ) | nindent 8 }} serviceAccountName: {{ template "filebeat.serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} - {{- if .Values.hostNetworking }} + {{- if .Values.daemonset.hostNetworking }} hostNetwork: true dnsPolicy: ClusterFirstWithHostNet {{- end }} @@ -152,6 +164,13 @@ spec: mountPath: /usr/share/filebeat/{{ $path }} readOnly: true subPath: {{ $path }} + {{ else }} + {{- range $path, $config := .Values.daemonset.filebeatConfig }} + - name: filebeat-config + mountPath: /usr/share/filebeat/{{ $path }} + readOnly: true + subPath: {{ $path }} + {{- end }} {{- end }} - name: data mountPath: /usr/share/filebeat/data diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index c02e030b7..eb8503eb9 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -8,10 +8,17 @@ metadata: labels: app: "{{ template "filebeat.fullname" . }}" chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: '{{ .Release.Service }}' release: {{ .Release.Name }} - {{- range $key, $value := .Values.labels }} # Comment 3 + {{- if .Values.deployment.labels }} + {{- range $key, $value := .Values.deployment.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- else }} + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} {{- if .Values.deployment.annotations }} annotations: {{- range $key, $value := .Values.deployment.annotations }} @@ -23,7 +30,6 @@ spec: selector: matchLabels: app: "{{ template "filebeat.fullname" . }}" - heritage: '{{ .Release.Service }}' release: {{ .Release.Name | quote }} template: metadata: From f4346a657db41b17a5bdb3dd807ff2e609b018f5 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 01:16:15 +0200 Subject: [PATCH 14/36] updated daemonset hostNetworking --- filebeat/tests/filebeat_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 7ad39304a..db247ebbd 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -6,6 +6,7 @@ project = "filebeat" name = "release-name-" + project +name = "RELEASE-NAME-" + project def test_defaults(): @@ -147,7 +148,8 @@ def test_override_the_default_update_strategy(): def test_host_networking(): config = """ -hostNetworking: true +daemonset: + hostNetworking: true """ r = helm_template(config) assert r["daemonset"][name]["spec"]["template"]["spec"]["hostNetwork"] is True From f6eb5f93d7a96ec274eef527dd716441cc90ad01 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 12:33:18 +0200 Subject: [PATCH 15/36] Add Labels in deployment pods --- filebeat/templates/deployment.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index eb8503eb9..9fcba3465 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -44,11 +44,16 @@ spec: labels: app: '{{ template "filebeat.fullname" . }}' chart: '{{ .Chart.Name }}-{{ .Chart.Version }}' - heritage: '{{ .Release.Service }}' release: '{{ .Release.Name }}' + {{- if .Values.deployment.labels }} + {{- range $key, $value := .Values.deployment.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- else }} {{- range $key, $value := .Values.labels }} {{ $key }}: {{ $value | quote }} {{- end }} + {{- end }} spec: affinity: {{ toYaml .Values.deployment.affinity | nindent 8 }} nodeSelector: {{ toYaml .Values.deployment.nodeSelector | nindent 8 }} From 8404054cdd93ddb508d1510517f8247cde4adfe3 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 12:35:51 +0200 Subject: [PATCH 16/36] rename mountpoint of daemonset data --- filebeat/templates/daemonset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 8f97c828b..9697d9b9e 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -91,7 +91,7 @@ spec: {{- end }} - name: data hostPath: - path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-{{ .Release.Namespace }}-data + path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-data type: DirectoryOrCreate - name: varlibdockercontainers hostPath: From 3448e073cd413cb915767bd89fd50b59f016a867 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 12:39:43 +0200 Subject: [PATCH 17/36] Made deployment non-root, enabled by default to pass tests --- filebeat/values.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 5d74d3cc3..8649bb571 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -51,7 +51,7 @@ daemonset: # # - User that the container will execute as. Typically necessary to run as root (0) in order to properly collect host container logs. # - Whether to execute the Filebeat containers as privileged containers. Typically not necessarily unless running within environments such as OpenShift. - podSecurityContext: + securityContext: runAsUser: 0 privileged: false resources: @@ -70,7 +70,7 @@ deployment: labels: {} affinity: {} # Include the deployment - enabled: false + enabled: true # Extra environment variables for Filebeat container. envFrom: [] # - configMapRef: @@ -99,8 +99,12 @@ deployment: # - name: filebeat-certificates # secretName: filebeat-certificates # path: /usr/share/filebeat/certs + # + # - User that the container will execute as. + # Not necessary to run as root (0) as the Filebeat Deployment use cases do not need access to Kubernetes Node internals + # - Typically not necessarily unless running within environments such as OpenShift. securityContext: - runAsUser: 0 + runAsUser: 1001 privileged: false resources: requests: From 34f339df2cc8ad9ed5ca8a9ff8e30796bca396de Mon Sep 17 00:00:00 2001 From: John Torakis Date: Sun, 6 Dec 2020 12:40:22 +0200 Subject: [PATCH 18/36] Refactored tests to pass on default-ON Deployment and daemonset with new values --- filebeat/tests/filebeat_test.py | 1015 ++++++++++++++++++++++++++++--- 1 file changed, 920 insertions(+), 95 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 9dbb19e43..5a1798a11 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -7,7 +7,6 @@ project = "filebeat" name = "release-name-" + project - def test_defaults(): config = """ """ @@ -15,6 +14,7 @@ def test_defaults(): r = helm_template(config) assert name in r["daemonset"] + assert name in r["deployment"] c = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0] assert c["name"] == project @@ -27,37 +27,119 @@ def test_defaults(): assert "filebeat test output" in c["readinessProbe"]["exec"]["command"][-1] - # Empty customizable defaults - assert "imagePullSecrets" not in r["daemonset"][name]["spec"]["template"]["spec"] + assert r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] == [] + + assert "hostNetwork" not in r["daemonset"][name]["spec"]["template"]["spec"] + assert "dnsPolicy" not in r["daemonset"][name]["spec"]["template"]["spec"] + assert ( + "hostNetwork" + not in r["deployment"][name]["spec"]["template"]["spec"] + ) assert ( - r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] + "dnsPolicy" + not in r["deployment"][name]["spec"]["template"]["spec"] + ) + + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] ) + + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 0 + ) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 1001 + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + + # Empty customizable defaults + assert "imagePullSecrets" not in r["daemonset"][name]["spec"]["template"]["spec"] + assert r["daemonset"][name]["spec"]["updateStrategy"]["type"] == "RollingUpdate" assert ( r["daemonset"][name]["spec"]["template"]["spec"]["serviceAccountName"] == name ) - volumes = r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] + cfg = r["configmap"] + + assert name + "-config" not in cfg + assert name + "-daemonset-config" in cfg + assert name + "-deployment-config" in cfg + + assert "filebeat.yml" in cfg[name + "-daemonset-config"]["data"] + assert "filebeat.yml" in cfg[name + "-deployment-config"]["data"] + + daemonset = r["daemonset"][name]["spec"]["template"]["spec"] + + assert { + "configMap": {"name": name + "-config", "defaultMode": 0o600}, + "name": project + "-config", + } not in daemonset["volumes"] + assert { + "configMap": {"name": name + "-daemonset-config", "defaultMode": 0o600}, + "name": project + "-config", + } in daemonset["volumes"] + assert { "name": "data", "hostPath": { - "path": "/var/lib/" + name + "-default-data", + "path": "/var/lib/" + name + "-data", "type": "DirectoryOrCreate", }, - } in volumes + } in daemonset["volumes"] + assert { + "mountPath": "/usr/share/filebeat/filebeat.yml", + "name": project + "-config", + "subPath": "filebeat.yml", + "readOnly": True, + } in daemonset["containers"][0]["volumeMounts"] -def test_adding_envs(): - config = """ -extraEnvs: -- name: LOG_LEVEL - value: DEBUG -""" - r = helm_template(config) - envs = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["env"] - assert {"name": "LOG_LEVEL", "value": "DEBUG"} in envs + deployment = r["deployment"][name]["spec"]["template"]["spec"] + + assert { + "configMap": {"name": name + "-config", "defaultMode": 0o600}, + "name": project + "-config", + } not in deployment["volumes"] + assert { + "configMap": {"name": name + "-deployment-config", "defaultMode": 0o600}, + "name": project + "-config", + } in deployment["volumes"] + + assert { + "mountPath": "/usr/share/filebeat/filebeat.yml", + "name": project + "-config", + "subPath": "filebeat.yml", + "readOnly": True, + } in deployment["containers"][0]["volumeMounts"] + + assert daemonset["containers"][0]["resources"] == { + "requests": {"cpu": "100m", "memory": "100Mi"}, + "limits": {"cpu": "1000m", "memory": "200Mi"}, + } + assert deployment["containers"][0]["resources"] == { + "requests": {"cpu": "100m", "memory": "100Mi"}, + "limits": {"cpu": "1000m", "memory": "200Mi"}, + } def test_adding_a_extra_container(): @@ -68,12 +150,23 @@ def test_adding_a_extra_container(): command: ['do', 'something'] """ r = helm_template(config) - extraContainer = r["daemonset"][name]["spec"]["template"]["spec"]["containers"] + extraContainerDaemonset = r["daemonset"][name]["spec"]["template"]["spec"][ + "containers" + ] assert { "name": "do-something", "image": "busybox", "command": ["do", "something"], - } in extraContainer + } in extraContainerDaemonset + deployment_name = name + extraContainerDeployment = r["deployment"][deployment_name]["spec"]["template"][ + "spec" + ]["containers"] + assert { + "name": "do-something", + "image": "busybox", + "command": ["do", "something"], + } in extraContainerDeployment def test_adding_init_containers_as_yaml(): @@ -92,20 +185,76 @@ def test_adding_init_containers_as_yaml(): } in initContainers -def test_adding_init_containers(): +def test_adding_a_extra_init_container(): config = """ extraInitContainers: | - - name: dummy-init + - name: do-something image: busybox - command: ['echo', 'hey'] + command: ['do', 'something'] """ r = helm_template(config) - initContainers = r["daemonset"][name]["spec"]["template"]["spec"]["initContainers"] + extraInitContainerDaemonset = r["daemonset"][name]["spec"]["template"]["spec"][ + "initContainers" + ] assert { - "name": "dummy-init", + "name": "do-something", "image": "busybox", - "command": ["echo", "hey"], - } in initContainers + "command": ["do", "something"], + } in extraInitContainerDaemonset + deployment_name = name + extraInitContainerDeployment = r["deployment"][deployment_name]["spec"]["template"][ + "spec" + ]["initContainers"] + assert { + "name": "do-something", + "image": "busybox", + "command": ["do", "something"], + } in extraInitContainerDeployment + + +def test_adding_envs(): + config = """ +daemonset: + extraEnvs: + - name: LOG_LEVEL + value: DEBUG +""" + r = helm_template(config) + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} not in r["deployment"][ + name + ]["spec"]["template"]["spec"]["containers"][0]["env"] + + config = """ +deployment: + extraEnvs: + - name: LOG_LEVEL + value: DEBUG +""" + r = helm_template(config) + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][ + name + ]["spec"]["template"]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} not in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] + + +def test_adding_deprecated_envs(): + config = """ +extraEnvs: +- name: LOG_LEVEL + value: DEBUG +""" + r = helm_template(config) + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][ + name + ]["spec"]["template"]["spec"]["containers"][0]["env"] def test_adding_image_pull_secrets(): @@ -120,8 +269,68 @@ def test_adding_image_pull_secrets(): ) +def test_adding_host_networking(): + config = """ +daemonset: + hostNetworking: true +""" + r = helm_template(config) + assert r["daemonset"][name]["spec"]["template"]["spec"]["hostNetwork"] is True + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["dnsPolicy"] + == "ClusterFirstWithHostNet" + ) + assert ( + "hostNetwork" + not in r["deployment"][name]["spec"]["template"]["spec"] + ) + assert ( + "dnsPolicy" + not in r["deployment"][name]["spec"]["template"]["spec"] + ) + + def test_adding_tolerations(): config = """ +daemonset: + tolerations: + - key: "key1" + operator: "Equal" + value: "value1" + effect: "NoExecute" + tolerationSeconds: 3600 +""" + r = helm_template(config) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"][0]["key"] + == "key1" + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] + == [] + ) + + config = """ +deployment: + tolerations: + - key: "key1" + operator: "Equal" + value: "value1" + effect: "NoExecute" + tolerationSeconds: 3600 +""" + r = helm_template(config) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][ + 0 + ]["key"] + == "key1" + ) + assert r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] == [] + + +def test_adding_deprecated_tolerations(): + config = """ tolerations: - key: "key1" operator: "Equal" @@ -134,6 +343,12 @@ def test_adding_tolerations(): r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"][0]["key"] == "key1" ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][ + 0 + ]["key"] + == "key1" + ) def test_override_the_default_update_strategy(): @@ -145,20 +360,6 @@ def test_override_the_default_update_strategy(): assert r["daemonset"][name]["spec"]["updateStrategy"]["type"] == "OnDelete" -def test_host_networking(): - config = """ -daemonset: - hostNetworking: true -""" - r = helm_template(config) - assert r["daemonset"][name]["spec"]["template"]["spec"]["hostNetwork"] is True - config = """ -hostNetworking: false -""" - r = helm_template(config) - assert "hostNetwork" not in r["daemonset"][name]["spec"]["template"]["spec"] - - def test_setting_a_custom_service_account(): config = """ serviceAccount: notdefault @@ -182,84 +383,384 @@ def test_self_managing_rbac_resources(): def test_setting_pod_security_context(): config = """ +daemonset: + securityContext: + runAsUser: 1001 + privileged: false +""" + r = helm_template(config) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 1001 + ) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 1001 + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + + config = """ +deployment: + securityContext: + runAsUser: 0 + privileged: false +""" + r = helm_template(config) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 0 + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 0 + ) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + + +def test_setting_deprecated_pod_security_context(): + config = """ podSecurityContext: runAsUser: 1001 privileged: false """ r = helm_template(config) - c = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0] - assert c["securityContext"]["runAsUser"] == 1001 - assert c["securityContext"]["privileged"] == False + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 1001 + ) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 1001 + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) def test_adding_in_filebeat_config(): config = """ +daemonset: + filebeatConfig: + filebeat.yml: | + key: daemonset + daemonset-config.yml: | + hello = daemonset + +deployment: + filebeatConfig: + filebeat.yml: | + key: deployment + deployment-config.yml: | + hello = deployment +""" + r = helm_template(config) + cfg = r["configmap"] + + assert "filebeat.yml" in cfg[name + "-daemonset-config"]["data"] + assert "daemonset-config.yml" in cfg[name + "-daemonset-config"]["data"] + assert "deployment-config.yml" not in cfg[name + "-daemonset-config"]["data"] + assert "filebeat.yml" in cfg[name + "-deployment-config"]["data"] + assert "deployment-config.yml" in cfg[name + "-deployment-config"]["data"] + assert "daemonset-config.yml" not in cfg[name + "-deployment-config"]["data"] + + assert "key: daemonset" in cfg[name + "-daemonset-config"]["data"]["filebeat.yml"] + assert ( + "key: deployment" in cfg[name + "-deployment-config"]["data"]["filebeat.yml"] + ) + + assert ( + "hello = daemonset" + in cfg[name + "-daemonset-config"]["data"]["daemonset-config.yml"] + ) + assert ( + "hello = deployment" + in cfg[name + "-deployment-config"]["data"]["deployment-config.yml"] + ) + + daemonset = r["daemonset"][name]["spec"]["template"]["spec"] + assert { + "mountPath": "/usr/share/filebeat/daemonset-config.yml", + "name": project + "-config", + "subPath": "daemonset-config.yml", + "readOnly": True, + } in daemonset["containers"][0]["volumeMounts"] + + deployment = r["deployment"][name]["spec"]["template"]["spec"] + assert { + "mountPath": "/usr/share/filebeat/deployment-config.yml", + "name": project + "-config", + "subPath": "deployment-config.yml", + "readOnly": True, + } in deployment["containers"][0]["volumeMounts"] + + +def test_adding_in_deprecated_filebeat_config(): + config = """ filebeatConfig: filebeat.yml: | key: nestedkey: value dot.notation: test - - other-config.yml: | - hello = world """ r = helm_template(config) c = r["configmap"][name + "-config"]["data"] assert "filebeat.yml" in c - assert "other-config.yml" in c assert "nestedkey: value" in c["filebeat.yml"] assert "dot.notation: test" in c["filebeat.yml"] - assert "hello = world" in c["other-config.yml"] - d = r["daemonset"][name]["spec"]["template"]["spec"] + daemonset = r["daemonset"][name]["spec"]["template"]["spec"] assert { "configMap": {"name": name + "-config", "defaultMode": 0o600}, "name": project + "-config", - } in d["volumes"] + } in daemonset["volumes"] assert { "mountPath": "/usr/share/filebeat/filebeat.yml", "name": project + "-config", "subPath": "filebeat.yml", "readOnly": True, - } in d["containers"][0]["volumeMounts"] + } in daemonset["containers"][0]["volumeMounts"] + + assert ( + "configChecksum" + in r["daemonset"][name]["spec"]["template"]["metadata"]["annotations"] + ) + + deployment = r["deployment"][name]["spec"]["template"]["spec"] + + assert { + "configMap": {"name": name + "-config", "defaultMode": 0o600}, + "name": project + "-config", + } in deployment["volumes"] assert { - "mountPath": "/usr/share/filebeat/other-config.yml", + "mountPath": "/usr/share/filebeat/filebeat.yml", "name": project + "-config", - "subPath": "other-config.yml", + "subPath": "filebeat.yml", "readOnly": True, - } in d["containers"][0]["volumeMounts"] + } in deployment["containers"][0]["volumeMounts"] assert ( "configChecksum" - in r["daemonset"][name]["spec"]["template"]["metadata"]["annotations"] + in r["deployment"][name]["spec"]["template"]["metadata"][ + "annotations" + ] ) def test_adding_a_secret_mount(): config = """ +daemonset: + secretMounts: + - name: elastic-certificates + secretName: elastic-certificates-name + path: /usr/share/filebeat/config/certs +""" + r = helm_template(config) + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) + assert { + "name": "elastic-certificates", + "secret": {"secretName": "elastic-certificates-name"}, + } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] + + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["deployment"][name]["spec"]["template"]["spec"][ + "containers" + ][0]["volumeMounts"] + ) + assert { + "name": "elastic-certificates", + "secret": {"secretName": "elastic-certificates-name"}, + } not in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] + + config = """ +deployment: + secretMounts: + - name: elastic-certificates + secretName: elastic-certificates-name + path: /usr/share/filebeat/config/certs +""" + r = helm_template(config) + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["volumeMounts"] + ) + assert { + "name": "elastic-certificates", + "secret": {"secretName": "elastic-certificates-name"}, + } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] + + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) + assert { + "name": "elastic-certificates", + "secret": {"secretName": "elastic-certificates-name"}, + } not in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] + + +def test_adding_a_deprecated_secret_mount(): + config = """ secretMounts: - name: elastic-certificates - secretName: elastic-certs + secretName: elastic-certificates-name path: /usr/share/filebeat/config/certs """ r = helm_template(config) - s = r["daemonset"][name]["spec"]["template"]["spec"] - assert s["containers"][0]["volumeMounts"][0] == { + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) + assert { + "name": "elastic-certificates", + "secret": {"secretName": "elastic-certificates-name"}, + } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] + + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["volumeMounts"][0] == { "mountPath": "/usr/share/filebeat/config/certs", "name": "elastic-certificates", } - assert s["volumes"][0] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["volumes"][ + 0 + ] == { "name": "elastic-certificates", - "secret": {"secretName": "elastic-certs"}, + "secret": {"secretName": "elastic-certificates-name"}, } def test_adding_a_extra_volume_with_volume_mount(): config = """ +daemonset: + extraVolumes: + - name: extras + emptyDir: {} + extraVolumeMounts: + - name: extras + mountPath: /usr/share/extras + readOnly: true +""" + r = helm_template(config) + assert {"name": "extras", "emptyDir": {}} in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["volumes"] + assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ + "daemonset" + ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] + assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name][ + "spec" + ]["template"]["spec"]["volumes"] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["deployment"][name]["spec"]["template"]["spec"][ + "containers" + ][0]["volumeMounts"] + ) + + config = """ +deployment: + extraVolumes: + - name: extras + emptyDir: {} + extraVolumeMounts: + - name: extras + mountPath: /usr/share/extras + readOnly: true +""" + r = helm_template(config) + assert {"name": "extras", "emptyDir": {}} in r["deployment"][name][ + "spec" + ]["template"]["spec"]["volumes"] + assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ + "deployment" + ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] + assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["volumes"] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) + + +def test_adding_a_deprecated_extra_volume_with_volume_mount(): + config = """ extraVolumes: - name: extras emptyDir: {} @@ -269,51 +770,52 @@ def test_adding_a_extra_volume_with_volume_mount(): readOnly: true """ r = helm_template(config) - extraVolume = r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert {"name": "extras", "emptyDir": {}} in extraVolume - extraVolumeMounts = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["volumeMounts"] - assert { - "name": "extras", - "mountPath": "/usr/share/extras", - "readOnly": True, - } in extraVolumeMounts + assert {"name": "extras", "emptyDir": {}} in r["daemonset"][name]["spec"][ + "template" + ]["spec"]["volumes"] + assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ + "daemonset" + ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] + assert {"name": "extras", "emptyDir": {}} in r["deployment"][name][ + "spec" + ]["template"]["spec"]["volumes"] + assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ + "deployment" + ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] -def test_adding_pod_labels(): +def test_adding_a_node_selector(): config = """ -labels: - app.kubernetes.io/name: filebeat +daemonset: + nodeSelector: + disktype: ssd """ r = helm_template(config) assert ( - r["daemonset"][name]["metadata"]["labels"]["app.kubernetes.io/name"] - == "filebeat" + r["daemonset"][name]["spec"]["template"]["spec"]["nodeSelector"]["disktype"] + == "ssd" ) assert ( - r["daemonset"][name]["spec"]["template"]["metadata"]["labels"][ - "app.kubernetes.io/name" - ] - == "filebeat" + r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"] + == {} ) - -def test_adding_serviceaccount_annotations(): config = """ -serviceAccountAnnotations: - eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +deployment: + nodeSelector: + disktype: ssd """ r = helm_template(config) assert ( - r["serviceaccount"][name]["metadata"]["annotations"][ - "eks.amazonaws.com/role-arn" + r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"][ + "disktype" ] - == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + == "ssd" ) + assert r["daemonset"][name]["spec"]["template"]["spec"]["nodeSelector"] == {} -def test_adding_a_node_selector(): +def test_adding_deprecated_node_selector(): config = """ nodeSelector: disktype: ssd @@ -346,6 +848,53 @@ def test_adding_an_affinity_rule(): ][0]["topologyKey"] == "kubernetes.io/hostname" ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} + ) + + config = """ +daemonset: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filebeat + topologyKey: kubernetes.io/hostname +""" + + r = helm_template(config) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["affinity"]["podAntiAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ][0]["topologyKey"] + == "kubernetes.io/hostname" + ) + + config = """ +deployment: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filebeat + topologyKey: kubernetes.io/hostname +""" + + r = helm_template(config) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["affinity"][ + "podAntiAffinity" + ]["requiredDuringSchedulingIgnoredDuringExecution"][0]["topologyKey"] + == "kubernetes.io/hostname" + ) def test_priority_class_name(): @@ -353,30 +902,256 @@ def test_priority_class_name(): priorityClassName: "" """ r = helm_template(config) - spec = r["daemonset"][name]["spec"]["template"]["spec"] - assert "priorityClassName" not in spec + daemonset_spec = r["daemonset"][name]["spec"]["template"]["spec"] + deployment_spec = r["deployment"][name]["spec"]["template"]["spec"] + assert "priorityClassName" not in daemonset_spec + assert "priorityClassName" not in deployment_spec config = """ priorityClassName: "highest" """ r = helm_template(config) - priority_class_name = r["daemonset"][name]["spec"]["template"]["spec"][ + daemonset_priority_class_name = r["daemonset"][name]["spec"]["template"]["spec"][ "priorityClassName" ] - assert priority_class_name == "highest" + deployment_priority_class_name = r["deployment"][name]["spec"][ + "template" + ]["spec"]["priorityClassName"] + assert daemonset_priority_class_name == "highest" + assert deployment_priority_class_name == "highest" + + +def test_adding_deprecated_labels(): + config = """ +labels: + app-test: filebeat +""" + r = helm_template(config) + assert r["daemonset"][name]["metadata"]["labels"]["app-test"] == "filebeat" + assert ( + r["deployment"][name]["metadata"]["labels"]["app-test"] + == "filebeat" + ) + assert ( + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "filebeat" + ) + assert ( + r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ + "app-test" + ] + == "filebeat" + ) + + +def test_adding_daemonset_labels(): + config = """ +daemonset: + labels: + app-test: filebeat +""" + r = helm_template(config) + assert r["daemonset"][name]["metadata"]["labels"]["app-test"] == "filebeat" + assert ( + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "filebeat" + ) + + +def test_adding_daemonset_labels_surpasses_root_labels(): + config = """ +labels: + app-test: root-filebeat +daemonset: + labels: + app-test: daemonset-filebeat +""" + r = helm_template(config) + assert ( + r["daemonset"][name]["metadata"]["labels"]["app-test"] == "daemonset-filebeat" + ) + assert ( + r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] + == "daemonset-filebeat" + ) + + +def test_adding_deployment_labels(): + config = """ +deployment: + labels: + app-test: filebeat +""" + r = helm_template(config) + assert ( + r["deployment"][name]["metadata"]["labels"]["app-test"] + == "filebeat" + ) + assert ( + r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ + "app-test" + ] + == "filebeat" + ) + + +def test_adding_deployment_labels_surpasses_root_labels(): + config = """ +labels: + app-test: root-filebeat +deployment: + labels: + app-test: deployment-filebeat +""" + r = helm_template(config) + assert ( + r["deployment"][name]["metadata"]["labels"]["app-test"] + == "deployment-filebeat" + ) + assert ( + r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ + "app-test" + ] + == "deployment-filebeat" + ) + + +def test_adding_serviceaccount_annotations(): + config = """ +serviceAccountAnnotations: + eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +""" + r = helm_template(config) + assert ( + r["serviceaccount"][name]["metadata"]["annotations"][ + "eks.amazonaws.com/role-arn" + ] + == "arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount" + ) def test_adding_env_from(): config = """ +daemonset: + envFrom: + - configMapRef: + name: configmap-name +""" + r = helm_template(config) + assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"][ + 0 + ]["configMapRef"] == {"name": "configmap-name"} + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "envFrom" + ] + == [] + ) + + config = """ +deployment: + envFrom: + - configMapRef: + name: configmap-name +""" + r = helm_template(config) + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["envFrom"][0]["configMapRef"] == {"name": "configmap-name"} + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"] + == [] + ) + + +def test_adding_deprecated_env_from(): + config = """ envFrom: - configMapRef: name: configmap-name """ r = helm_template(config) - configMapRef = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "envFrom" - ][0]["configMapRef"] - assert configMapRef == {"name": "configmap-name"} + assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"][ + 0 + ]["configMapRef"] == {"name": "configmap-name"} + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["envFrom"][0]["configMapRef"] == {"name": "configmap-name"} + + +def test_overriding_resources(): + config = """ +daemonset: + resources: + limits: + cpu: "25m" + memory: "128Mi" + requests: + cpu: "25m" + memory: "128Mi" +""" + r = helm_template(config) + assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { + "requests": {"cpu": "25m", "memory": "128Mi"}, + "limits": {"cpu": "25m", "memory": "128Mi"}, + } + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["resources"] == { + "requests": {"cpu": "100m", "memory": "100Mi"}, + "limits": {"cpu": "1000m", "memory": "200Mi"}, + } + + config = """ +deployment: + resources: + limits: + cpu: "25m" + memory: "128Mi" + requests: + cpu: "25m" + memory: "128Mi" +""" + r = helm_template(config) + assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { + "requests": {"cpu": "100m", "memory": "100Mi"}, + "limits": {"cpu": "1000m", "memory": "200Mi"}, + } + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["resources"] == { + "requests": {"cpu": "25m", "memory": "128Mi"}, + "limits": {"cpu": "25m", "memory": "128Mi"}, + } + + +def test_adding_deprecated_resources(): + config = """ +resources: + limits: + cpu: "25m" + memory: "128Mi" + requests: + cpu: "25m" + memory: "128Mi" +""" + r = helm_template(config) + assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { + "requests": {"cpu": "25m", "memory": "128Mi"}, + "limits": {"cpu": "25m", "memory": "128Mi"}, + } + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ + 0 + ]["resources"] == { + "requests": {"cpu": "25m", "memory": "128Mi"}, + "limits": {"cpu": "25m", "memory": "128Mi"}, + } def test_setting_fullnameOverride(): @@ -399,7 +1174,57 @@ def test_setting_fullnameOverride(): assert { "name": "data", "hostPath": { - "path": "/var/lib/" + custom_name + "-default-data", + "path": "/var/lib/" + custom_name + "-data", "type": "DirectoryOrCreate", }, } in volumes + + +def test_adding_annotations(): + config = """ +daemonset: + annotations: + foo: "bar" +""" + r = helm_template(config) + assert "foo" in r["daemonset"][name]["metadata"]["annotations"] + assert r["daemonset"][name]["metadata"]["annotations"]["foo"] == "bar" + assert "annotations" not in r["deployment"][name]["metadata"] + config = """ +deployment: + annotations: + grault: "waldo" +""" + r = helm_template(config) + assert "grault" in r["deployment"][name]["metadata"]["annotations"] + assert ( + r["deployment"][name]["metadata"]["annotations"]["grault"] + == "waldo" + ) + assert "annotations" not in r["daemonset"][name]["metadata"] + + +def test_disable_daemonset(): + config = """ +daemonset: + enabled: false +""" + r = helm_template(config) + cfg = r["configmap"] + + assert name not in r.get("daemonset", {}) + assert name + "-daemonset-config" not in cfg + assert name + "-deployment-config" in cfg + + +def test_disable_deployment(): + config = """ +deployment: + enabled: false +""" + r = helm_template(config) + cfg = r["configmap"] + + assert name + "-daemonset-config" in cfg + assert name + "-deployment-config" not in cfg + From 4c463bf90a8285c1a4faf2ecc60fdbca072fb61a Mon Sep 17 00:00:00 2001 From: John Torakis Date: Tue, 8 Dec 2020 10:25:25 +0200 Subject: [PATCH 19/36] Changed Filebeat deployment UserID to 0 from 1001 as it cannot read the conf file --- filebeat/tests/filebeat_test.py | 8 ++++---- filebeat/values.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 5a1798a11..8752e45f5 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -61,7 +61,7 @@ def test_defaults(): r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ "securityContext" ]["runAsUser"] - == 1001 + == 0 ) assert ( r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ @@ -405,7 +405,7 @@ def test_setting_pod_security_context(): r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ "securityContext" ]["runAsUser"] - == 1001 + == 0 ) assert ( r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ @@ -417,7 +417,7 @@ def test_setting_pod_security_context(): config = """ deployment: securityContext: - runAsUser: 0 + runAsUser: 1001 privileged: false """ r = helm_template(config) @@ -425,7 +425,7 @@ def test_setting_pod_security_context(): r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ "securityContext" ]["runAsUser"] - == 0 + == 1001 ) assert ( r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 8649bb571..ee3b18d1a 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -104,7 +104,7 @@ deployment: # Not necessary to run as root (0) as the Filebeat Deployment use cases do not need access to Kubernetes Node internals # - Typically not necessarily unless running within environments such as OpenShift. securityContext: - runAsUser: 1001 + runAsUser: 0 privileged: false resources: requests: From 6618cf53f7669404f780b16f733c4404fdb82dc6 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Tue, 8 Dec 2020 11:54:03 +0200 Subject: [PATCH 20/36] Fixed python linting --- filebeat/tests/filebeat_test.py | 196 ++++++++++++-------------------- 1 file changed, 71 insertions(+), 125 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 8752e45f5..27e026e8c 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -7,6 +7,7 @@ project = "filebeat" name = "release-name-" + project + def test_defaults(): config = """ """ @@ -31,19 +32,10 @@ def test_defaults(): assert "hostNetwork" not in r["daemonset"][name]["spec"]["template"]["spec"] assert "dnsPolicy" not in r["daemonset"][name]["spec"]["template"]["spec"] - assert ( - "hostNetwork" - not in r["deployment"][name]["spec"]["template"]["spec"] - ) - assert ( - "dnsPolicy" - not in r["deployment"][name]["spec"]["template"]["spec"] - ) + assert "hostNetwork" not in r["deployment"][name]["spec"]["template"]["spec"] + assert "dnsPolicy" not in r["deployment"][name]["spec"]["template"]["spec"] - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] - == [] - ) + assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] assert ( r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ @@ -223,9 +215,9 @@ def test_adding_envs(): assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["daemonset"][name]["spec"][ "template" ]["spec"]["containers"][0]["env"] - assert {"name": "LOG_LEVEL", "value": "DEBUG"} not in r["deployment"][ - name - ]["spec"]["template"]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} not in r["deployment"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] config = """ deployment: @@ -234,9 +226,9 @@ def test_adding_envs(): value: DEBUG """ r = helm_template(config) - assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][ - name - ]["spec"]["template"]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] assert {"name": "LOG_LEVEL", "value": "DEBUG"} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["containers"][0]["env"] @@ -252,9 +244,9 @@ def test_adding_deprecated_envs(): assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["daemonset"][name]["spec"][ "template" ]["spec"]["containers"][0]["env"] - assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][ - name - ]["spec"]["template"]["spec"]["containers"][0]["env"] + assert {"name": "LOG_LEVEL", "value": "DEBUG"} in r["deployment"][name]["spec"][ + "template" + ]["spec"]["containers"][0]["env"] def test_adding_image_pull_secrets(): @@ -280,14 +272,8 @@ def test_adding_host_networking(): r["daemonset"][name]["spec"]["template"]["spec"]["dnsPolicy"] == "ClusterFirstWithHostNet" ) - assert ( - "hostNetwork" - not in r["deployment"][name]["spec"]["template"]["spec"] - ) - assert ( - "dnsPolicy" - not in r["deployment"][name]["spec"]["template"]["spec"] - ) + assert "hostNetwork" not in r["deployment"][name]["spec"]["template"]["spec"] + assert "dnsPolicy" not in r["deployment"][name]["spec"]["template"]["spec"] def test_adding_tolerations(): @@ -305,10 +291,7 @@ def test_adding_tolerations(): r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"][0]["key"] == "key1" ) - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] - == [] - ) + assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] config = """ deployment: @@ -321,9 +304,7 @@ def test_adding_tolerations(): """ r = helm_template(config) assert ( - r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][ - 0 - ]["key"] + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][0]["key"] == "key1" ) assert r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] == [] @@ -344,9 +325,7 @@ def test_adding_deprecated_tolerations(): == "key1" ) assert ( - r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][ - 0 - ]["key"] + r["deployment"][name]["spec"]["template"]["spec"]["tolerations"][0]["key"] == "key1" ) @@ -507,9 +486,7 @@ def test_adding_in_filebeat_config(): assert "daemonset-config.yml" not in cfg[name + "-deployment-config"]["data"] assert "key: daemonset" in cfg[name + "-daemonset-config"]["data"]["filebeat.yml"] - assert ( - "key: deployment" in cfg[name + "-deployment-config"]["data"]["filebeat.yml"] - ) + assert "key: deployment" in cfg[name + "-deployment-config"]["data"]["filebeat.yml"] assert ( "hello = daemonset" @@ -553,7 +530,6 @@ def test_adding_in_deprecated_filebeat_config(): assert "nestedkey: value" in c["filebeat.yml"] assert "dot.notation: test" in c["filebeat.yml"] - daemonset = r["daemonset"][name]["spec"]["template"]["spec"] assert { @@ -587,9 +563,7 @@ def test_adding_in_deprecated_filebeat_config(): assert ( "configChecksum" - in r["deployment"][name]["spec"]["template"]["metadata"][ - "annotations" - ] + in r["deployment"][name]["spec"]["template"]["metadata"]["annotations"] ) @@ -621,9 +595,9 @@ def test_adding_a_secret_mount(): "mountPath": "/usr/share/filebeat/config/certs", "name": "elastic-certificates", } - not in r["deployment"][name]["spec"]["template"]["spec"][ - "containers" - ][0]["volumeMounts"] + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] ) assert { "name": "elastic-certificates", @@ -643,9 +617,9 @@ def test_adding_a_secret_mount(): "mountPath": "/usr/share/filebeat/config/certs", "name": "elastic-certificates", } - in r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["volumeMounts"] + in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] ) assert { "name": "elastic-certificates", @@ -689,15 +663,13 @@ def test_adding_a_deprecated_secret_mount(): "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["volumeMounts"][0] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ][0] == { "mountPath": "/usr/share/filebeat/config/certs", "name": "elastic-certificates", } - assert r["deployment"][name]["spec"]["template"]["spec"]["volumes"][ - 0 - ] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["volumes"][0] == { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } @@ -721,14 +693,14 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ "daemonset" ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] - assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name][ - "spec" - ]["template"]["spec"]["volumes"] + assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ + "template" + ]["spec"]["volumes"] assert ( {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["deployment"][name]["spec"]["template"]["spec"][ - "containers" - ][0]["volumeMounts"] + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] ) config = """ @@ -742,9 +714,9 @@ def test_adding_a_extra_volume_with_volume_mount(): readOnly: true """ r = helm_template(config) - assert {"name": "extras", "emptyDir": {}} in r["deployment"][name][ - "spec" - ]["template"]["spec"]["volumes"] + assert {"name": "extras", "emptyDir": {}} in r["deployment"][name]["spec"][ + "template" + ]["spec"]["volumes"] assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ "deployment" ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] @@ -776,9 +748,9 @@ def test_adding_a_deprecated_extra_volume_with_volume_mount(): assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ "daemonset" ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] - assert {"name": "extras", "emptyDir": {}} in r["deployment"][name][ - "spec" - ]["template"]["spec"]["volumes"] + assert {"name": "extras", "emptyDir": {}} in r["deployment"][name]["spec"][ + "template" + ]["spec"]["volumes"] assert {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} in r[ "deployment" ][name]["spec"]["template"]["spec"]["containers"][0]["volumeMounts"] @@ -795,10 +767,7 @@ def test_adding_a_node_selector(): r["daemonset"][name]["spec"]["template"]["spec"]["nodeSelector"]["disktype"] == "ssd" ) - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"] - == {} - ) + assert r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"] == {} config = """ deployment: @@ -807,9 +776,7 @@ def test_adding_a_node_selector(): """ r = helm_template(config) assert ( - r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"][ - "disktype" - ] + r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"]["disktype"] == "ssd" ) assert r["daemonset"][name]["spec"]["template"]["spec"]["nodeSelector"] == {} @@ -848,9 +815,7 @@ def test_adding_an_affinity_rule(): ][0]["topologyKey"] == "kubernetes.io/hostname" ) - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} - ) + assert r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} config = """ daemonset: @@ -914,9 +879,9 @@ def test_priority_class_name(): daemonset_priority_class_name = r["daemonset"][name]["spec"]["template"]["spec"][ "priorityClassName" ] - deployment_priority_class_name = r["deployment"][name]["spec"][ - "template" - ]["spec"]["priorityClassName"] + deployment_priority_class_name = r["deployment"][name]["spec"]["template"]["spec"][ + "priorityClassName" + ] assert daemonset_priority_class_name == "highest" assert deployment_priority_class_name == "highest" @@ -928,18 +893,13 @@ def test_adding_deprecated_labels(): """ r = helm_template(config) assert r["daemonset"][name]["metadata"]["labels"]["app-test"] == "filebeat" - assert ( - r["deployment"][name]["metadata"]["labels"]["app-test"] - == "filebeat" - ) + assert r["deployment"][name]["metadata"]["labels"]["app-test"] == "filebeat" assert ( r["daemonset"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] == "filebeat" ) assert ( - r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ - "app-test" - ] + r["deployment"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] == "filebeat" ) @@ -983,14 +943,9 @@ def test_adding_deployment_labels(): app-test: filebeat """ r = helm_template(config) + assert r["deployment"][name]["metadata"]["labels"]["app-test"] == "filebeat" assert ( - r["deployment"][name]["metadata"]["labels"]["app-test"] - == "filebeat" - ) - assert ( - r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ - "app-test" - ] + r["deployment"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] == "filebeat" ) @@ -1005,13 +960,10 @@ def test_adding_deployment_labels_surpasses_root_labels(): """ r = helm_template(config) assert ( - r["deployment"][name]["metadata"]["labels"]["app-test"] - == "deployment-filebeat" + r["deployment"][name]["metadata"]["labels"]["app-test"] == "deployment-filebeat" ) assert ( - r["deployment"][name]["spec"]["template"]["metadata"]["labels"][ - "app-test" - ] + r["deployment"][name]["spec"]["template"]["metadata"]["labels"]["app-test"] == "deployment-filebeat" ) @@ -1042,9 +994,7 @@ def test_adding_env_from(): 0 ]["configMapRef"] == {"name": "configmap-name"} assert ( - r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "envFrom" - ] + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"] == [] ) @@ -1055,9 +1005,9 @@ def test_adding_env_from(): name: configmap-name """ r = helm_template(config) - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["envFrom"][0]["configMapRef"] == {"name": "configmap-name"} + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "envFrom" + ][0]["configMapRef"] == {"name": "configmap-name"} assert ( r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"] == [] @@ -1074,9 +1024,9 @@ def test_adding_deprecated_env_from(): assert r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0]["envFrom"][ 0 ]["configMapRef"] == {"name": "configmap-name"} - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["envFrom"][0]["configMapRef"] == {"name": "configmap-name"} + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "envFrom" + ][0]["configMapRef"] == {"name": "configmap-name"} def test_overriding_resources(): @@ -1097,9 +1047,9 @@ def test_overriding_resources(): "requests": {"cpu": "25m", "memory": "128Mi"}, "limits": {"cpu": "25m", "memory": "128Mi"}, } - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["resources"] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { "requests": {"cpu": "100m", "memory": "100Mi"}, "limits": {"cpu": "1000m", "memory": "200Mi"}, } @@ -1121,9 +1071,9 @@ def test_overriding_resources(): "requests": {"cpu": "100m", "memory": "100Mi"}, "limits": {"cpu": "1000m", "memory": "200Mi"}, } - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["resources"] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { "requests": {"cpu": "25m", "memory": "128Mi"}, "limits": {"cpu": "25m", "memory": "128Mi"}, } @@ -1146,9 +1096,9 @@ def test_adding_deprecated_resources(): "requests": {"cpu": "25m", "memory": "128Mi"}, "limits": {"cpu": "25m", "memory": "128Mi"}, } - assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][ - 0 - ]["resources"] == { + assert r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "resources" + ] == { "requests": {"cpu": "25m", "memory": "128Mi"}, "limits": {"cpu": "25m", "memory": "128Mi"}, } @@ -1197,10 +1147,7 @@ def test_adding_annotations(): """ r = helm_template(config) assert "grault" in r["deployment"][name]["metadata"]["annotations"] - assert ( - r["deployment"][name]["metadata"]["annotations"]["grault"] - == "waldo" - ) + assert r["deployment"][name]["metadata"]["annotations"]["grault"] == "waldo" assert "annotations" not in r["daemonset"][name]["metadata"] @@ -1227,4 +1174,3 @@ def test_disable_deployment(): assert name + "-daemonset-config" in cfg assert name + "-deployment-config" not in cfg - From 6a27ef63d16774a9dfe32a319e71ef0a42963354 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Mon, 14 Dec 2020 16:25:58 +0200 Subject: [PATCH 21/36] Fix linting in python --- filebeat/tests/filebeat_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 99921bcaf..a2caf5016 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -1128,8 +1128,8 @@ def test_setting_fullnameOverride(): "type": "DirectoryOrCreate", }, } in volumes - - + + def test_adding_annotations(): config = """ daemonset: From e69c6c13fcda4e3e7f039432ffafa1b094acb4f2 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Mon, 14 Dec 2020 17:34:12 +0200 Subject: [PATCH 22/36] Added default HTTP input Deployment configuration --- filebeat/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 8c1f6088c..c7a727271 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -89,6 +89,10 @@ deployment: # such as filebeat.yml for deployment filebeatConfig: filebeat.yml: | + filebeat.inputs: + - type: http_endpoint + enabled: true + output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' From 4a204b9f78671b9f0565eb1bf6d212f0c6effc15 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 11:31:32 +0200 Subject: [PATCH 23/36] Fix OSS integration test --- filebeat/values.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/filebeat/values.yaml b/filebeat/values.yaml index c7a727271..4c650aa8f 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -90,9 +90,10 @@ deployment: filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: http_endpoint - enabled: true - + - type: tcp + max_message_size: 10MiB + host: "localhost:9000" + output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' From 51d8410ddbb641d3c5d667a5c84db0c6ca8ab11a Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 11:32:39 +0200 Subject: [PATCH 24/36] Fix Security Integration test --- filebeat/examples/security/values.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/filebeat/examples/security/values.yaml b/filebeat/examples/security/values.yaml index 5ed7c5529..e60b69530 100644 --- a/filebeat/examples/security/values.yaml +++ b/filebeat/examples/security/values.yaml @@ -1,15 +1,9 @@ filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" + - type: tcp + max_message_size: 10MiB + host: "localhost:9000" output.elasticsearch: username: '${ELASTICSEARCH_USERNAME}' From 3646043146eb27e97d0b2d8e842675bd94f02abe Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Wed, 16 Dec 2020 11:04:31 +0100 Subject: [PATCH 25/36] [filebeat] handle hostAliases for deployment This commit update deployment to handle hostAliases. Also some nit formating in the README --- filebeat/README.md | 118 +++++++++++++++-------------- filebeat/templates/daemonset.yaml | 4 +- filebeat/templates/deployment.yaml | 3 + filebeat/tests/filebeat_test.py | 20 +++++ filebeat/values.yaml | 13 +++- 5 files changed, 98 insertions(+), 60 deletions(-) diff --git a/filebeat/README.md b/filebeat/README.md index c5dd99ece..62e9405c8 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -17,6 +17,7 @@ This Helm chart is a lightweight way to configure and run our official - [Upgrading](#upgrading) - [Usage notes](#usage-notes) - [Configuration](#configuration) + - [Deprecated](#deprecated) - [FAQ](#faq) - [How to use Filebeat with Elasticsearch with security (authentication and TLS) enabled?](#how-to-use-filebeat-with-elasticsearch-with-security-authentication-and-tls-enabled) - [How to install OSS version of Filebeat?](#how-to-install-oss-version-of-filebeat) @@ -86,73 +87,76 @@ activate it by setting `hostNetworking: true` in [values.yaml][]. as a reference. They are also used in the automated testing of this chart. -| Parameter | Description | Default | -|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------| -| `daemonset.annotations` | Configurable [annotations][] for filebeat daemonset | `{}` | -| `daemonset.labels` | Configurable [labels][] applied to all filebeat DaemonSet pods | `{}` | -| `daemonset.affinity` | Configurable [affinity][] for filebeat daemonset | `{}` | -| `daemonset.enabled` | If true, enable daemonset | `true` | -| `daemonset.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for DaemonSet | `[]` | -| `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for DaemonSet | `[]` | -| `daemonset.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | -| `daemonset.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function or DaemonSet | `[]` | -| `daemonset.hostNetworking` | Enable filebeat DaemonSet to use `hostNetwork` | `false` | -| `daemonset.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat DaemonSet | see [values.yaml][] | -| `daemonset.nodeSelector` | Configurable [nodeSelector][] for filebeat DaemonSet | `{}` | -| `daemonset.secretMounts` | Allows you easily mount a secret as a file inside the DaemonSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `daemonset.podSecurityContext` | Configurable [podSecurityContext][] for filebeat DaemonSet pod execution environment | see [values.yaml][] | -| `daemonset.resources` | Allows you to set the [resources][] for filebeat DaemonSet | see [values.yaml][] | -| `daemonset.tolerations` | Configurable [tolerations][] for filebeat DaemonSet | `[]` | -| `deployment.annotations` | Configurable [annotations][] for filebeat Deployment | `{}` | -| `deployment.labels` | Configurable [labels][] applied to all filebeat Deployment pods | `{}` | -| `deployment.affinity` | Configurable [affinity][] for filebeat Deployment | `{}` | -| `deployment.enabled` | If true, enable deployment | `false` | -| `deployment.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for Deployment | `[]` | -| `deployment.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for Deployment | `[]` | -| `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function or DaemonSet | `[]` | -| `deployment.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function or Deployment | `[]` | -| `deployment.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat Deployment | see [values.yaml][] | -| `deployment.nodeSelector` | Configurable [nodeSelector][] for filebeat Deployment | `{}` | -| `deployment.secretMounts` | Allows you easily mount a secret as a file inside the Deployment Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | -| `deployment.resources` | Allows you to set the [resources][] for filebeat Deployment | see [values.yaml][] | -| `deployment.securityContext` | Configurable [securityContext][] for filebeat Deployment pod execution environment | see [values.yaml][] | -| `deployment.tolerations` | Configurable [tolerations][] for filebeat Deployment | `[]` | -| `replicas` | The replica count for the Filebeat deployment | `1` | -| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | -| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | -| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist filebeat registry data | `/var/lib` | -| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | -| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | -| `imageTag` | The filebeat Docker image tag | `8.0.0-SNAPSHOT` | -| `image` | The filebeat Docker image | `docker.elastic.co/beats/filebeat` | -| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | -| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | -| `podAnnotations` | Configurable [annotations][] applied to all filebeat pods | `{}` | -| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | -| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | -| `serviceAccount` | Custom [serviceAccount][] that filebeat will use during execution. By default will use the service account created by this chart | `""` | -| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | -| `terminationGracePeriod` | Termination period (in seconds) to wait before killing filebeat pod process on pod shutdown | `30` | -| `updateStrategy` | The [updateStrategy][] for the DaemonSet By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | +| Parameter | Description | Default | +|--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------| +| `daemonset.annotations` | Configurable [annotations][] for filebeat daemonset | `{}` | +| `daemonset.labels` | Configurable [labels][] applied to all filebeat DaemonSet pods | `{}` | +| `daemonset.affinity` | Configurable [affinity][] for filebeat daemonset | `{}` | +| `daemonset.enabled` | If true, enable daemonset | `true` | +| `daemonset.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for DaemonSet | `[]` | +| `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for DaemonSet | `[]` | +| `daemonset.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet | `[]` | +| `daemonset.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for DaemonSet | `[]` | +| `daemonset.hostAliases` | Configurable [hostAliases][] for filebeat DaemonSet | `[]` | +| `daemonset.hostNetworking` | Enable filebeat DaemonSet to use `hostNetwork` | `false` | +| `daemonset.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat DaemonSet | see [values.yaml][] | +| `daemonset.nodeSelector` | Configurable [nodeSelector][] for filebeat DaemonSet | `{}` | +| `daemonset.secretMounts` | Allows you easily mount a secret as a file inside the DaemonSet. Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `daemonset.podSecurityContext` | Configurable [podSecurityContext][] for filebeat DaemonSet pod execution environment | see [values.yaml][] | +| `daemonset.resources` | Allows you to set the [resources][] for filebeat DaemonSet | see [values.yaml][] | +| `daemonset.tolerations` | Configurable [tolerations][] for filebeat DaemonSet | `[]` | +| `deployment.annotations` | Configurable [annotations][] for filebeat Deployment | `{}` | +| `deployment.labels` | Configurable [labels][] applied to all filebeat Deployment pods | `{}` | +| `deployment.affinity` | Configurable [affinity][] for filebeat Deployment | `{}` | +| `deployment.enabled` | If true, enable deployment | `false` | +| `deployment.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for Deployment | `[]` | +| `deployment.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for Deployment | `[]` | +| `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet | `[]` | +| `deployment.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for Deployment | `[]` | +| `daemonset.hostAliases` | Configurable [hostAliases][] for filebeat Deployment | `[]` | +| `deployment.filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for filebeat Deployment | see [values.yaml][] | +| `deployment.nodeSelector` | Configurable [nodeSelector][] for filebeat Deployment | `{}` | +| `deployment.secretMounts` | Allows you easily mount a secret as a file inside the Deployment Useful for mounting certificates and other secrets. See [values.yaml][] for an example | `[]` | +| `deployment.resources` | Allows you to set the [resources][] for filebeat Deployment | see [values.yaml][] | +| `deployment.securityContext` | Configurable [securityContext][] for filebeat Deployment pod execution environment | see [values.yaml][] | +| `deployment.tolerations` | Configurable [tolerations][] for filebeat Deployment | `[]` | +| `replicas` | The replica count for the Filebeat deployment | `1` | +| `extraContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `extraInitContainers` | Templatable string of additional containers to be passed to the `tpl` function | `""` | +| `fullnameOverride` | Overrides the full name of the resources. If not set the name will default to " `.Release.Name` - `.Values.nameOverride or .Chart.Name` " | `""` | +| `hostPathRoot` | Fully-qualified [hostPath][] that will be used to persist filebeat registry data | `/var/lib` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy][] value | `IfNotPresent` | +| `imagePullSecrets` | Configuration for [imagePullSecrets][] so that you can use a private registry for your image | `[]` | +| `imageTag` | The filebeat Docker image tag | `8.0.0-SNAPSHOT` | +| `image` | The filebeat Docker image | `docker.elastic.co/beats/filebeat` | +| `livenessProbe` | Parameters to pass to liveness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `managedServiceAccount` | Whether the `serviceAccount` should be managed by this helm chart. Set this to `false` in order to manage your own service account and related roles | `true` | +| `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | +| `podAnnotations` | Configurable [annotations][] applied to all filebeat pods | `{}` | +| `priorityClassName` | The name of the [PriorityClass][]. No default is supplied as the PriorityClass must be created first | `""` | +| `readinessProbe` | Parameters to pass to readiness [probe][] checks for values such as timeouts and thresholds | see [values.yaml][] | +| `serviceAccount` | Custom [serviceAccount][] that filebeat will use during execution. By default will use the service account created by this chart | `""` | +| `serviceAccountAnnotations` | Annotations to be added to the ServiceAccount that is created by this chart. | `{}` | +| `terminationGracePeriod` | Termination period (in seconds) to wait before killing filebeat pod process on pod shutdown | `30` | +| `updateStrategy` | The [updateStrategy][] for the DaemonSet By default Kubernetes will kill and recreate pods on updates. Setting this to `OnDelete` will require that pods be deleted manually | `RollingUpdate` | ### Deprecated -| Parameter | Description | Default | -|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| Parameter | Description | Default | +|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------| | `affinity` | Configurable [affinity][] for filebeat DaemonSet | `{}` | | `envFrom` | Templatable string to be passed to the [environment from variables][] which will be appended to filebeat container for both DaemonSet and Deployment | `[]` | | `extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for both DaemonSet and Deployment | `[]` | -| `extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | -| `extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | -| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for both filebeat DaemonSet and Deployment | `{}` | +| `extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | +| `extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for both DaemonSet and Deployment | `[]` | +| `filebeatConfig` | Allows you to add any config files in `/usr/share/filebeat` such as `filebeat.yml` for both filebeat DaemonSet and Deployment | `{}` | +| `hostAliases` | Configurable [hostAliases][] | `[]` | | `nodeSelector` | Configurable [nodeSelector][] for filebeat DaemonSet | `{}` | | `podSecurityContext` | Configurable [securityContext][] for filebeat DaemonSet and Deployment pod execution environment | `{}` | | `resources` | Allows you to set the [resources][] for both filebeat DaemonSet and Deployment | `{}` | -| `secretMounts` | Allows you easily mount a secret as a file inside DaemonSet and Deployment Useful for mounting certificates and other secrets | `[]` | +| `secretMounts` | Allows you easily mount a secret as a file inside DaemonSet and Deployment Useful for mounting certificates and other secrets | `[]` | | `tolerations` | Configurable [tolerations][] for both filebeat DaemonSet and Deployment | `[]` | -| `labels` | Configurable [labels][] applied to all filebeat pods +| `labels` | Configurable [labels][] applied to all filebeat pods | `{}` | ## FAQ diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 838f08570..4782a20f2 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -72,8 +72,8 @@ spec: {{- if .Values.dnsConfig }} dnsConfig: {{ toYaml .Values.dnsConfig | nindent 8 }} {{- end }} - {{- if .Values.hostAliases }} - hostAliases: {{ toYaml .Values.hostAliases | nindent 6 }} + {{- if .Values.hostAliases | default .Values.daemonset.hostAliases }} + hostAliases: {{ toYaml ( .Values.hostAliases | default .Values.daemonset.hostAliases ) | nindent 8 }} {{- end }} volumes: {{- range .Values.secretMounts | default .Values.daemonset.secretMounts }} diff --git a/filebeat/templates/deployment.yaml b/filebeat/templates/deployment.yaml index 9fcba3465..a8fd82649 100644 --- a/filebeat/templates/deployment.yaml +++ b/filebeat/templates/deployment.yaml @@ -63,6 +63,9 @@ spec: {{- end }} serviceAccountName: {{ template "filebeat.serviceAccount" . }} terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} + {{- if .Values.deployment.hostAliases }} + hostAliases: {{ toYaml .Values.deployment.hostAliases | nindent 8 }} + {{- end }} volumes: {{- range .Values.secretMounts | default .Values.deployment.secretMounts }} - name: {{ .name }} diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index a2caf5016..883deb05b 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -1178,6 +1178,7 @@ def test_disable_deployment(): def test_hostaliases(): config = """ +daemonset: hostAliases: - ip: "127.0.0.1" hostnames: @@ -1185,5 +1186,24 @@ def test_hostaliases(): - "bar.local" """ r = helm_template(config) + assert ( + "hostAliases" + not in r["deployment"][name]["spec"]["template"]["spec"] + ) hostAliases = r["daemonset"][name]["spec"]["template"]["spec"]["hostAliases"] assert {"ip": "127.0.0.1", "hostnames": ["foo.local", "bar.local"]} in hostAliases + + config = """ +deployment: + hostAliases: + - ip: "127.0.0.1" + hostnames: + - "foo.local" + - "bar.local" +""" + r = helm_template(config) + assert "hostAliases" not in r["daemonset"][name]["spec"]["template"]["spec"] + hostAliases = r["deployment"][name ]["spec"]["template"]["spec"][ + "hostAliases" + ] + assert {"ip": "127.0.0.1", "hostnames": ["foo.local", "bar.local"]} in hostAliases diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 4c650aa8f..cf1334e01 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -20,7 +20,12 @@ daemonset: extraVolumeMounts: [] # - name: extras # mountPath: /usr/share/extras - # readOnly: true + # readOnly: true + hostAliases: [] + #- ip: "127.0.0.1" + # hostnames: + # - "foo.local" + # - "bar.local" hostNetworking: false # Allows you to add any config files in /usr/share/filebeat # such as filebeat.yml for daemonset @@ -86,6 +91,11 @@ deployment: # - name: extras # mountPath: /usr/share/extras # readOnly: true + hostAliases: [] + #- ip: "127.0.0.1" + # hostnames: + # - "foo.local" + # - "bar.local" # such as filebeat.yml for deployment filebeatConfig: filebeat.yml: | @@ -216,3 +226,4 @@ resources: {} secretMounts: [] tolerations: [] labels: {} +hostAliases: [] From 4da932b99e4ce9b38b1dc126e03202a5ba2c80b7 Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Wed, 16 Dec 2020 11:12:30 +0100 Subject: [PATCH 26/36] nit - move toleration tests together --- filebeat/tests/filebeat_test.py | 113 +++++++++++++------------------- 1 file changed, 47 insertions(+), 66 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 883deb05b..d69fccc1b 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -29,14 +29,13 @@ def test_defaults(): assert "filebeat test output" in c["readinessProbe"]["exec"]["command"][-1] assert r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] == [] + assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] assert "hostNetwork" not in r["daemonset"][name]["spec"]["template"]["spec"] assert "dnsPolicy" not in r["daemonset"][name]["spec"]["template"]["spec"] assert "hostNetwork" not in r["deployment"][name]["spec"]["template"]["spec"] assert "dnsPolicy" not in r["deployment"][name]["spec"]["template"]["spec"] - assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] - assert ( r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ "securityContext" @@ -576,29 +575,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -612,29 +605,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -649,15 +636,12 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -696,12 +680,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] config = """ deployment: @@ -723,12 +708,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] def test_adding_a_deprecated_extra_volume_with_volume_mount(): @@ -1186,10 +1172,7 @@ def test_hostaliases(): - "bar.local" """ r = helm_template(config) - assert ( - "hostAliases" - not in r["deployment"][name]["spec"]["template"]["spec"] - ) + assert "hostAliases" not in r["deployment"][name]["spec"]["template"]["spec"] hostAliases = r["daemonset"][name]["spec"]["template"]["spec"]["hostAliases"] assert {"ip": "127.0.0.1", "hostnames": ["foo.local", "bar.local"]} in hostAliases @@ -1203,7 +1186,5 @@ def test_hostaliases(): """ r = helm_template(config) assert "hostAliases" not in r["daemonset"][name]["spec"]["template"]["spec"] - hostAliases = r["deployment"][name ]["spec"]["template"]["spec"][ - "hostAliases" - ] + hostAliases = r["deployment"][name]["spec"]["template"]["spec"]["hostAliases"] assert {"ip": "127.0.0.1", "hostnames": ["foo.local", "bar.local"]} in hostAliases From a8e5ded6aa3c7f3c5bb2862156c2ad268dd2457e Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Wed, 16 Dec 2020 11:38:16 +0100 Subject: [PATCH 27/36] fixup! [filebeat] handle hostAliases for deployment --- filebeat/tests/filebeat_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index d69fccc1b..06d014b43 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -1165,11 +1165,11 @@ def test_disable_deployment(): def test_hostaliases(): config = """ daemonset: -hostAliases: -- ip: "127.0.0.1" - hostnames: - - "foo.local" - - "bar.local" + hostAliases: + - ip: "127.0.0.1" + hostnames: + - "foo.local" + - "bar.local" """ r = helm_template(config) assert "hostAliases" not in r["deployment"][name]["spec"]["template"]["spec"] From c548a20f1e6bcdf40a2986bc0a4fcf63f90d674e Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 12:45:21 +0200 Subject: [PATCH 28/36] Revert the namespace change in volumeMount --- filebeat/templates/daemonset.yaml | 2 +- filebeat/tests/filebeat_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/templates/daemonset.yaml b/filebeat/templates/daemonset.yaml index 838f08570..e9ee59049 100644 --- a/filebeat/templates/daemonset.yaml +++ b/filebeat/templates/daemonset.yaml @@ -94,7 +94,7 @@ spec: {{- end }} - name: data hostPath: - path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-data + path: {{ .Values.hostPathRoot }}/{{ template "filebeat.fullname" . }}-{{ .Release.Namespace }}-data type: DirectoryOrCreate - name: varlibdockercontainers hostPath: diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index a2caf5016..8ec8f5e8c 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -94,7 +94,7 @@ def test_defaults(): assert { "name": "data", "hostPath": { - "path": "/var/lib/" + name + "-data", + "path": "/var/lib/" + name + "-default-data", "type": "DirectoryOrCreate", }, } in daemonset["volumes"] @@ -1124,7 +1124,7 @@ def test_setting_fullnameOverride(): assert { "name": "data", "hostPath": { - "path": "/var/lib/" + custom_name + "-data", + "path": "/var/lib/" + custom_name + "-default-data", "type": "DirectoryOrCreate", }, } in volumes From 9623f4865b97f05372015309be1bcc8774af4baf Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 13:01:47 +0200 Subject: [PATCH 29/36] Made Deployment non-default --- filebeat/values.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/filebeat/values.yaml b/filebeat/values.yaml index cf1334e01..140168b25 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -20,12 +20,7 @@ daemonset: extraVolumeMounts: [] # - name: extras # mountPath: /usr/share/extras - # readOnly: true - hostAliases: [] - #- ip: "127.0.0.1" - # hostnames: - # - "foo.local" - # - "bar.local" + # readOnly: true hostNetworking: false # Allows you to add any config files in /usr/share/filebeat # such as filebeat.yml for daemonset @@ -75,7 +70,7 @@ deployment: labels: {} affinity: {} # Include the deployment - enabled: true + enabled: false # Extra environment variables for Filebeat container. envFrom: [] # - configMapRef: @@ -91,11 +86,6 @@ deployment: # - name: extras # mountPath: /usr/share/extras # readOnly: true - hostAliases: [] - #- ip: "127.0.0.1" - # hostnames: - # - "foo.local" - # - "bar.local" # such as filebeat.yml for deployment filebeatConfig: filebeat.yml: | @@ -103,7 +93,7 @@ deployment: - type: tcp max_message_size: 10MiB host: "localhost:9000" - + output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' @@ -226,4 +216,3 @@ resources: {} secretMounts: [] tolerations: [] labels: {} -hostAliases: [] From be0a49a86d6cc61242ed2514e39a9a59d974a18d Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 13:02:02 +0200 Subject: [PATCH 30/36] Changed all tests to enable Deployment --- filebeat/tests/filebeat_test.py | 96 +++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 2f80faa94..da1c224e1 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -10,6 +10,8 @@ def test_defaults(): config = """ +deployment: + enabled: true """ r = helm_template(config) @@ -135,6 +137,8 @@ def test_defaults(): def test_adding_a_extra_container(): config = """ +deployment: + enabled: true extraContainers: | - name: do-something image: busybox @@ -162,6 +166,8 @@ def test_adding_a_extra_container(): def test_adding_init_containers_as_yaml(): config = """ +deployment: + enabled: true extraInitContainers: - name: dummy-init image: busybox @@ -178,6 +184,8 @@ def test_adding_init_containers_as_yaml(): def test_adding_a_extra_init_container(): config = """ +deployment: + enabled: true extraInitContainers: | - name: do-something image: busybox @@ -205,6 +213,8 @@ def test_adding_a_extra_init_container(): def test_adding_envs(): config = """ +deployment: + enabled: true daemonset: extraEnvs: - name: LOG_LEVEL @@ -220,6 +230,7 @@ def test_adding_envs(): config = """ deployment: + enabled: true extraEnvs: - name: LOG_LEVEL value: DEBUG @@ -235,6 +246,8 @@ def test_adding_envs(): def test_adding_deprecated_envs(): config = """ +deployment: + enabled: true extraEnvs: - name: LOG_LEVEL value: DEBUG @@ -250,6 +263,8 @@ def test_adding_deprecated_envs(): def test_adding_image_pull_secrets(): config = """ +deployment: + enabled: true imagePullSecrets: - name: test-registry """ @@ -262,6 +277,8 @@ def test_adding_image_pull_secrets(): def test_adding_host_networking(): config = """ +deployment: + enabled: true daemonset: hostNetworking: true """ @@ -277,6 +294,8 @@ def test_adding_host_networking(): def test_adding_tolerations(): config = """ +deployment: + enabled: true daemonset: tolerations: - key: "key1" @@ -294,6 +313,7 @@ def test_adding_tolerations(): config = """ deployment: + enabled: true tolerations: - key: "key1" operator: "Equal" @@ -311,6 +331,8 @@ def test_adding_tolerations(): def test_adding_deprecated_tolerations(): config = """ +deployment: + enabled: true tolerations: - key: "key1" operator: "Equal" @@ -331,6 +353,8 @@ def test_adding_deprecated_tolerations(): def test_override_the_default_update_strategy(): config = """ +deployment: + enabled: true updateStrategy: OnDelete """ @@ -340,6 +364,8 @@ def test_override_the_default_update_strategy(): def test_setting_a_custom_service_account(): config = """ +deployment: + enabled: true serviceAccount: notdefault """ r = helm_template(config) @@ -351,6 +377,8 @@ def test_setting_a_custom_service_account(): def test_self_managing_rbac_resources(): config = """ +deployment: + enabled: true managedServiceAccount: false """ r = helm_template(config) @@ -361,6 +389,8 @@ def test_self_managing_rbac_resources(): def test_setting_pod_security_context(): config = """ +deployment: + enabled: true daemonset: securityContext: runAsUser: 1001 @@ -394,6 +424,7 @@ def test_setting_pod_security_context(): config = """ deployment: + enabled: true securityContext: runAsUser: 1001 privileged: false @@ -427,6 +458,8 @@ def test_setting_pod_security_context(): def test_setting_deprecated_pod_security_context(): config = """ +deployment: + enabled: true podSecurityContext: runAsUser: 1001 privileged: false @@ -468,6 +501,7 @@ def test_adding_in_filebeat_config(): hello = daemonset deployment: + enabled: true filebeatConfig: filebeat.yml: | key: deployment @@ -515,6 +549,8 @@ def test_adding_in_filebeat_config(): def test_adding_in_deprecated_filebeat_config(): config = """ +deployment: + enabled: true filebeatConfig: filebeat.yml: | key: @@ -568,6 +604,8 @@ def test_adding_in_deprecated_filebeat_config(): def test_adding_a_secret_mount(): config = """ +deployment: + enabled: true daemonset: secretMounts: - name: elastic-certificates @@ -599,6 +637,7 @@ def test_adding_a_secret_mount(): config = """ deployment: + enabled: true secretMounts: - name: elastic-certificates secretName: elastic-certificates-name @@ -630,6 +669,8 @@ def test_adding_a_secret_mount(): def test_adding_a_deprecated_secret_mount(): config = """ +deployment: + enabled: true secretMounts: - name: elastic-certificates secretName: elastic-certificates-name @@ -661,6 +702,8 @@ def test_adding_a_deprecated_secret_mount(): def test_adding_a_extra_volume_with_volume_mount(): config = """ +deployment: + enabled: true daemonset: extraVolumes: - name: extras @@ -690,6 +733,7 @@ def test_adding_a_extra_volume_with_volume_mount(): config = """ deployment: + enabled: true extraVolumes: - name: extras emptyDir: {} @@ -719,6 +763,8 @@ def test_adding_a_extra_volume_with_volume_mount(): def test_adding_a_deprecated_extra_volume_with_volume_mount(): config = """ +deployment: + enabled: true extraVolumes: - name: extras emptyDir: {} @@ -744,6 +790,8 @@ def test_adding_a_deprecated_extra_volume_with_volume_mount(): def test_adding_a_node_selector(): config = """ +deployment: + enabled: true daemonset: nodeSelector: disktype: ssd @@ -757,6 +805,7 @@ def test_adding_a_node_selector(): config = """ deployment: + enabled: true nodeSelector: disktype: ssd """ @@ -770,6 +819,8 @@ def test_adding_a_node_selector(): def test_adding_deprecated_node_selector(): config = """ +deployment: + enabled: true nodeSelector: disktype: ssd """ @@ -782,6 +833,8 @@ def test_adding_deprecated_node_selector(): def test_adding_an_affinity_rule(): config = """ +deployment: + enabled: true affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -804,6 +857,8 @@ def test_adding_an_affinity_rule(): assert r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} config = """ +deployment: + enabled: true daemonset: affinity: podAntiAffinity: @@ -827,6 +882,7 @@ def test_adding_an_affinity_rule(): config = """ deployment: + enabled: true affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -850,6 +906,8 @@ def test_adding_an_affinity_rule(): def test_priority_class_name(): config = """ +deployment: + enabled: true priorityClassName: "" """ r = helm_template(config) @@ -859,6 +917,8 @@ def test_priority_class_name(): assert "priorityClassName" not in deployment_spec config = """ +deployment: + enabled: true priorityClassName: "highest" """ r = helm_template(config) @@ -874,6 +934,8 @@ def test_priority_class_name(): def test_adding_deprecated_labels(): config = """ +deployment: + enabled: true labels: app-test: filebeat """ @@ -892,6 +954,8 @@ def test_adding_deprecated_labels(): def test_adding_daemonset_labels(): config = """ +deployment: + enabled: true daemonset: labels: app-test: filebeat @@ -906,6 +970,8 @@ def test_adding_daemonset_labels(): def test_adding_daemonset_labels_surpasses_root_labels(): config = """ +deployment: + enabled: true labels: app-test: root-filebeat daemonset: @@ -925,6 +991,7 @@ def test_adding_daemonset_labels_surpasses_root_labels(): def test_adding_deployment_labels(): config = """ deployment: + enabled: true labels: app-test: filebeat """ @@ -941,6 +1008,7 @@ def test_adding_deployment_labels_surpasses_root_labels(): labels: app-test: root-filebeat deployment: + enabled: true labels: app-test: deployment-filebeat """ @@ -956,6 +1024,8 @@ def test_adding_deployment_labels_surpasses_root_labels(): def test_adding_serviceaccount_annotations(): config = """ +deployment: + enabled: true serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount """ @@ -970,6 +1040,8 @@ def test_adding_serviceaccount_annotations(): def test_adding_env_from(): config = """ +deployment: + enabled: true daemonset: envFrom: - configMapRef: @@ -986,6 +1058,7 @@ def test_adding_env_from(): config = """ deployment: + enabled: true envFrom: - configMapRef: name: configmap-name @@ -1002,6 +1075,8 @@ def test_adding_env_from(): def test_adding_deprecated_env_from(): config = """ +deployment: + enabled: true envFrom: - configMapRef: name: configmap-name @@ -1017,6 +1092,8 @@ def test_adding_deprecated_env_from(): def test_overriding_resources(): config = """ +deployment: + enabled: true daemonset: resources: limits: @@ -1042,6 +1119,7 @@ def test_overriding_resources(): config = """ deployment: + enabled: true resources: limits: cpu: "25m" @@ -1067,6 +1145,8 @@ def test_overriding_resources(): def test_adding_deprecated_resources(): config = """ +deployment: + enabled: true resources: limits: cpu: "25m" @@ -1092,6 +1172,8 @@ def test_adding_deprecated_resources(): def test_setting_fullnameOverride(): config = """ +deployment: + enabled: true fullnameOverride: 'filebeat-custom' """ r = helm_template(config) @@ -1118,6 +1200,8 @@ def test_setting_fullnameOverride(): def test_adding_annotations(): config = """ +deployment: + enabled: true daemonset: annotations: foo: "bar" @@ -1128,8 +1212,9 @@ def test_adding_annotations(): assert "annotations" not in r["deployment"][name]["metadata"] config = """ deployment: - annotations: - grault: "waldo" + enabled: true + annotations: + grault: "waldo" """ r = helm_template(config) assert "grault" in r["deployment"][name]["metadata"]["annotations"] @@ -1139,6 +1224,8 @@ def test_adding_annotations(): def test_disable_daemonset(): config = """ +deployment: + enabled: true daemonset: enabled: false """ @@ -1153,7 +1240,7 @@ def test_disable_daemonset(): def test_disable_deployment(): config = """ deployment: - enabled: false + enabled: false """ r = helm_template(config) cfg = r["configmap"] @@ -1164,6 +1251,8 @@ def test_disable_deployment(): def test_hostaliases(): config = """ +deployment: + enabled: true daemonset: hostAliases: - ip: "127.0.0.1" @@ -1178,6 +1267,7 @@ def test_hostaliases(): config = """ deployment: + enabled: true hostAliases: - ip: "127.0.0.1" hostnames: From e3a1178f174a4ff2d559d3cd338919d67afe43c7 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 14:24:23 +0200 Subject: [PATCH 31/36] Python linting --- filebeat/tests/filebeat_test.py | 101 ++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index da1c224e1..6b1638b92 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -613,23 +613,29 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -644,23 +650,29 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -677,12 +689,15 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -723,13 +738,12 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ "template" ]["spec"]["volumes"] - assert { - "name": "extras", - "mountPath": "/usr/share/extras", - "readOnly": True, - } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) config = """ deployment: @@ -752,13 +766,12 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert { - "name": "extras", - "mountPath": "/usr/share/extras", - "readOnly": True, - } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) def test_adding_a_deprecated_extra_volume_with_volume_mount(): From 339bf398cfc77fddebc594ebbaec62d3674571d0 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 15:35:16 +0200 Subject: [PATCH 32/36] Integration test for deployment --- filebeat/examples/deployment/Makefile | 13 ++++++++++ filebeat/examples/deployment/README.md | 27 +++++++++++++++++++++ filebeat/examples/deployment/test/goss.yaml | 6 +++++ filebeat/examples/deployment/values.yaml | 16 ++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 filebeat/examples/deployment/Makefile create mode 100644 filebeat/examples/deployment/README.md create mode 100644 filebeat/examples/deployment/test/goss.yaml create mode 100644 filebeat/examples/deployment/values.yaml diff --git a/filebeat/examples/deployment/Makefile b/filebeat/examples/deployment/Makefile new file mode 100644 index 000000000..0bc285379 --- /dev/null +++ b/filebeat/examples/deployment/Makefile @@ -0,0 +1,13 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-filebeat-deployment + +install: + helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../ + +test: install goss + +purge: + helm del $(RELEASE) diff --git a/filebeat/examples/deployment/README.md b/filebeat/examples/deployment/README.md new file mode 100644 index 000000000..b41e681e4 --- /dev/null +++ b/filebeat/examples/deployment/README.md @@ -0,0 +1,27 @@ +# Default + +This example deploy Filebeat 8.0.0-SNAPSHOT using [default values][] as a Kubernetes Deployment. + + +## Usage + +* Deploy [Elasticsearch Helm chart][]. + +* Deploy Filebeat chart with the default values: `make install` + +* You can now setup a port forward to query Filebeat indices: + + ``` + kubectl port-forward svc/elasticsearch-master 9200 + curl localhost:9200/_cat/indices + ``` + + +## Testing + +You can also run [goss integration tests][] using `make test` + + +[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/default/ +[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/filebeat/examples/deployment/test/goss.yaml +[default values]: https://github.com/elastic/helm-charts/tree/master/filebeat/values.yaml diff --git a/filebeat/examples/deployment/test/goss.yaml b/filebeat/examples/deployment/test/goss.yaml new file mode 100644 index 000000000..b6b86bddb --- /dev/null +++ b/filebeat/examples/deployment/test/goss.yaml @@ -0,0 +1,6 @@ +http: + http://elasticsearch-master:9200/_cat/indices: + status: 200 + timeout: 2000 + body: + - 'filebeat-8.0.0' diff --git a/filebeat/examples/deployment/values.yaml b/filebeat/examples/deployment/values.yaml new file mode 100644 index 000000000..bf1cf06c1 --- /dev/null +++ b/filebeat/examples/deployment/values.yaml @@ -0,0 +1,16 @@ +deployment: + enabled: true + +daemonset: + enabled: false + +filebeatConfig: + filebeat.yml: | + filebeat.inputs: + - type: log + paths: + - /usr/share/filebeat/logs/filebeat + + output.elasticsearch: + host: '${NODE_NAME}' + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' \ No newline at end of file From b218547881c4aadac99e239b7f8fb972c6757fe6 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Wed, 16 Dec 2020 15:43:55 +0200 Subject: [PATCH 33/36] Reverted Security integration test --- filebeat/examples/security/values.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/filebeat/examples/security/values.yaml b/filebeat/examples/security/values.yaml index e60b69530..5ed7c5529 100644 --- a/filebeat/examples/security/values.yaml +++ b/filebeat/examples/security/values.yaml @@ -1,9 +1,15 @@ filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: tcp - max_message_size: 10MiB - host: "localhost:9000" + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" output.elasticsearch: username: '${ELASTICSEARCH_USERNAME}' From 8126d64f237fe6478064a4c8d757759d4045e30d Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Fri, 18 Dec 2020 12:51:21 +0100 Subject: [PATCH 34/36] [filebeat] update tests for filebeat deployment This commit update test to enable deployment only when the deployment resources is really tested, also disable deployment in default test (test with only chart default values) and added a new default test for deployment with only default values. --- filebeat/tests/filebeat_test.py | 303 ++++++++++++++++++-------------- 1 file changed, 174 insertions(+), 129 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 6b1638b92..327162a71 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -10,14 +10,12 @@ def test_defaults(): config = """ -deployment: - enabled: true """ r = helm_template(config) assert name in r["daemonset"] - assert name in r["deployment"] + assert "deployment" not in r c = r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0] assert c["name"] == project @@ -31,12 +29,9 @@ def test_defaults(): assert "filebeat test output" in c["readinessProbe"]["exec"]["command"][-1] assert r["daemonset"][name]["spec"]["template"]["spec"]["tolerations"] == [] - assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] assert "hostNetwork" not in r["daemonset"][name]["spec"]["template"]["spec"] assert "dnsPolicy" not in r["daemonset"][name]["spec"]["template"]["spec"] - assert "hostNetwork" not in r["deployment"][name]["spec"]["template"]["spec"] - assert "dnsPolicy" not in r["deployment"][name]["spec"]["template"]["spec"] assert ( r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ @@ -50,18 +45,6 @@ def test_defaults(): ]["privileged"] == False ) - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "securityContext" - ]["runAsUser"] - == 0 - ) - assert ( - r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "securityContext" - ]["privileged"] - == False - ) # Empty customizable defaults assert "imagePullSecrets" not in r["daemonset"][name]["spec"]["template"]["spec"] @@ -76,10 +59,8 @@ def test_defaults(): assert name + "-config" not in cfg assert name + "-daemonset-config" in cfg - assert name + "-deployment-config" in cfg assert "filebeat.yml" in cfg[name + "-daemonset-config"]["data"] - assert "filebeat.yml" in cfg[name + "-deployment-config"]["data"] daemonset = r["daemonset"][name]["spec"]["template"]["spec"] @@ -107,6 +88,65 @@ def test_defaults(): "readOnly": True, } in daemonset["containers"][0]["volumeMounts"] + assert daemonset["containers"][0]["resources"] == { + "requests": {"cpu": "100m", "memory": "100Mi"}, + "limits": {"cpu": "1000m", "memory": "200Mi"}, + } + + +def test_enable_deployment(): + config = """ +deployment: + enabled: true + """ + + r = helm_template(config) + + assert name in r["deployment"] + + c = r["deployment"][name]["spec"]["template"]["spec"]["containers"][0] + assert c["name"] == project + assert c["image"].startswith("docker.elastic.co/beats/" + project + ":") + + assert c["env"][0]["name"] == "POD_NAMESPACE" + assert c["env"][0]["valueFrom"]["fieldRef"]["fieldPath"] == "metadata.namespace" + + assert "curl --fail 127.0.0.1:5066" in c["livenessProbe"]["exec"]["command"][-1] + + assert "filebeat test output" in c["readinessProbe"]["exec"]["command"][-1] + + assert r["deployment"][name]["spec"]["template"]["spec"]["tolerations"] == [] + + assert "hostNetwork" not in r["deployment"][name]["spec"]["template"]["spec"] + assert "dnsPolicy" not in r["deployment"][name]["spec"]["template"]["spec"] + + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["runAsUser"] + == 0 + ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "securityContext" + ]["privileged"] + == False + ) + + # Empty customizable defaults + assert "imagePullSecrets" not in r["deployment"][name]["spec"]["template"]["spec"] + + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["serviceAccountName"] == name + ) + + cfg = r["configmap"] + + assert name + "-config" not in cfg + assert name + "-deployment-config" in cfg + + assert "filebeat.yml" in cfg[name + "-deployment-config"]["data"] + deployment = r["deployment"][name]["spec"]["template"]["spec"] assert { @@ -125,10 +165,6 @@ def test_defaults(): "readOnly": True, } in deployment["containers"][0]["volumeMounts"] - assert daemonset["containers"][0]["resources"] == { - "requests": {"cpu": "100m", "memory": "100Mi"}, - "limits": {"cpu": "1000m", "memory": "200Mi"}, - } assert deployment["containers"][0]["resources"] == { "requests": {"cpu": "100m", "memory": "100Mi"}, "limits": {"cpu": "1000m", "memory": "200Mi"}, @@ -174,12 +210,23 @@ def test_adding_init_containers_as_yaml(): command: ['echo', 'hey'] """ r = helm_template(config) - initContainers = r["daemonset"][name]["spec"]["template"]["spec"]["initContainers"] + initContainersDaemonset = r["daemonset"][name]["spec"]["template"]["spec"][ + "initContainers" + ] assert { "name": "dummy-init", "image": "busybox", "command": ["echo", "hey"], - } in initContainers + } in initContainersDaemonset + deployment_name = name + initContainersDeployment = r["deployment"][deployment_name]["spec"]["template"][ + "spec" + ]["initContainers"] + assert { + "name": "dummy-init", + "image": "busybox", + "command": ["echo", "hey"], + } in initContainersDeployment def test_adding_a_extra_init_container(): @@ -273,6 +320,10 @@ def test_adding_image_pull_secrets(): r["daemonset"][name]["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] == "test-registry" ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["imagePullSecrets"][0]["name"] + == "test-registry" + ) def test_adding_host_networking(): @@ -353,8 +404,6 @@ def test_adding_deprecated_tolerations(): def test_override_the_default_update_strategy(): config = """ -deployment: - enabled: true updateStrategy: OnDelete """ @@ -373,12 +422,14 @@ def test_setting_a_custom_service_account(): r["daemonset"][name]["spec"]["template"]["spec"]["serviceAccountName"] == "notdefault" ) + assert ( + r["deployment"][name]["spec"]["template"]["spec"]["serviceAccountName"] + == "notdefault" + ) def test_self_managing_rbac_resources(): config = """ -deployment: - enabled: true managedServiceAccount: false """ r = helm_template(config) @@ -613,29 +664,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -650,29 +695,23 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -689,15 +728,12 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert ( - { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } - in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -738,12 +774,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] config = """ deployment: @@ -766,12 +803,13 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert ( - {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} - not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] - ) + assert { + "name": "extras", + "mountPath": "/usr/share/extras", + "readOnly": True, + } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] def test_adding_a_deprecated_extra_volume_with_volume_mount(): @@ -842,33 +880,13 @@ def test_adding_deprecated_node_selector(): r["daemonset"][name]["spec"]["template"]["spec"]["nodeSelector"]["disktype"] == "ssd" ) - - -def test_adding_an_affinity_rule(): - config = """ -deployment: - enabled: true -affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - filebeat - topologyKey: kubernetes.io/hostname -""" - - r = helm_template(config) assert ( - r["daemonset"][name]["spec"]["template"]["spec"]["affinity"]["podAntiAffinity"][ - "requiredDuringSchedulingIgnoredDuringExecution" - ][0]["topologyKey"] - == "kubernetes.io/hostname" + "disktype" + not in r["deployment"][name]["spec"]["template"]["spec"]["nodeSelector"] ) - assert r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} + +def test_adding_an_affinity_rule(): config = """ deployment: enabled: true @@ -892,6 +910,10 @@ def test_adding_an_affinity_rule(): ][0]["topologyKey"] == "kubernetes.io/hostname" ) + assert ( + "podAntiAffinity" + not in r["deployment"][name]["spec"]["template"]["spec"]["affinity"] + ) config = """ deployment: @@ -915,6 +937,36 @@ def test_adding_an_affinity_rule(): ]["requiredDuringSchedulingIgnoredDuringExecution"][0]["topologyKey"] == "kubernetes.io/hostname" ) + assert ( + "podAntiAffinity" + not in r["daemonset"][name]["spec"]["template"]["spec"]["affinity"] + ) + + +def test_adding_deprecated_affinity_rule(): + config = """ +deployment: + enabled: true +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - filebeat + topologyKey: kubernetes.io/hostname +""" + + r = helm_template(config) + assert ( + r["daemonset"][name]["spec"]["template"]["spec"]["affinity"]["podAntiAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ][0]["topologyKey"] + == "kubernetes.io/hostname" + ) + assert r["deployment"][name]["spec"]["template"]["spec"]["affinity"] == {} def test_priority_class_name(): @@ -967,8 +1019,6 @@ def test_adding_deprecated_labels(): def test_adding_daemonset_labels(): config = """ -deployment: - enabled: true daemonset: labels: app-test: filebeat @@ -983,8 +1033,6 @@ def test_adding_daemonset_labels(): def test_adding_daemonset_labels_surpasses_root_labels(): config = """ -deployment: - enabled: true labels: app-test: root-filebeat daemonset: @@ -1037,8 +1085,6 @@ def test_adding_deployment_labels_surpasses_root_labels(): def test_adding_serviceaccount_annotations(): config = """ -deployment: - enabled: true serviceAccountAnnotations: eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount """ @@ -1209,6 +1255,17 @@ def test_setting_fullnameOverride(): "type": "DirectoryOrCreate", }, } in volumes + assert custom_name in r["deployment"] + assert ( + r["deployment"][custom_name]["spec"]["template"]["spec"]["containers"][0][ + "name" + ] + == project + ) + assert ( + r["deployment"][custom_name]["spec"]["template"]["spec"]["serviceAccountName"] + == name + ) def test_adding_annotations(): @@ -1250,18 +1307,6 @@ def test_disable_daemonset(): assert name + "-deployment-config" in cfg -def test_disable_deployment(): - config = """ -deployment: - enabled: false -""" - r = helm_template(config) - cfg = r["configmap"] - - assert name + "-daemonset-config" in cfg - assert name + "-deployment-config" not in cfg - - def test_hostaliases(): config = """ deployment: From a4eb430991a296df6d4838618811d2a71fc21754 Mon Sep 17 00:00:00 2001 From: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> Date: Fri, 18 Dec 2020 12:58:59 +0100 Subject: [PATCH 35/36] [filebeat] add deployment test to ci --- helpers/matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/matrix.yml b/helpers/matrix.yml index aa2f4b3c4..575afaba6 100644 --- a/helpers/matrix.yml +++ b/helpers/matrix.yml @@ -19,6 +19,7 @@ KIBANA_SUITE: - upgrade FILEBEAT_SUITE: - default + - deployment - oss - security - upgrade From 6e6978fa40d71978ee7a8049f36a2b3b98289ad1 Mon Sep 17 00:00:00 2001 From: John Torakis Date: Fri, 18 Dec 2020 14:13:36 +0200 Subject: [PATCH 36/36] Linting again --- filebeat/tests/filebeat_test.py | 101 ++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py index 327162a71..9460e8ec6 100644 --- a/filebeat/tests/filebeat_test.py +++ b/filebeat/tests/filebeat_test.py @@ -664,23 +664,29 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"] - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -695,23 +701,29 @@ def test_adding_a_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"] - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -728,12 +740,15 @@ def test_adding_a_deprecated_secret_mount(): path: /usr/share/filebeat/config/certs """ r = helm_template(config) - assert { - "mountPath": "/usr/share/filebeat/config/certs", - "name": "elastic-certificates", - } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + { + "mountPath": "/usr/share/filebeat/config/certs", + "name": "elastic-certificates", + } + in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) assert { "name": "elastic-certificates", "secret": {"secretName": "elastic-certificates-name"}, @@ -774,13 +789,12 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][ "template" ]["spec"]["volumes"] - assert { - "name": "extras", - "mountPath": "/usr/share/extras", - "readOnly": True, - } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) config = """ deployment: @@ -803,13 +817,12 @@ def test_adding_a_extra_volume_with_volume_mount(): assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][ "template" ]["spec"]["volumes"] - assert { - "name": "extras", - "mountPath": "/usr/share/extras", - "readOnly": True, - } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ - "volumeMounts" - ] + assert ( + {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,} + not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][ + "volumeMounts" + ] + ) def test_adding_a_deprecated_extra_volume_with_volume_mount():