From ea93518c6aed87c3af59b6d5e0f7a5081d1af275 Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 13:00:23 +0100 Subject: [PATCH 1/8] Add separate resources for worker, web and scheduler for airflow --- bitnami/airflow/README.md | 6 ++- .../templates/deployment-scheduler.yaml | 2 +- bitnami/airflow/templates/deployment-web.yaml | 2 +- .../airflow/templates/statefulset-worker.yaml | 2 +- bitnami/airflow/values-production.yaml | 50 +++++++++++++++---- bitnami/airflow/values.yaml | 42 ++++++++++++++-- 6 files changed, 85 insertions(+), 19 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index df6f14701c268e..cadb2091d151d1 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -93,12 +93,15 @@ The following tables lists the configurable parameters of the Airflow chart and | `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | +| `airflow.worker.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | | `airflow.auth.forcePassword` | Force users to specify a password | `false` | | `airflow.auth.username` | Username to access web UI | `user` | | `airflow.auth.password` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | +| `airflow.web.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `airflow.scheduler.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | | `airflow.webserverConfigMap` | Config map name for ~/airflow/webserver_config.py | `nil` | | `securityContext.enabled` | Enable security context | `true` | | `securityContext.fsGroup` | Group ID for the container | `1001` | @@ -106,7 +109,7 @@ The following tables lists the configurable parameters of the Airflow chart and | `service.type` | Kubernetes Service type | `ClusterIP` | | `service.port` | Airflow Web port | `8080` | | `service.nodePort` | Kubernetes Service nodePort | `nil` | -| `service.loadBalancerIP` | loadBalancerIP for Airflow Service | `nil` | +| `service.loadBalancerIP` | loadBalancerIP for Airflow Service | `nil` | | `service.annotations` | Service annotations | `` | | `ingress.enabled` | Enable ingress controller resource | `false` | | `ingress.certManager` | Add annotations for cert-manager | `false` | @@ -122,7 +125,6 @@ The following tables lists the configurable parameters of the Airflow chart and | `nodeSelector` | Node labels for pod assignment | `{}` | | `tolerations` | Toleration labels for pod assignment | `[]` | | `affinity` | Map of node/pod affinities | `{}` | -| `resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | | `livenessProbe.enabled` | would you like a livessProbed to be enabled | `true` | | `livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | 180 | | `livenessProbe.periodSeconds` | How often to perform the probe | 20 | diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index bc637a4c7b0701..14ca87f15fbc91 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -221,7 +221,7 @@ spec: mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg {{- end }} - resources: {{- toYaml .Values.resources | nindent 12 }} + resources: {{- toYaml .Values.airflow.scheduler.resources | nindent 12 }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index c2c4ba972182b0..7c3be6a18d2a50 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -256,7 +256,7 @@ spec: mountPath: /opt/bitnami/airflow/webserver_config.py subPath: webserver_config.py {{- end }} - resources: {{- toYaml .Values.resources | nindent 12 }} + resources: {{- toYaml .Values.airflow.web.resources | nindent 12 }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index c34b7d27ce36bc..c8f5af90395593 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -102,7 +102,7 @@ spec: - name: airflow-worker image: "{{ template "airflow.workerImage" . }}" imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}" - resources: {{- toYaml .Values.resources | nindent 12 }} + resources: {{- toYaml .Values.airflow.worker.resources | nindent 12 }} env: {{- if .Values.workerImage.debug }} - name: BASH_DEBUG diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index 7a2430b1189904..e5ccd550c4ef72 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -148,6 +148,45 @@ airflow: worker: port: 8793 replicas: 3 + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Airflow web specific configuration + # + web: + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + + ## Airflow scheduler specific configuration + # + scheduler: + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html @@ -276,17 +315,6 @@ tolerations: [] ## affinity: {} -## Configure resource requests and limits -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ -## -resources: -# limits: -# cpu: 200m -# memory: 1Gi -# requests: -# memory: 256Mi -# cpu: 250m - ## Configure extra options for liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) livenessProbe: diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 16974fc6bcc475..6c35e69073b7a2 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -143,11 +143,47 @@ airflow: # path: ## URL used to access to airflow web ui ## - # baseUrl: - ## Airflow worker component configuration worker: + # baseUrl: + ## Airflow worker component configuration port: 8793 - replicas: 1 + replicas: 2 + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + ## Airflow web specific configuration + # + web: + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m + ## Airflow scheduler specific configuration + # + scheduler: + ## Configure resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + # limits: + # cpu: 200m + # memory: 1Gi + # requests: + # memory: 256Mi + # cpu: 250m ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html From 8e891914ab86a9a728d9d80245aa3b1cb1b56570 Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 13:07:46 +0100 Subject: [PATCH 2/8] update chart version --- bitnami/airflow/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml index 6985e166aec836..e23a57d3ad9662 100644 --- a/bitnami/airflow/Chart.yaml +++ b/bitnami/airflow/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: airflow -version: 4.3.3 +version: 4.3.4 appVersion: 1.10.9 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows. keywords: From ba1f2bdad151df3cf1b7783f65b4c1ba29106146 Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 14:51:36 +0100 Subject: [PATCH 3/8] change worker replicas to 1 --- bitnami/airflow/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 6c35e69073b7a2..9a9a6515e6da39 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -147,7 +147,7 @@ airflow: # baseUrl: ## Airflow worker component configuration port: 8793 - replicas: 2 + replicas: 1 ## Configure resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## From 957e8eea4d21955f3258ea37e583df8903272429 Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 17:25:37 +0100 Subject: [PATCH 4/8] change version, in readme set the default values, values.yaml and values-production.yaml --- bitnami/airflow/Chart.yaml | 2 +- bitnami/airflow/README.md | 6 +-- bitnami/airflow/values-production.yaml | 58 ++++++++++++++++---------- bitnami/airflow/values.yaml | 54 ++++++++++++++---------- 4 files changed, 72 insertions(+), 48 deletions(-) diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml index e23a57d3ad9662..bb129c53a2a853 100644 --- a/bitnami/airflow/Chart.yaml +++ b/bitnami/airflow/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: airflow -version: 4.3.4 +version: 4.4.0 appVersion: 1.10.9 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows. keywords: diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index cadb2091d151d1..cf47c8f33399ae 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -93,15 +93,15 @@ The following tables lists the configurable parameters of the Airflow chart and | `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | -| `airflow.worker.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `airflow.worker.resources` | CPU/Memory resource requests/limits | `{}` | | `airflow.auth.forcePassword` | Force users to specify a password | `false` | | `airflow.auth.username` | Username to access web UI | `user` | | `airflow.auth.password` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | -| `airflow.web.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | -| `airflow.scheduler.resources` | CPU/Memory resource requests/limits | Memory: `256Mi`, CPU: `250m` | +| `airflow.web.resources` | CPU/Memory resource requests/limits | `{}` | +| `airflow.scheduler.resources` | CPU/Memory resource requests/limits | `{}` | | `airflow.webserverConfigMap` | Config map name for ~/airflow/webserver_config.py | `nil` | | `securityContext.enabled` | Enable security context | `true` | | `securityContext.fsGroup` | Group ID for the container | `1001` | diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index e5ccd550c4ef72..5e441728efa5f6 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -148,44 +148,56 @@ airflow: worker: port: 8793 replicas: 3 - ## Configure resource requests and limits + ## Worker containers' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## Airflow web specific configuration # web: - ## Configure resource requests and limits + ## Web container' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m - - ## Airflow scheduler specific configuration + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m + + ## Scheduler containers specific configuration # scheduler: - ## Configure resource requests and limits + ## Scheduler containers' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## All the parameters from the configuration file can be overwritten by using environment variables with this format: diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 9a9a6515e6da39..f3e0231086abbe 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -148,42 +148,54 @@ airflow: ## Airflow worker component configuration port: 8793 replicas: 1 - ## Configure resource requests and limits + ## Worker containers' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## Airflow web specific configuration # web: - ## Configure resource requests and limits + ## Web container' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## Airflow scheduler specific configuration # scheduler: - ## Configure resource requests and limits + ## Scheduler containers' resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ ## resources: - # limits: - # cpu: 200m - # memory: 1Gi - # requests: - # memory: 256Mi - # cpu: 250m + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + limits: {} + # cpu: 200m + # memory: 1Gi + requests: {} + # memory: 256Mi + # cpu: 250m ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html From fffb41c61beda4a51a8d81acb263301291fe34c5 Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 17:55:21 +0100 Subject: [PATCH 5/8] Add if in resources and separate limits and requests --- bitnami/airflow/README.md | 9 ++++++--- bitnami/airflow/templates/deployment-scheduler.yaml | 2 ++ bitnami/airflow/templates/deployment-web.yaml | 2 ++ bitnami/airflow/templates/metrics-deployment.yaml | 2 ++ bitnami/airflow/templates/statefulset-worker.yaml | 2 ++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index cf47c8f33399ae..245b122a77bbac 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -93,15 +93,18 @@ The following tables lists the configurable parameters of the Airflow chart and | `airflow.baseUrl` | URL used to access to airflow web ui | `nil` | | `airflow.worker.port` | Airflow Worker port | `8793` | | `airflow.worker.replicas` | Number of Airflow Worker replicas | `2` | -| `airflow.worker.resources` | CPU/Memory resource requests/limits | `{}` | +| `airflow.worker.resources.limits` | The resources limits for the Worker containers | `{}` | +| `airflow.worker.resources.requests` | The requested resources for the Worker containers | `{}` | | `airflow.auth.forcePassword` | Force users to specify a password | `false` | | `airflow.auth.username` | Username to access web UI | `user` | | `airflow.auth.password` | Password to access web UI | `nil` | | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | -| `airflow.web.resources` | CPU/Memory resource requests/limits | `{}` | -| `airflow.scheduler.resources` | CPU/Memory resource requests/limits | `{}` | +| `airflow.web.resources.limits` | The resources limits for the Worker containers | `{}` | +| `airflow.web.resources.requests` | The requested resources for the Worker containers | `{}` | +| `airflow.scheduler.resources.limits` | The resources limits for the Worker containers | `{}` | +| `airflow.scheduler.resources.requests` | The requested resources for the Worker containers | `{}` | | `airflow.webserverConfigMap` | Config map name for ~/airflow/webserver_config.py | `nil` | | `securityContext.enabled` | Enable security context | `true` | | `securityContext.fsGroup` | Group ID for the container | `1001` | diff --git a/bitnami/airflow/templates/deployment-scheduler.yaml b/bitnami/airflow/templates/deployment-scheduler.yaml index 14ca87f15fbc91..5eed22394c8654 100644 --- a/bitnami/airflow/templates/deployment-scheduler.yaml +++ b/bitnami/airflow/templates/deployment-scheduler.yaml @@ -221,7 +221,9 @@ spec: mountPath: /opt/bitnami/airflow/airflow.cfg subPath: airflow.cfg {{- end }} + {{- if .Values.airflow.scheduler.resources }} resources: {{- toYaml .Values.airflow.scheduler.resources | nindent 12 }} + {{- end }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/deployment-web.yaml b/bitnami/airflow/templates/deployment-web.yaml index 7c3be6a18d2a50..67f25f88634048 100644 --- a/bitnami/airflow/templates/deployment-web.yaml +++ b/bitnami/airflow/templates/deployment-web.yaml @@ -256,7 +256,9 @@ spec: mountPath: /opt/bitnami/airflow/webserver_config.py subPath: webserver_config.py {{- end }} + {{- if .Values.airflow.web.resources }} resources: {{- toYaml .Values.airflow.web.resources | nindent 12 }} + {{- end }} volumes: {{- if .Files.Glob "files/dags/*.py" }} - name: local-dag-files diff --git a/bitnami/airflow/templates/metrics-deployment.yaml b/bitnami/airflow/templates/metrics-deployment.yaml index a36fb5f0105d79..36ed0fd1fac465 100644 --- a/bitnami/airflow/templates/metrics-deployment.yaml +++ b/bitnami/airflow/templates/metrics-deployment.yaml @@ -67,5 +67,7 @@ spec: ports: - name: metrics containerPort: 9112 + {{- if .Values.metrics.resources}} resources: {{- toYaml .Values.metrics.resources | nindent 12 }} + {{- end }} {{- end }} diff --git a/bitnami/airflow/templates/statefulset-worker.yaml b/bitnami/airflow/templates/statefulset-worker.yaml index c8f5af90395593..1335aebf6f7f84 100644 --- a/bitnami/airflow/templates/statefulset-worker.yaml +++ b/bitnami/airflow/templates/statefulset-worker.yaml @@ -102,7 +102,9 @@ spec: - name: airflow-worker image: "{{ template "airflow.workerImage" . }}" imagePullPolicy: "{{ .Values.workerImage.pullPolicy }}" + {{- if .Values.airflow.worker.resources }} resources: {{- toYaml .Values.airflow.worker.resources | nindent 12 }} + {{- end }} env: {{- if .Values.workerImage.debug }} - name: BASH_DEBUG From 0e85557f3a52887bef6c40a9581951dda9c7596a Mon Sep 17 00:00:00 2001 From: mauraza Date: Wed, 19 Feb 2020 19:52:51 +0100 Subject: [PATCH 6/8] Fix typpo in readme --- bitnami/airflow/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 245b122a77bbac..8d3111dca6da11 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -101,10 +101,10 @@ The following tables lists the configurable parameters of the Airflow chart and | `airflow.auth.fernetKey` | Fernet key to secure connections | `nil` | | `airflow.auth.existingSecret` | Name of an existing secret containing airflow password and fernet key | `nil` | | `airflow.extraEnvVars` | Extra environment variables to add to airflow web, worker and scheduler pods | `nil` | -| `airflow.web.resources.limits` | The resources limits for the Worker containers | `{}` | -| `airflow.web.resources.requests` | The requested resources for the Worker containers | `{}` | -| `airflow.scheduler.resources.limits` | The resources limits for the Worker containers | `{}` | -| `airflow.scheduler.resources.requests` | The requested resources for the Worker containers | `{}` | +| `airflow.web.resources.limits` | The resources limits for the web containers | `{}` | +| `airflow.web.resources.requests` | The requested resources for the web containers | `{}` | +| `airflow.scheduler.resources.limits` | The resources limits for the scheduler containers | `{}` | +| `airflow.scheduler.resources.requests` | The requested resources for the scheduler containers | `{}` | | `airflow.webserverConfigMap` | Config map name for ~/airflow/webserver_config.py | `nil` | | `securityContext.enabled` | Enable security context | `true` | | `securityContext.fsGroup` | Group ID for the container | `1001` | From 128a5d434419a10a693c30b8d5c1ec475899d20e Mon Sep 17 00:00:00 2001 From: mauraza Date: Thu, 20 Feb 2020 10:02:16 +0100 Subject: [PATCH 7/8] Delete the old resources, delete a blank line in values-production and in values look in the same way to values-production --- bitnami/airflow/values-production.yaml | 1 - bitnami/airflow/values.yaml | 17 +++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index 5e441728efa5f6..46db3ff71aced9 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -199,7 +199,6 @@ airflow: # memory: 256Mi # cpu: 250m - ## All the parameters from the configuration file can be overwritten by using environment variables with this format: ## AIRFLOW__{SECTION}__{KEY}. Note the double underscores. More info at https://airflow.readthedocs.io/en/stable/howto/set-config.html ## diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index f3e0231086abbe..3266b72644ea51 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -143,9 +143,9 @@ airflow: # path: ## URL used to access to airflow web ui ## - worker: - # baseUrl: - ## Airflow worker component configuration + # baseUrl: + ## Airflow worker component configuration + worker: port: 8793 replicas: 1 ## Worker containers' resource requests and limits @@ -324,17 +324,6 @@ tolerations: [] ## affinity: {} -## Configure resource requests and limits -## ref: http://kubernetes.io/docs/user-guide/compute-resources/ -## -resources: -# limits: -# cpu: 200m -# memory: 1Gi -# requests: -# memory: 256Mi -# cpu: 250m - ## Configure extra options for liveness and readiness probes ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes) livenessProbe: From 42f882b1bb91ff971491cc412160ecd4d2bbe4bd Mon Sep 17 00:00:00 2001 From: mauraza Date: Thu, 20 Feb 2020 10:12:18 +0100 Subject: [PATCH 8/8] Changes in values-production and values --- bitnami/airflow/values-production.yaml | 20 ++++++++++---------- bitnami/airflow/values.yaml | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bitnami/airflow/values-production.yaml b/bitnami/airflow/values-production.yaml index 46db3ff71aced9..e0fad2d3ca82c7 100644 --- a/bitnami/airflow/values-production.yaml +++ b/bitnami/airflow/values-production.yaml @@ -125,10 +125,10 @@ airflow: ## Airflow generic configuration ## loadExamples: false - + ## Interval to pull the git repository containing the plugins and/or DAG files # - gitSyncInterval: 60 + gitSyncInterval: 60 ## Enable in order to download DAG files from git repository. ## cloneDagFilesFromGit: @@ -162,13 +162,13 @@ airflow: requests: {} # memory: 256Mi # cpu: 250m - + ## Airflow web specific configuration # web: - ## Web container' resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## + ## Web container' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## resources: # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -181,12 +181,12 @@ airflow: # memory: 256Mi # cpu: 250m - ## Scheduler containers specific configuration + ## Airflow scheduler specific configuration # scheduler: - ## Scheduler containers' resource requests and limits - ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## + ## Scheduler containers' resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## resources: # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 3266b72644ea51..80a86cd5aac71f 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -145,7 +145,7 @@ airflow: ## # baseUrl: ## Airflow worker component configuration - worker: + worker: port: 8793 replicas: 1 ## Worker containers' resource requests and limits @@ -162,6 +162,7 @@ airflow: requests: {} # memory: 256Mi # cpu: 250m + ## Airflow web specific configuration # web: @@ -179,6 +180,7 @@ airflow: requests: {} # memory: 256Mi # cpu: 250m + ## Airflow scheduler specific configuration # scheduler: