[bitnami/airflow] Add support for plugins loaded from a git repo

bitnami/airflow/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: airflow
-version: 4.2.1
+version: 4.3.0
 appVersion: 1.10.9
 description: Apache Airflow is a platform to programmatically author, schedule and monitor workflows.
 keywords:

bitnami/airflow/README.md

@@ -81,11 +81,15 @@ The following tables lists the configurable parameters of the Kafka chart and th
 | `airflow.configurationConfigMap`          | Name of an existing config map containing the Airflow config file                                    | `nil`                                                        |
 | `airflow.dagsConfigMap`                   | Name of an existing config map containing all the DAGs files you want to load in Airflow.            | `nil`                                                        |
 | `airflow.loadExamples`                    | Switch to load some Airflow examples                                                                 | `true`                                                       |
+| `airflow.gitSyncInterval`                 | Interval (in seconds) to pull the git repository containing the plugins and/or DAG files             | `60`                                                         |
 | `airflow.cloneDagFilesFromGit.enabled`    | Enable in order to download DAG files from git repository.                                           | `false`                                                      |
 | `airflow.cloneDagFilesFromGit.repository` | Repository where download DAG files from                                                             | `nil`                                                        |
 | `airflow.cloneDagFilesFromGit.branch`     | Branch from repository to checkout                                                                   | `nil`                                                        |
-| `airflow.cloneDagFilesFromGit.interval`   | Interval to pull the repository on sidecar container                                                 | `nil`                                                        |
 | `airflow.cloneDagFilesFromGit.path`       | Path to a folder in the repository containing DAGs. If not set, all DAGS from the repo are loaded.   | `nil`                                                        |
+| `airflow.clonePluginsFromGit.enabled`     | Enable in order to download plugins from git repository.                                             | `false`                                                      |
+| `airflow.clonePluginsFromGit.repository`  | Repository where download plugins from                                                               | `nil`                                                        |
+| `airflow.clonePluginsFromGit.branch`      | Branch from repository to checkout                                                                   | `nil`                                                        |
+| `airflow.clonePluginsFromGit.path`        | Path to a folder in the repository containing the plugins.                                           | `nil`                                                        |
 | `airflow.baseUrl`                         | URL used to access to airflow web ui                                                                 | `nil`                                                        |
 | `airflow.worker.port`                     | Airflow Worker port                                                                                  | `8793`                                                       |
 | `airflow.worker.replicas`                 | Number of Airflow Worker replicas                                                                    | `2`                                                          |
@@ -186,24 +190,28 @@ Bitnami will release a new chart updating its containers if a new version of the
 This chart includes a `values-production.yaml` file where you can find some parameters oriented to production configuration in comparison to the regular `values.yaml`. You can use this file instead of the default one.
 
 - URL used to access to airflow web ui:
+
 ```diff
 - # airflow.baseUrl:
 + airflow.baseUrl: http://airflow.local
 ```
 
 - Number of Airflow Worker replicas:
+
 ```diff
 - airflow.worker.replicas: 1
 + airflow.worker.replicas: 3
 ```
 
 - Force users to specify a password:
+
 ```diff
 - airflow.auth.forcePassword: false
 + airflow.auth.forcePassword: true
 ```
 
 - Enable ingress controller resource:
+
 ```diff
 - ingress.enabled: false
 + ingress.enabled: true
@@ -235,7 +243,17 @@ You can store all your DAG files on a GitHub repository and then clone to the Ai
 airflow.cloneDagFilesFromGit.enabled=true
 airflow.cloneDagFilesFromGit.repository=https://github.com/USERNAME/REPOSITORY
 airflow.cloneDagFilesFromGit.branch=master
-airflow.cloneDagFilesFromGit.interval=60
+```
+
+### Loading Plugins
+
+You can load plugins into the chart by specifying a git repository containing the plugin files. The repository will be periodically updated using a sidecar container. In order to do that, you can deploy airflow with the following options:
+
+```console
+airflow.clonePluginsFromGit.enabled=true
+airflow.clonePluginsFromGit.repository=https://github.com/teamclairvoyant/airflow-rest-api-plugin.git
+airflow.clonePluginsFromGit.branch=v1.0.9-branch
+airflow.clonePluginsFromGit.path=plugins
 ```
 
 ## Persistence

bitnami/airflow/templates/_helpers.tpl

@@ -266,6 +266,23 @@ airflow: airflow.cloneDagFilesFromGit.branch
 {{- end -}}
 {{- end -}}
 
+{{/* Validate values of Airflow - "airflow.clonePluginsFromGit.repository" must be provided when "airflow.clonePluginsFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.clonePluginsFromGit.repository" -}}
+{{- if and .Values.airflow.clonePluginsFromGit.enabled (empty .Values.airflow.clonePluginsFromGit.repository) -}}
+airflow: airflow.clonePluginsFromGit.repository
+    The repository must be provided when enabling downloading plugins
+    from git repository (--set airflow.clonePluginsFromGit.repository="xxx")
+{{- end -}}
+{{- end -}}
+{{/* Validate values of Airflow - "airflow.clonePluginsFromGit.branch" must be provided when "airflow.clonePluginsFromGit.enabled" is "true" */}}
+{{- define "airflow.validateValues.clonePluginsFromGit.branch" -}}
+{{- if and .Values.airflow.clonePluginsFromGit.enabled (empty .Values.airflow.clonePluginsFromGit.branch) -}}
+airflow: airflow.clonePluginsFromGit.branch
+    The branch must be provided when enabling downloading plugins
+    from git repository (--set airflow.clonePluginsFromGit.branch="xxx")
+{{- end -}}
+{{- end -}}
+
 {{/* Check if there are rolling tags in the images */}}
 {{- define "airflow.checkRollingTags" -}}
 {{- if and (contains "bitnami/" .Values.image.repository) (not (.Values.image.tag | toString | regexFind "-r\\d+$|sha256:")) }}

bitnami/airflow/templates/deployment-scheduler.yaml

@@ -45,7 +45,7 @@ spec:
         fsGroup: {{ .Values.securityContext.fsGroup }}
         runAsUser: {{ .Values.securityContext.runAsUser }}
       {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
       initContainers:
         - name: git-clone-repository
           image: "{{ template "git.image" . }}"
@@ -54,10 +54,21 @@ spec:
             - /bin/bash
             - -ec
             - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       containers:
         - name: git-repo-syncer
           image: "{{ template "git.image" . }}"
@@ -67,12 +78,23 @@ spec:
             - -ec
             - |
               while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                   cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
               done
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       {{- else }}
       containers:
       {{- end }}
@@ -188,6 +210,13 @@ spec:
               subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
               {{- end }}
           {{- end }}
+          {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+          {{- end }}
           {{- if .Values.airflow.configurationConfigMap }}
             - name: custom-configuration-file
               mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -209,6 +238,10 @@ spec:
         - name: git-cloned-dag-files
           emptyDir: {}
       {{- end }}
+      {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+        - name: git-cloned-plugins
+          emptyDir: {}
+      {{- end }}
       {{- if .Values.airflow.configurationConfigMap }}
         - name: custom-configuration-file
           configMap:

bitnami/airflow/templates/deployment-web.yaml

@@ -45,7 +45,7 @@ spec:
         fsGroup: {{ .Values.securityContext.fsGroup }}
         runAsUser: {{ .Values.securityContext.runAsUser }}
       {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
       initContainers:
         - name: git-clone-repository
           image: "{{ template "git.image" . }}"
@@ -54,10 +54,21 @@ spec:
             - /bin/bash
             - -ec
             - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       containers:
         - name: git-repo-syncer
           image: "{{ template "git.image" . }}"
@@ -67,12 +78,23 @@ spec:
             - -ec
             - |
               while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                   cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
               done
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       {{- else }}
       containers:
       {{- end }}
@@ -218,6 +240,13 @@ spec:
               subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
               {{- end }}
             {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+            {{- end }}
             {{- if .Values.airflow.configurationConfigMap }}
             - name: custom-configuration-file
               mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -244,6 +273,10 @@ spec:
         - name: git-cloned-dag-files
           emptyDir: {}
         {{- end }}
+        {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+        - name: git-cloned-plugins
+          emptyDir: {}
+        {{- end }}
         {{- if .Values.airflow.configurationConfigMap }}
         - name: custom-configuration-file
           configMap:

bitnami/airflow/templates/statefulset-worker.yaml

@@ -46,7 +46,7 @@ spec:
       {{- if .Values.affinity }}
       affinity: {{- toYaml .Values.affinity | nindent 8 }}
       {{- end }}
-      {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+      {{- if or .Values.airflow.cloneDagFilesFromGit.enabled .Values.airflow.clonePluginsFromGit.enabled }}
       initContainers:
         - name: git-clone-repository
           image: "{{ template "git.image" . }}"
@@ -55,10 +55,21 @@ spec:
             - /bin/bash
             - -ec
             - |
-              git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
+                git clone {{ .Values.airflow.cloneDagFilesFromGit.repository }} --branch {{ .Values.airflow.cloneDagFilesFromGit.branch }} /dags
+              {{- end }}
+              {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                git clone {{ .Values.airflow.clonePluginsFromGit.repository }} --branch {{ .Values.airflow.clonePluginsFromGit.branch }} /plugins
+              {{- end }}
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       containers:
         - name: git-repo-syncer
           image: "{{ template "git.image" . }}"
@@ -68,12 +79,23 @@ spec:
             - -ec
             - |
               while true; do
+                  {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
                   cd /dags && git pull origin {{ .Values.airflow.cloneDagFilesFromGit.branch }}
-                  sleep {{ default "60" .Values.airflow.cloneDagFilesFromGit.interval }}
+                  {{- end }}
+                  {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+                  cd /plugins && git pull origin {{ .Values.airflow.clonePluginsFromGit.branch }}
+                  {{- end }}
+                  sleep {{ default "60" .Values.airflow.gitSyncInterval }}
               done
           volumeMounts:
+            {{- if .Values.airflow.cloneDagFilesFromGit.enabled }}
             - name: git-cloned-dag-files
               mountPath: /dags
+            {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /plugins
+            {{- end }}
       {{- else }}
       containers:
       {{- end }}
@@ -201,6 +223,13 @@ spec:
               subPath: {{ .Values.airflow.cloneDagFilesFromGit.path }}
               {{- end }}
             {{- end }}
+            {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+            - name: git-cloned-plugins
+              mountPath: /opt/bitnami/airflow/plugins
+              {{- if .Values.airflow.clonePluginsFromGit.path }}
+              subPath: {{ .Values.airflow.clonePluginsFromGit.path }}
+              {{- end }}
+            {{- end }}
             {{- if .Values.airflow.configurationConfigMap }}
             - name: custom-configuration-file
               mountPath: /opt/bitnami/airflow/airflow.cfg
@@ -221,6 +250,10 @@ spec:
       - name: git-cloned-dag-files
         emptyDir: {}
       {{- end }}
+      {{- if .Values.airflow.clonePluginsFromGit.enabled }}
+      - name: git-cloned-plugins
+        emptyDir: {}
+      {{- end }}
       {{- if .Values.airflow.configurationConfigMap }}
       - name: custom-configuration-file
         configMap:

bitnami/airflow/values-production.yaml

@@ -125,13 +125,21 @@ airflow:
   ## Airflow generic configuration
   ##
   loadExamples: false
+
+  ## Interval to pull the git repository containing the plugins and/or DAG files
+  #
+  gitSyncInterval: 60  
   ## Enable in order to download DAG files from git repository.
   ##
   cloneDagFilesFromGit:
     enabled: false
     # repository:
     # branch:
-    # interval:
+    # path:
+  clonePluginsFromGit:
+    enabled: false
+    # repository:
+    # branch:
     # path:
   ## URL used to access to airflow web ui
   ##

bitnami/airflow/values.yaml

@@ -125,13 +125,21 @@ airflow:
   ## Airflow generic configuration
   ##
   loadExamples: false
+
+  ## Interval to pull the git repository containing the plugins and/or DAG files
+  #
+  gitSyncInterval: 60
   ## Enable in order to download DAG files from git repository.
   ##
   cloneDagFilesFromGit:
     enabled: false
     # repository:
     # branch:
-    # interval:
+    # path:
+  clonePluginsFromGit:
+    enabled: false
+    # repository:
+    # branch:
     # path:
   ## URL used to access to airflow web ui
   ##