From 81ef4db757c67c994ebe31702d8d54dc42399a78 Mon Sep 17 00:00:00 2001 From: r-richmond Date: Sun, 15 Mar 2026 15:03:52 -0700 Subject: [PATCH] Add option for syncing to api-server --- chart/docs/manage-dag-files.rst | 1 + .../api-server/api-server-deployment.yaml | 22 +++++++ chart/values.schema.json | 5 ++ chart/values.yaml | 3 + .../airflow_core/test_api_server.py | 64 +++++++++++++++++++ 5 files changed, 95 insertions(+) diff --git a/chart/docs/manage-dag-files.rst b/chart/docs/manage-dag-files.rst index 3ab6529565435..d1a77009f6ce0 100644 --- a/chart/docs/manage-dag-files.rst +++ b/chart/docs/manage-dag-files.rst @@ -103,6 +103,7 @@ Mounting Dags using Git-Sync sidecar without persistence ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This option will always use running Git-Sync sidecar on every dag-processor, worker and triggerer pods +(and optionally on API server pods when ``dags.gitSync.syncToApiServer=true``) (In Airflow 2.11, if separate dag-processor is not enabled, the Git-Sync sidecar will run on scheduler for Dag parsing as well). The Git-Sync sidecar containers will sync Dags from a git repository every configured number of diff --git a/chart/templates/api-server/api-server-deployment.yaml b/chart/templates/api-server/api-server-deployment.yaml index 4ef8a5be3eae4..3caa43433bf94 100644 --- a/chart/templates/api-server/api-server-deployment.yaml +++ b/chart/templates/api-server/api-server-deployment.yaml @@ -30,6 +30,8 @@ {{- $containerSecurityContext := include "containerSecurityContext" (list .Values.apiServer .Values) }} {{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.apiServer.waitForMigrations .Values) }} {{- $containerLifecycleHooks := or .Values.apiServer.containerLifecycleHooks .Values.containerLifecycleHooks }} +{{- $syncGitToApiServer := and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer (not .Values.dags.persistence.enabled) }} +{{- $mountDagsToApiServer := and .Values.dags.gitSync.syncToApiServer (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }} apiVersion: apps/v1 kind: Deployment metadata: @@ -152,6 +154,9 @@ spec: {{- tpl (toYaml .Values.apiServer.waitForMigrations.env) $ | nindent 12 }} {{- end }} {{- end }} + {{- if $syncGitToApiServer }} + {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }} + {{- end }} {{- if .Values.apiServer.extraInitContainers }} {{- tpl (toYaml .Values.apiServer.extraInitContainers) . | nindent 8 }} {{- end }} @@ -182,6 +187,9 @@ spec: subPath: {{ .Values.logs.persistence.subPath }} {{- end }} {{- end }} + {{- if $mountDagsToApiServer }} + {{- include "airflow_dags_mount" . | nindent 12 }} + {{- end }} {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 12 }} {{- end }} @@ -223,6 +231,9 @@ spec: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" true) .) | indent 10 }} {{- include "container_extra_envs" (list . .Values.apiServer.env) | indent 10 }} + {{- if $syncGitToApiServer }} + {{- include "git_sync_container" . | indent 8 }} + {{- end }} {{- if .Values.apiServer.extraContainers }} {{- tpl (toYaml .Values.apiServer.extraContainers) . | nindent 8 }} {{- end }} @@ -240,6 +251,17 @@ spec: persistentVolumeClaim: claimName: {{ template "airflow_logs_volume_claim" . }} {{- end }} + {{- if and .Values.dags.persistence.enabled .Values.dags.gitSync.syncToApiServer }} + - name: dags + persistentVolumeClaim: + claimName: {{ template "airflow_dags_volume_claim" . }} + {{- else if and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer }} + - name: dags + emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }} + {{- end }} + {{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }} + {{- include "git_sync_ssh_key_volume" . | indent 8 }} + {{- end }} {{- if .Values.volumes }} {{- toYaml .Values.volumes | nindent 8 }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 31e6333e88538..7dac2a66744bf 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -10655,6 +10655,11 @@ "type": "boolean", "default": false }, + "syncToApiServer": { + "description": "Enable git sync and dags mount for API server pods (Airflow 3+).", + "type": "boolean", + "default": false + }, "repo": { "description": "Git repository.", "type": "string", diff --git a/chart/values.yaml b/chart/values.yaml index a5159c84faa52..89c0ad1a3229c 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -3589,6 +3589,9 @@ dags: gitSync: enabled: false + # enable git sync and dags mount for the api-server pods (Airflow 3+) + syncToApiServer: false + # git repo clone url # ssh example: git@github.com:apache/airflow.git # https example: https://github.com/apache/airflow.git diff --git a/helm-tests/tests/helm_tests/airflow_core/test_api_server.py b/helm-tests/tests/helm_tests/airflow_core/test_api_server.py index b8769976b83ff..97938c4819d98 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_api_server.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_api_server.py @@ -622,6 +622,70 @@ def test_api_server_pod_hostaliases(self): assert jmespath.search("spec.template.spec.hostAliases[0].ip", docs[0]) == "127.0.0.1" assert jmespath.search("spec.template.spec.hostAliases[0].hostnames[0]", docs[0]) == "foo.local" + def test_dags_gitsync_not_added_by_default(self): + docs = render_chart( + show_only=["templates/api-server/api-server-deployment.yaml"], + ) + + container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])] + init_container_names = [ + c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) + ] + volume_mount_names = [ + vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + ] + volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])] + + assert "git-sync" not in container_names + assert "git-sync-init" not in init_container_names + assert "dags" not in volume_mount_names + assert "dags" not in volume_names + + def test_dags_gitsync_sidecar_and_init_container_when_enabled(self): + docs = render_chart( + values={"dags": {"gitSync": {"enabled": True, "syncToApiServer": True}}}, + show_only=["templates/api-server/api-server-deployment.yaml"], + ) + + container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])] + init_container_names = [ + c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) + ] + volume_mount_names = [ + vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + ] + volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])] + + assert "git-sync" in container_names + assert "git-sync-init" in init_container_names + assert "dags" in volume_mount_names + assert "dags" in volume_names + + def test_dags_gitsync_with_persistence_no_sidecar_or_init_container(self): + docs = render_chart( + values={ + "dags": { + "gitSync": {"enabled": True, "syncToApiServer": True}, + "persistence": {"enabled": True}, + } + }, + show_only=["templates/api-server/api-server-deployment.yaml"], + ) + + container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])] + init_container_names = [ + c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0]) + ] + volume_mount_names = [ + vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + ] + volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])] + + assert "git-sync" not in container_names + assert "git-sync-init" not in init_container_names + assert "dags" in volume_mount_names + assert "dags" in volume_names + def test_can_be_disabled(self): """ API server can be disabled by configuration.