Skip to content

Add boolean helm chart option dags.gitSync.syncToApiServer #63677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
r-richmond wants to merge 1 commit into from
Closed

Add boolean helm chart option dags.gitSync.syncToApiServer #63677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions chart/docs/manage-dag-files.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ Mounting Dags using Git-Sync sidecar without persistence
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

This option will always use running Git-Sync sidecar on every dag-processor, worker and triggerer pods
(and optionally on API server pods when ``dags.gitSync.syncToApiServer=true``)
(In Airflow 2.11, if separate dag-processor is not enabled, the Git-Sync sidecar will run on scheduler for Dag parsing as well).

The Git-Sync sidecar containers will sync Dags from a git repository every configured number of
Expand Down
22 changes: 22 additions & 0 deletions chart/templates/api-server/api-server-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
{{- $containerSecurityContext := include "containerSecurityContext" (list .Values.apiServer .Values) }}
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list .Values.apiServer.waitForMigrations .Values) }}
{{- $containerLifecycleHooks := or .Values.apiServer.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $syncGitToApiServer := and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer (not .Values.dags.persistence.enabled) }}
{{- $mountDagsToApiServer := and .Values.dags.gitSync.syncToApiServer (or .Values.dags.persistence.enabled .Values.dags.gitSync.enabled) }}
apiVersion: apps/v1
kind: Deployment
metadata:
Expand Down Expand Up @@ -152,6 +154,9 @@ spec:
{{- tpl (toYaml .Values.apiServer.waitForMigrations.env) $ | nindent 12 }}
{{- end }}
{{- end }}
{{- if $syncGitToApiServer }}
{{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 8 }}
{{- end }}
{{- if .Values.apiServer.extraInitContainers }}
{{- tpl (toYaml .Values.apiServer.extraInitContainers) . | nindent 8 }}
{{- end }}
Expand Down Expand Up @@ -182,6 +187,9 @@ spec:
subPath: {{ .Values.logs.persistence.subPath }}
{{- end }}
{{- end }}
{{- if $mountDagsToApiServer }}
{{- include "airflow_dags_mount" . | nindent 12 }}
{{- end }}
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
Expand Down Expand Up @@ -223,6 +231,9 @@ spec:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" true) .) | indent 10 }}
{{- include "container_extra_envs" (list . .Values.apiServer.env) | indent 10 }}
{{- if $syncGitToApiServer }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
{{- if .Values.apiServer.extraContainers }}
{{- tpl (toYaml .Values.apiServer.extraContainers) . | nindent 8 }}
{{- end }}
Expand All @@ -240,6 +251,17 @@ spec:
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- end }}
{{- if and .Values.dags.persistence.enabled .Values.dags.gitSync.syncToApiServer }}
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer }}
- name: dags
emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.syncToApiServer (or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey) }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
Expand Down
5 changes: 5 additions & 0 deletions chart/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10655,6 +10655,11 @@
"type": "boolean",
"default": false
},
"syncToApiServer": {
"description": "Enable git sync and dags mount for API server pods (Airflow 3+).",
"type": "boolean",
"default": false
},
"repo": {
"description": "Git repository.",
"type": "string",
Expand Down
3 changes: 3 additions & 0 deletions chart/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3589,6 +3589,9 @@ dags:
gitSync:
enabled: false

# enable git sync and dags mount for the api-server pods (Airflow 3+)
syncToApiServer: false

# git repo clone url
# ssh example: git@github.com:apache/airflow.git
# https example: https://github.com/apache/airflow.git
Expand Down
64 changes: 64 additions & 0 deletions helm-tests/tests/helm_tests/airflow_core/test_api_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,6 +622,70 @@ def test_api_server_pod_hostaliases(self):
assert jmespath.search("spec.template.spec.hostAliases[0].ip", docs[0]) == "127.0.0.1"
assert jmespath.search("spec.template.spec.hostAliases[0].hostnames[0]", docs[0]) == "foo.local"

def test_dags_gitsync_not_added_by_default(self):
docs = render_chart(
show_only=["templates/api-server/api-server-deployment.yaml"],
)

container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])]
init_container_names = [
c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0])
]
volume_mount_names = [
vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
]
volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])]

assert "git-sync" not in container_names
assert "git-sync-init" not in init_container_names
assert "dags" not in volume_mount_names
assert "dags" not in volume_names

def test_dags_gitsync_sidecar_and_init_container_when_enabled(self):
docs = render_chart(
values={"dags": {"gitSync": {"enabled": True, "syncToApiServer": True}}},
show_only=["templates/api-server/api-server-deployment.yaml"],
)

container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])]
init_container_names = [
c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0])
]
volume_mount_names = [
vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
]
volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])]

assert "git-sync" in container_names
assert "git-sync-init" in init_container_names
assert "dags" in volume_mount_names
assert "dags" in volume_names

def test_dags_gitsync_with_persistence_no_sidecar_or_init_container(self):
docs = render_chart(
values={
"dags": {
"gitSync": {"enabled": True, "syncToApiServer": True},
"persistence": {"enabled": True},
}
},
show_only=["templates/api-server/api-server-deployment.yaml"],
)

container_names = [c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0])]
init_container_names = [
c["name"] for c in jmespath.search("spec.template.spec.initContainers", docs[0])
]
volume_mount_names = [
vm["name"] for vm in jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0])
]
volume_names = [v["name"] for v in jmespath.search("spec.template.spec.volumes", docs[0])]

assert "git-sync" not in container_names
assert "git-sync-init" not in init_container_names
assert "dags" in volume_mount_names
assert "dags" in volume_names

def test_can_be_disabled(self):
"""
API server can be disabled by configuration.
Expand Down
Loading