From 1e7d0dd7dff09ffbc7be4ebf6a9f2df10f05d46b Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Wed, 23 Apr 2025 14:37:57 +0000 Subject: [PATCH 01/13] [bitnami/airflow] Release 23.0.0 updating components versions Signed-off-by: Bitnami Bot --- bitnami/airflow/Chart.lock | 6 +++--- bitnami/airflow/Chart.yaml | 6 +++--- bitnami/airflow/values.yaml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitnami/airflow/Chart.lock b/bitnami/airflow/Chart.lock index 2fbb7891b138f0..670ab5d6392892 100644 --- a/bitnami/airflow/Chart.lock +++ b/bitnami/airflow/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: redis repository: oci://registry-1.docker.io/bitnamicharts - version: 20.12.1 + version: 20.13.1 - name: postgresql repository: oci://registry-1.docker.io/bitnamicharts version: 16.6.3 - name: common repository: oci://registry-1.docker.io/bitnamicharts version: 2.30.0 -digest: sha256:8cd79ef253d7a517de839fd46e82e8b482a5c790cd4adc0b901f5fd878f42a02 -generated: "2025-04-21T01:13:19.809190595Z" +digest: sha256:9da8ff790e57d014ce6015b9dddb7de1dfddf14e592cdd148dd07fb760b1340b +generated: "2025-04-23T14:37:53.595077325Z" diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml index 012083660f9e6b..25f6e49013c3b8 100644 --- a/bitnami/airflow/Chart.yaml +++ b/bitnami/airflow/Chart.yaml @@ -7,11 +7,11 @@ annotations: licenses: Apache-2.0 images: | - name: airflow - image: docker.io/bitnami/airflow:2.10.5-debian-12-r11 + image: docker.io/bitnami/airflow:3.0.0-debian-12-r0 - name: statsd-exporter image: docker.io/bitnami/statsd-exporter:0.28.0-debian-12-r15 apiVersion: v2 -appVersion: 2.10.5 +appVersion: 3.0.0 dependencies: - condition: redis.enabled name: redis @@ -40,4 +40,4 @@ maintainers: name: airflow sources: - https://github.com/bitnami/charts/tree/main/bitnami/airflow -version: 22.7.3 +version: 23.0.0 diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 6f97e0167a6bfd..2ade0fdfe84d44 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -94,7 +94,7 @@ diagnosticMode: image: registry: docker.io repository: bitnami/airflow - tag: 2.10.5-debian-12-r11 + tag: 3.0.0-debian-12-r0 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From fbf6996b2b9ca2867e871d1618324f4ecee7d2c8 Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Wed, 23 Apr 2025 14:45:50 +0000 Subject: [PATCH 02/13] Update CHANGELOG.md Signed-off-by: Bitnami Bot --- bitnami/airflow/CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bitnami/airflow/CHANGELOG.md b/bitnami/airflow/CHANGELOG.md index 68d52c9cb3fea3..7b492f5e75e5a2 100644 --- a/bitnami/airflow/CHANGELOG.md +++ b/bitnami/airflow/CHANGELOG.md @@ -1,8 +1,12 @@ # Changelog -## 22.7.3 (2025-04-21) +## 23.0.0 (2025-04-23) -* [bitnami/airflow] Release 22.7.3 ([#33087](https://github.com/bitnami/charts/pull/33087)) +* [bitnami/airflow] Release 23.0.0 ([#33141](https://github.com/bitnami/charts/pull/33141)) + +## 22.7.3 (2025-04-21) + +* [bitnami/airflow] Release 22.7.3 (#33087) ([c62ad9b](https://github.com/bitnami/charts/commit/c62ad9b97dad33d09d2888d21066d9d25cafeed5)), closes [#33087](https://github.com/bitnami/charts/issues/33087) ## 22.7.2 (2025-03-25) From 4e7fdcb2f3680d9d6291b81a6f5ed9b1c24393c2 Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 14:04:26 +0200 Subject: [PATCH 03/13] [bitnami/airflow] Update chart logic and tests for Airflow 3.x Signed-off-by: Gonzalo Gomez Gracia --- .vib/airflow/cypress/cypress.config.js | 8 +++- .../airflow/cypress/cypress/e2e/airflow.cy.js | 22 ++++++----- .../cypress/cypress/support/commands.js | 23 +++++++---- .vib/airflow/runtime-parameters.yaml | 5 +++ .vib/airflow/vib-verify.json | 3 +- bitnami/airflow/Chart.yaml | 2 +- bitnami/airflow/README.md | 8 +++- bitnami/airflow/templates/_helpers.tpl | 11 ++++++ .../templates/_init_containers_sidecars.tpl | 39 ++++++++++++++++--- .../airflow/templates/config/configmap.yaml | 16 +++++++- bitnami/airflow/templates/config/secret.yaml | 7 ++++ .../templates/dag-processor/deployment.yaml | 19 +++++---- .../templates/scheduler/deployment.yaml | 19 +++++---- bitnami/airflow/templates/setup-db-job.yaml | 10 +++++ .../templates/triggerer/statefulset.yaml | 19 +++++---- bitnami/airflow/templates/web/configmap.yaml | 4 ++ bitnami/airflow/templates/web/deployment.yaml | 15 ++++++- bitnami/airflow/templates/web/service.yaml | 4 +- .../airflow/templates/worker/statefulset.yaml | 7 ++++ bitnami/airflow/values.yaml | 12 ++++-- 20 files changed, 197 insertions(+), 56 deletions(-) diff --git a/.vib/airflow/cypress/cypress.config.js b/.vib/airflow/cypress/cypress.config.js index c735ac508c48a6..c520f589a1640a 100644 --- a/.vib/airflow/cypress/cypress.config.js +++ b/.vib/airflow/cypress/cypress.config.js @@ -1,12 +1,18 @@ module.exports = { + viewportWidth: 1920, + viewportHeight: 1080, + chromeWebSecurity: false, pageLoadTimeout: 240000, defaultCommandTimeout: 80000, env: { username: 'user', password: 'ComplicatedPassword123!4', + baseUrl: 'http://vmware-airflow.my', }, e2e: { setupNodeEvents(on, config) {}, - baseUrl: 'http://localhost', + }, + hosts: { + 'vmware-airflow.my': '{{ TARGET_IP }}', }, } diff --git a/.vib/airflow/cypress/cypress/e2e/airflow.cy.js b/.vib/airflow/cypress/cypress/e2e/airflow.cy.js index be8e448e21d72a..7569233681c9f6 100644 --- a/.vib/airflow/cypress/cypress/e2e/airflow.cy.js +++ b/.vib/airflow/cypress/cypress/e2e/airflow.cy.js @@ -9,18 +9,22 @@ import { random } from '../support/utils'; it('allows triggering execution of a sample DAG', () => { cy.login(); cy.fixture('DAGs').then((dags) => { - cy.visit(`dags/${dags.triggered.id}/grid`); - cy.get('[aria-label="Trigger DAG"]').click({force: true}); + cy.visit(`/dags/${dags.triggered.id}`); + cy.get('[aria-label="Trigger Dag"]').click({force: true}); + // Trigger button text is prefixed by a white space + cy.get('button').contains(' Trigger').click({force: true}); - // Verify the DAG appears in the list of active jobs - cy.visit('home?status=active'); - cy.get(`[href='/dags/${dags.triggered.id}/grid']`); + // Verify the DAG was scheduled + cy.wait(3000); + cy.visit('/dags'); + cy.get(`[href='/dags/${dags.triggered.id}']`); + cy.get('div').contains(new Date().toISOString().split('T')[0]); }); }); it('allows to create a user', () => { cy.login(); - cy.visit('users/add'); + cy.visit('/auth/users/add'); cy.fixture('users').then((users) => { cy.get('#first_name').type(users.newUser.firstName); cy.get('#last_name').type(users.newUser.lastName); @@ -30,8 +34,8 @@ it('allows to create a user', () => { cy.get('#password').type(users.newUser.password); cy.get('#conf_password').type(users.newUser.password); cy.contains('Save').click(); - - // Verify the user was created successfully - cy.contains('div', 'Added Row'); }); + // Verify the user was created successfully + cy.visit('/auth/users/add'); + cy.contains('div', 'Added Row'); }); diff --git a/.vib/airflow/cypress/cypress/support/commands.js b/.vib/airflow/cypress/cypress/support/commands.js index b98bbdac1c733a..76185f667ab5c9 100644 --- a/.vib/airflow/cypress/cypress/support/commands.js +++ b/.vib/airflow/cypress/cypress/support/commands.js @@ -17,10 +17,22 @@ for (const command of ['click']) { }); } +// Due to a bug when using "hosts" in Cypress, we cannot set a "baseUrl" in the +// cypress.json file. Workaround this by modifying the "visit" command to preprend +// the base URL. +// +// Further details: https://github.com/cypress-io/cypress/issues/20647 +Cypress.Commands.overwrite('visit', (originalFn, url, options) => { + // Only replace relative URLs + const targetUrl = url.includes('://') ? url : `${Cypress.env('baseUrl')}${url}`; + return originalFn(targetUrl, options); +}); + + Cypress.Commands.add( 'login', (username = Cypress.env('username'), password = Cypress.env('password')) => { - cy.visit('/login'); + cy.visit('/'); // Wait for DOM content to load cy.wait(5000); cy.get('form[name="login"]').should('exist').and('be.visible'); // Needed to ensure stability of the test @@ -31,9 +43,6 @@ Cypress.Commands.add( ); Cypress.on('uncaught:exception', (err, runnable) => { - if ( - err.message.includes('Cannot set properties of undefined') - ) { - return false; - } -}); \ No newline at end of file + // Skip all exceptions + return false; +}); diff --git a/.vib/airflow/runtime-parameters.yaml b/.vib/airflow/runtime-parameters.yaml index df4dd3d41331c6..9be206d0bf42c8 100644 --- a/.vib/airflow/runtime-parameters.yaml +++ b/.vib/airflow/runtime-parameters.yaml @@ -30,6 +30,7 @@ scheduler: periodSeconds: 30 timeoutSeconds: 20 web: + baseUrl: 'http://vmware-airflow.my' extraConfiguration: | # Theme CONFIG APP_THEME = "amelia.css" @@ -39,6 +40,10 @@ web: fsGroup: 1002 containerSecurityContext: runAsUser: 1002 + # Give some extra time for the service to start + startupProbe: + failureThreshold: 30 + periodSeconds: 10 worker: podSecurityContext: fsGroup: 1002 diff --git a/.vib/airflow/vib-verify.json b/.vib/airflow/vib-verify.json index 07e0bdc45a147c..318bd13d0e85a8 100644 --- a/.vib/airflow/vib-verify.json +++ b/.vib/airflow/vib-verify.json @@ -60,8 +60,9 @@ "resources": { "path": "/.vib/airflow/cypress" }, - "endpoint": "lb-airflow-http", + "endpoint": "lb-airflow-web-http", "app_protocol": "HTTP", + "port": 80, "env": { "username": "user", "password": "ComplicatedPassword123!4" diff --git a/bitnami/airflow/Chart.yaml b/bitnami/airflow/Chart.yaml index 25f6e49013c3b8..d3c8e1a17494d8 100644 --- a/bitnami/airflow/Chart.yaml +++ b/bitnami/airflow/Chart.yaml @@ -7,7 +7,7 @@ annotations: licenses: Apache-2.0 images: | - name: airflow - image: docker.io/bitnami/airflow:3.0.0-debian-12-r0 + image: docker.io/bitnami/airflow:3.0.0-debian-12-r2 - name: statsd-exporter image: docker.io/bitnami/statsd-exporter:0.28.0-debian-12-r15 apiVersion: v2 diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 4d297e27d35174..1592bc21b6b2fe 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -99,7 +99,7 @@ Bitnami charts configure credentials at first boot. Any further change in the se - Update the password secret with the new values (replace the SECRET_NAME, PASSWORD, FERNET_KEY and SECRET_KEY placeholders) ```shell -kubectl create secret generic SECRET_NAME --from-literal=airflow-password=PASSWORD --from-literal=airflow-fernet-key=FERNET_KEY --from-literal=airflow-secret-key=SECRET_KEY --dry-run -o yaml | kubectl apply -f - +kubectl create secret generic SECRET_NAME --from-literal=airflow-password=PASSWORD --from-literal=airflow-fernet-key=FERNET_KEY --from-literal=airflow-secret-key=SECRET_KEY --from-literal=airflow-jwt-secret-key=JWT_SECRET_KEY --dry-run -o yaml | kubectl apply -f - ``` ### Airflow configuration file @@ -1287,6 +1287,10 @@ Find more information about how to deal with common errors related to Bitnami's ## Upgrading +### To 23.0.0 + +This major release adds support for Airflow 3 versions. Additionally, previous Airflow 2 images can be deployed by setting the new `image.majorVersion=2` option and updating the rest of the image related parameters accordingly. + ### To 22.4.0 This version introduces image verification for security purposes. To disable it, set `global.security.allowInsecureImages` to `true`. More details at [GitHub issue](https://github.com/bitnami/charts/issues/30850). @@ -1526,4 +1530,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index c1fe887de5125b..0131091d1b2455 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -352,6 +352,10 @@ Add environment variables to configure airflow common values value: "cat /opt/bitnami/airflow/secrets/airflow-fernet-key" - name: AIRFLOW__WEBSERVER__SECRET_KEY_CMD value: "cat /opt/bitnami/airflow/secrets/airflow-secret-key" +{{- if eq (.Values.image.majorVersion | toString) "3" }} +- name: AIRFLOW__API_AUTH__JWT_SECRET_CMD + value: "cat /opt/bitnami/airflow/secrets/airflow-jwt-secret-key" +{{- end }} {{- else }} - name: AIRFLOW__CORE__FERNET_KEY valueFrom: @@ -363,6 +367,13 @@ Add environment variables to configure airflow common values secretKeyRef: name: {{ include "airflow.secretName" . }} key: airflow-secret-key +{{- if eq (.Values.image.majorVersion | toString) "3" }} +- name: AIRFLOW__API_AUTH__JWT_SECRET_CMD + valueFrom: + secretKeyRef: + name: {{ include "airflow.secretName" . }} + key: airflow-jwt-secret-key +{{- end -}} {{- end -}} {{- end -}} diff --git a/bitnami/airflow/templates/_init_containers_sidecars.tpl b/bitnami/airflow/templates/_init_containers_sidecars.tpl index 7b8431f9d7ebd9..a9a5740e3d3260 100644 --- a/bitnami/airflow/templates/_init_containers_sidecars.tpl +++ b/bitnami/airflow/templates/_init_containers_sidecars.tpl @@ -60,6 +60,8 @@ Returns an init-container that prepares the Airflow configuration files for main {{- end }} airflow_conf_set "celery" "broker_url" "redis://${redis_credentials}@${REDIS_HOST}:${REDIS_PORT_NUMBER}/${REDIS_DATABASE}" {{- end }} + # Configure authentication backend + airflow_conf_set "core" "auth_manager" "airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager" info "Airflow configuration ready" if [[ -f "/opt/bitnami/airflow/config/airflow_local_settings.py" ]]; then @@ -125,6 +127,9 @@ Returns an init-container that prepares the Airflow configuration files for main key: redis-password {{- end }} {{- end }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 4 }} + {{- end }} volumeMounts: - name: empty-dir mountPath: /emptydir @@ -187,6 +192,9 @@ Returns an init-container that prepares the Airflow Webserver configuration file key: bind-password {{- end }} {{- end }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 4 }} + {{- end }} volumeMounts: - name: empty-dir mountPath: /emptydir @@ -224,9 +232,16 @@ Returns an init-container that waits for db migrations to be ready info "Waiting for db migrations to be completed" airflow_wait_for_db_migrations + {{- if eq (.Values.image.majorVersion | toString) "3" }} + info "Waiting for the admin user to exist" + airflow_wait_for_admin_user + {{- end }} env: - name: BITNAMI_DEBUG value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 4 }} + {{- end }} volumeMounts: - name: empty-dir mountPath: /tmp @@ -276,9 +291,15 @@ Returns shared structure between load-dags and load-plugins init containers {{- else }} command: ["/bin/bash"] {{- end }} - {{- if .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars }} - env: {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars "context" .) | nindent 4 }} - {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + {{- if .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVars "context" .) | nindent 4 }} + {{- end }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 4 }} + {{- end }} {{- if or .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVarsCM .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVarsSecret }} envFrom: {{- if .Values.defaultInitContainers.loadDAGsPlugins.extraEnvVarsCM }} @@ -417,9 +438,15 @@ Returns shared structure between sync-dags and sync-plugins sidecars {{- else }} command: ["/bin/bash"] {{- end }} - {{- if .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars }} - env: {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars "context" .) | nindent 4 }} - {{- end }} + env: + - name: BITNAMI_DEBUG + value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }} + {{- if .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.defaultSidecars.syncDAGsPlugins.extraEnvVars "context" .) | nindent 4 }} + {{- end }} + {{- if .Values.extraEnvVars }} + {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 4 }} + {{- end }} {{- if or .Values.defaultSidecars.syncDAGsPlugins.extraEnvVarsCM .Values.defaultSidecars.syncDAGsPlugins.extraEnvVarsSecret }} envFrom: {{- if .Values.defaultSidecars.syncDAGsPlugins.extraEnvVarsCM }} diff --git a/bitnami/airflow/templates/config/configmap.yaml b/bitnami/airflow/templates/config/configmap.yaml index ccd56c46887878..543809908d83a7 100644 --- a/bitnami/airflow/templates/config/configmap.yaml +++ b/bitnami/airflow/templates/config/configmap.yaml @@ -14,6 +14,9 @@ ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.ht core: load_examples: {{ ternary "True" "False" .Values.loadExamples | squote }} executor: {{ .Values.executor | quote }} + {{- if eq (.Values.image.majorVersion | toString) "3" }} + execution_api_server_url: {{ printf "http://%s:%s/execution/" (include "airflow.web.fullname" .) (.Values.service.ports.http | toString) | quote }} + {{- end }} logging: colored_console_log: 'False' metrics: @@ -24,15 +27,26 @@ metrics: scheduler: standalone_dag_processor: {{ ternary "True" "False" .Values.dagProcessor.enabled | squote }} triggerer: + {{- if eq (.Values.image.majorVersion | toString) "2" }} default_capacity: {{ .Values.triggerer.defaultCapacity | quote }} + {{- else }} + capacity: {{ .Values.triggerer.defaultCapacity | quote }} + {{- end }} webserver: base_url: {{ include "airflow.baseUrl" . | quote }} enable_proxy_fix: {{ ternary "True" "False" (and .Values.ingress.enabled .Values.ingress.tls) | squote }} - web_server_port: {{ .Values.web.containerPorts.http | quote }} {{- if .Values.web.tls.enabled }} web_server_ssl_cert: "/opt/bitnami/airflow/certs/tls.crt" web_server_ssl_key: "/opt/bitnami/airflow/certs/tls.key" {{- end }} + {{- if eq (.Values.image.majorVersion | toString) "2" }} + web_server_port: {{ .Values.web.containerPorts.http | quote }} + {{- end }} +{{- if eq (.Values.image.majorVersion | toString) "3" }} +api: + base_url: {{ include "airflow.baseUrl" . | quote }} + port: {{ .Values.web.containerPorts.http | quote }} +{{- end }} {{- if contains "KubernetesExecutor" .Values.executor }} kubernetes_executor: namespace: {{ include "common.names.namespace" . | quote }} diff --git a/bitnami/airflow/templates/config/secret.yaml b/bitnami/airflow/templates/config/secret.yaml index 092a12fddd5692..b56fa92ca24087 100644 --- a/bitnami/airflow/templates/config/secret.yaml +++ b/bitnami/airflow/templates/config/secret.yaml @@ -29,4 +29,11 @@ data: {{- else }} airflow-secret-key: {{ randAlphaNum 32 | b64enc | b64enc | quote }} {{- end }} + {{- if eq (.Values.image.majorVersion | toString) "3" }} + {{- if or (include "common.secrets.exists" (dict "secret" (include "common.names.fullname" .) "context" $)) (not (empty .Values.auth.jwtSecretKey)) }} + airflow-jwt-secret-key: {{ include "common.secrets.passwords.manage" (dict "secret" (include "common.names.fullname" .) "key" "airflow-jwt-secret-key" "providedValues" (list "auth.jwtSecretKey") "context" $) }} + {{- else }} + airflow-jwt-secret-key: {{ randAlphaNum 32 | b64enc | b64enc | quote }} + {{- end }} + {{- end }} {{- end }} diff --git a/bitnami/airflow/templates/dag-processor/deployment.yaml b/bitnami/airflow/templates/dag-processor/deployment.yaml index f5a127aa2ca13e..db355a05d48878 100644 --- a/bitnami/airflow/templates/dag-processor/deployment.yaml +++ b/bitnami/airflow/templates/dag-processor/deployment.yaml @@ -113,6 +113,8 @@ spec: value: {{ include "airflow.redis.host" . | quote }} - name: REDIS_PORT_NUMBER value: {{ include "airflow.redis.port" . | quote }} + - name: PYTHONPYCACHEPREFIX + value: "/opt/bitnami/airflow/venv/tmp" {{- end }} {{- if .Values.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 12 }} @@ -151,19 +153,17 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type DagProcessorJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $livenessTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.dagProcessor.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.dagProcessor.readinessProbe.enabled }} - {{- $readinessTimeout := sub (int .Values.dagProcessor.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.dagProcessor.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - timeout {{ $readinessTimeout }} airflow jobs check --job-type DagProcessorJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow dag-processor" {{- end }} {{- if .Values.dagProcessor.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customStartupProbe "context" $) | nindent 12 }} @@ -176,7 +176,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $startupTimeout }} airflow jobs check --job-type DagProcessorJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $startupTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- end }} {{- if .Values.dagProcessor.lifecycleHooks }} @@ -209,6 +209,11 @@ spec: - name: empty-dir mountPath: /opt/bitnami/airflow/config/airflow_local_settings.py subPath: app-conf-dir/airflow_local_settings.py + {{- if and .Values.web.containerSecurityContext.enabled .Values.web.containerSecurityContext.readOnlyRootFilesystem }} + - name: empty-dir + mountPath: /opt/bitnami/airflow/venv/tmp + subPath: app-pyc-cache-dir + {{- end }} {{- if .Values.usePasswordFiles }} - name: airflow-secrets mountPath: /opt/bitnami/airflow/secrets diff --git a/bitnami/airflow/templates/scheduler/deployment.yaml b/bitnami/airflow/templates/scheduler/deployment.yaml index ac98901ba91a0b..be3df7fce98dea 100644 --- a/bitnami/airflow/templates/scheduler/deployment.yaml +++ b/bitnami/airflow/templates/scheduler/deployment.yaml @@ -108,6 +108,8 @@ spec: value: "scheduler" - name: AIRFLOW_EXECUTOR value: {{ .Values.executor }} + - name: PYTHONPYCACHEPREFIX + value: "/opt/bitnami/airflow/venv/tmp" {{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} - name: REDIS_HOST value: {{ include "airflow.redis.host" . | quote }} @@ -163,19 +165,17 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type SchedulerJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $livenessTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.scheduler.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.scheduler.readinessProbe.enabled }} - {{- $readinessTimeout := sub (int .Values.scheduler.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.scheduler.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - timeout {{ $readinessTimeout }} airflow jobs check --job-type SchedulerJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow scheduler" {{- end }} {{- if .Values.scheduler.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customStartupProbe "context" $) | nindent 12 }} @@ -188,7 +188,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $startupTimeout }} airflow jobs check --job-type SchedulerJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $startupTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- end }} {{- if .Values.scheduler.lifecycleHooks }} @@ -221,6 +221,11 @@ spec: - name: empty-dir mountPath: /opt/bitnami/airflow/config/airflow_local_settings.py subPath: app-conf-dir/airflow_local_settings.py + {{- if and .Values.web.containerSecurityContext.enabled .Values.web.containerSecurityContext.readOnlyRootFilesystem }} + - name: empty-dir + mountPath: /opt/bitnami/airflow/venv/tmp + subPath: app-pyc-cache-dir + {{- end }} {{- if .Values.usePasswordFiles }} - name: airflow-secrets mountPath: /opt/bitnami/airflow/secrets diff --git a/bitnami/airflow/templates/setup-db-job.yaml b/bitnami/airflow/templates/setup-db-job.yaml index 04f4bd7ae993ef..e8443458bac744 100644 --- a/bitnami/airflow/templates/setup-db-job.yaml +++ b/bitnami/airflow/templates/setup-db-job.yaml @@ -70,14 +70,24 @@ spec: if ! airflow_execute db check-migrations; then info "Populating database" + {{- if eq (.Values.image.majorVersion | toString) "2" }} airflow_execute db init + {{- else }} + airflow_execute db migrate + {{- end }} airflow_create_admin_user else info "Upgrading database schema" + {{- if eq (.Values.image.majorVersion | toString) "2" }} airflow_execute db upgrade + {{- else }} + airflow_execute db migrate + {{- end }} true fi + info "Synchronizing internal metadata" + airflow_execute sync-perm {{- end }} env: - name: BITNAMI_DEBUG diff --git a/bitnami/airflow/templates/triggerer/statefulset.yaml b/bitnami/airflow/templates/triggerer/statefulset.yaml index 37b6f2e3c91b95..42245f31ff15cb 100644 --- a/bitnami/airflow/templates/triggerer/statefulset.yaml +++ b/bitnami/airflow/templates/triggerer/statefulset.yaml @@ -112,6 +112,8 @@ spec: value: "triggerer" - name: AIRFLOW_EXECUTOR value: {{ .Values.executor }} + - name: PYTHONPYCACHEPREFIX + value: "/opt/bitnami/airflow/venv/tmp" {{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} - name: REDIS_HOST value: {{ include "airflow.redis.host" . | quote }} @@ -158,19 +160,17 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type TriggererJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $livenessTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.triggerer.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.triggerer.readinessProbe.enabled }} - {{- $readinessTimeout := sub (int .Values.triggerer.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.triggerer.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - timeout {{ $readinessTimeout }} airflow jobs check --job-type TriggererJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow triggerer" {{- end }} {{- if .Values.triggerer.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customStartupProbe "context" $) | nindent 12 }} @@ -183,7 +183,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $startupTimeout }} airflow jobs check --job-type TriggererJob --local {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $startupTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- end }} {{- if .Values.triggerer.lifecycleHooks }} @@ -213,6 +213,11 @@ spec: - name: empty-dir mountPath: /opt/bitnami/airflow/config/airflow_local_settings.py subPath: app-conf-dir/airflow_local_settings.py + {{- if and .Values.web.containerSecurityContext.enabled .Values.web.containerSecurityContext.readOnlyRootFilesystem }} + - name: empty-dir + mountPath: /opt/bitnami/airflow/venv/tmp + subPath: app-pyc-cache-dir + {{- end }} {{- if .Values.usePasswordFiles }} - name: airflow-secrets mountPath: /opt/bitnami/airflow/secrets diff --git a/bitnami/airflow/templates/web/configmap.yaml b/bitnami/airflow/templates/web/configmap.yaml index 682d7ea6f4c96e..07006b0fb7d72e 100644 --- a/bitnami/airflow/templates/web/configmap.yaml +++ b/bitnami/airflow/templates/web/configmap.yaml @@ -42,7 +42,11 @@ data: AUTH_LDAP_TLS_CACERTFILE = {{ printf "%s/%s" .Values.ldap.tls.certificatesMountPath .Values.ldap.tls.CAFilename | squote }} {{- end }} {{- else }} + {{- if eq (.Values.image.majorVersion | toString) "2" }} from airflow.www.fab_security.manager import AUTH_DB + {{- else }} + from flask_appbuilder.const import AUTH_DB + {{- end }} AUTH_TYPE = AUTH_DB {{- end }} diff --git a/bitnami/airflow/templates/web/deployment.yaml b/bitnami/airflow/templates/web/deployment.yaml index 4ef6da01d765e6..9e47cd14643e3e 100644 --- a/bitnami/airflow/templates/web/deployment.yaml +++ b/bitnami/airflow/templates/web/deployment.yaml @@ -123,6 +123,12 @@ spec: key: airflow-password {{- end }} {{- end }} + - name: PYTHONPYCACHEPREFIX + value: "/opt/bitnami/airflow/venv/tmp" + {{- if eq (.Values.image.majorVersion | toString) "2" }} + - name: AIRFLOW_COMPONENT_TYPE + value: "webserver" + {{- end }} {{- if .Values.extraEnvVars }} {{- include "common.tplvalues.render" (dict "value" .Values.extraEnvVars "context" $) | nindent 12 }} {{- end }} @@ -179,7 +185,7 @@ spec: command: - pgrep - -f - - airflow-webserver + - airflow {{- end }} {{- if .Values.web.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customReadinessProbe "context" $) | nindent 12 }} @@ -229,6 +235,11 @@ spec: - name: empty-dir mountPath: /opt/bitnami/airflow/webserver_config.py subPath: app-base-dir/webserver_config.py + {{- if and .Values.web.containerSecurityContext.enabled .Values.web.containerSecurityContext.readOnlyRootFilesystem }} + - name: empty-dir + mountPath: /opt/bitnami/airflow/venv/tmp + subPath: app-pyc-cache-dir + {{- end }} {{- if .Values.usePasswordFiles }} - name: airflow-secrets mountPath: /opt/bitnami/airflow/secrets @@ -282,7 +293,7 @@ spec: projected: sources: - secret: - name: {{ include "airflow.secretName" . }} + name: {{ include "airflow.secretName" . }} - secret: name: {{ include "airflow.database.secretName" . }} {{- if .Values.ldap.enabled }} diff --git a/bitnami/airflow/templates/web/service.yaml b/bitnami/airflow/templates/web/service.yaml index ff0a7b5bf490f2..d83cf2f092844f 100644 --- a/bitnami/airflow/templates/web/service.yaml +++ b/bitnami/airflow/templates/web/service.yaml @@ -6,7 +6,7 @@ SPDX-License-Identifier: APACHE-2.0 apiVersion: v1 kind: Service metadata: - name: {{ include "common.names.fullname" . }} + name: {{ include "airflow.web.fullname" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} app.kubernetes.io/component: web @@ -38,7 +38,7 @@ spec: - name: http port: {{ .Values.service.ports.http }} protocol: TCP - targetPort: http + targetPort: {{ .Values.web.containerPorts.http }} {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.http))) }} nodePort: {{ .Values.service.nodePorts.http }} {{- else if eq .Values.service.type "ClusterIP" }} diff --git a/bitnami/airflow/templates/worker/statefulset.yaml b/bitnami/airflow/templates/worker/statefulset.yaml index cbf3cdc51ab450..732118dbc2abfe 100644 --- a/bitnami/airflow/templates/worker/statefulset.yaml +++ b/bitnami/airflow/templates/worker/statefulset.yaml @@ -114,6 +114,8 @@ spec: value: "worker" - name: AIRFLOW_EXECUTOR value: {{ include "airflow.worker.executor" . }} + - name: PYTHONPYCACHEPREFIX + value: "/opt/bitnami/airflow/venv/tmp" {{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor "CeleryKubernetesExecutor") }} - name: REDIS_HOST value: {{ include "airflow.redis.host" . | quote }} @@ -221,6 +223,11 @@ spec: - name: empty-dir mountPath: /opt/bitnami/airflow/webserver_config.py subPath: app-base-dir/webserver_config.py + {{- if and .Values.web.containerSecurityContext.enabled .Values.web.containerSecurityContext.readOnlyRootFilesystem }} + - name: empty-dir + mountPath: /opt/bitnami/airflow/venv/tmp + subPath: app-pyc-cache-dir + {{- end }} {{- if .Values.usePasswordFiles }} - name: airflow-secrets mountPath: /opt/bitnami/airflow/secrets diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 2ade0fdfe84d44..2d0c3f1f87a6d7 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -84,6 +84,7 @@ diagnosticMode: ## Bitnami Airflow image version ## ref: https://hub.docker.com/r/bitnami/airflow/tags +## @param image.majorVersion [default: 3] Major version of the Airflow image ## @param image.registry [default: REGISTRY_NAME] Airflow image registry ## @param image.repository [default: REPOSITORY_NAME/airflow] Airflow image repository ## @skip image.tag Airflow image tag (immutable tags are recommended) @@ -92,9 +93,10 @@ diagnosticMode: ## @param image.pullSecrets Airflow image pull secrets ## @param image.debug Enable image debug mode image: + majorVersion: "3" registry: docker.io repository: bitnami/airflow - tag: 3.0.0-debian-12-r0 + tag: 3.0.0-debian-12-r2 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' @@ -132,6 +134,10 @@ auth: ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key ## secretKey: "" + ## @param auth.jwtSecretKey JWT secret key to run your flask app + ## ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.html#secret-key + ## + jwtSecretKey: "" ## @param auth.existingSecret Name of an existing secret to use for Airflow credentials ## `auth.password`, `auth.fernetKey`, and `auth.secretKey` will be ignored and picked up from this secret ## The secret must contain the keys `airflow-password`, `airflow-fernet-key` and `airflow-secret-key' @@ -1218,7 +1224,7 @@ scheduler: dagProcessor: ## @param dagProcessor.enabled Run Airflow Dag Processor Manager as a standalone component ## - enabled: false + enabled: true ## @param dagProcessor.replicaCount Number of Airflow Dag Processor replicas ## replicaCount: 1 @@ -1559,7 +1565,7 @@ dagProcessor: triggerer: ## @param triggerer.enabled Run Airflow Triggerer as a standalone component ## - enabled: false + enabled: true ## @param triggerer.defaultCapacity How many triggers a single Triggerer can run at once ## defaultCapacity: 1000 From b9345237c34ce3549a9d51e7f5463b0cb52c4dcc Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 14:30:58 +0200 Subject: [PATCH 04/13] [bitnami/airflow] Minor update to setup-db-job Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/templates/setup-db-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/bitnami/airflow/templates/setup-db-job.yaml b/bitnami/airflow/templates/setup-db-job.yaml index e8443458bac744..a82c820a20164b 100644 --- a/bitnami/airflow/templates/setup-db-job.yaml +++ b/bitnami/airflow/templates/setup-db-job.yaml @@ -77,6 +77,7 @@ spec: {{- end }} airflow_create_admin_user + true else info "Upgrading database schema" {{- if eq (.Values.image.majorVersion | toString) "2" }} From ad6aca4738425bdc5e6d8575b7a5a7a90ccb3832 Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Tue, 29 Apr 2025 13:02:28 +0000 Subject: [PATCH 05/13] Update CHANGELOG.md Signed-off-by: Bitnami Bot --- bitnami/airflow/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/airflow/CHANGELOG.md b/bitnami/airflow/CHANGELOG.md index 7b492f5e75e5a2..6decf3291530d2 100644 --- a/bitnami/airflow/CHANGELOG.md +++ b/bitnami/airflow/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 23.0.0 (2025-04-23) +## 23.0.0 (2025-04-29) * [bitnami/airflow] Release 23.0.0 ([#33141](https://github.com/bitnami/charts/pull/33141)) From 3ffc91fbe15be4c6ca54e7316fb17439fc5315e4 Mon Sep 17 00:00:00 2001 From: Bitnami Bot Date: Tue, 29 Apr 2025 13:02:30 +0000 Subject: [PATCH 06/13] Update README.md with readme-generator-for-helm Signed-off-by: Bitnami Bot --- bitnami/airflow/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 1592bc21b6b2fe..df79f2f2f820e9 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -400,6 +400,7 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means | Name | Description | Value | | --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +| `image.majorVersion` | Major version of the Airflow image | `3` | | `image.registry` | Airflow image registry | `REGISTRY_NAME` | | `image.repository` | Airflow image repository | `REPOSITORY_NAME/airflow` | | `image.digest` | Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | @@ -410,6 +411,7 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means | `auth.password` | Password to access web UI | `""` | | `auth.fernetKey` | Fernet key to secure connections | `""` | | `auth.secretKey` | Secret key to run your flask app | `""` | +| `auth.jwtSecretKey` | JWT secret key to run your flask app | `""` | | `auth.existingSecret` | Name of an existing secret to use for Airflow credentials | `""` | | `executor` | Airflow executor. Allowed values: `SequentialExecutor`, `LocalExecutor`, `CeleryExecutor`, `KubernetesExecutor`, `CeleryKubernetesExecutor` and `LocalKubernetesExecutor` | `CeleryExecutor` | | `loadExamples` | Switch to load some Airflow examples | `false` | @@ -712,7 +714,7 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means | Name | Description | Value | | ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | -| `dagProcessor.enabled` | Run Airflow Dag Processor Manager as a standalone component | `false` | +| `dagProcessor.enabled` | Run Airflow Dag Processor Manager as a standalone component | `true` | | `dagProcessor.replicaCount` | Number of Airflow Dag Processor replicas | `1` | | `dagProcessor.command` | Override default Airflow Dag Processor cmd | `[]` | | `dagProcessor.args` | Override default Airflow Dag Processor args | `[]` | @@ -806,7 +808,7 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means | Name | Description | Value | | ------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | -| `triggerer.enabled` | Run Airflow Triggerer as a standalone component | `false` | +| `triggerer.enabled` | Run Airflow Triggerer as a standalone component | `true` | | `triggerer.defaultCapacity` | How many triggers a single Triggerer can run at once | `1000` | | `triggerer.replicaCount` | Number of Airflow Triggerer replicas | `1` | | `triggerer.command` | Override default Airflow Triggerer cmd | `[]` | From 3617681e9592a169302834cc2a5cad63a7ee27c5 Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 15:12:22 +0200 Subject: [PATCH 07/13] [bitnami/airflow] Add airflow_wait_for_db_connection to wait for db init containers Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/templates/_init_containers_sidecars.tpl | 2 ++ bitnami/airflow/templates/setup-db-job.yaml | 4 ++-- bitnami/airflow/templates/web/deployment.yaml | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bitnami/airflow/templates/_init_containers_sidecars.tpl b/bitnami/airflow/templates/_init_containers_sidecars.tpl index a9a5740e3d3260..040c67dca99c4c 100644 --- a/bitnami/airflow/templates/_init_containers_sidecars.tpl +++ b/bitnami/airflow/templates/_init_containers_sidecars.tpl @@ -230,6 +230,8 @@ Returns an init-container that waits for db migrations to be ready . /opt/bitnami/scripts/airflow-env.sh . /opt/bitnami/scripts/libairflow.sh + info "Trying to connect to the database server" + airflow_wait_for_db_connection info "Waiting for db migrations to be completed" airflow_wait_for_db_migrations {{- if eq (.Values.image.majorVersion | toString) "3" }} diff --git a/bitnami/airflow/templates/setup-db-job.yaml b/bitnami/airflow/templates/setup-db-job.yaml index a82c820a20164b..0bb85d26c1ecc0 100644 --- a/bitnami/airflow/templates/setup-db-job.yaml +++ b/bitnami/airflow/templates/setup-db-job.yaml @@ -77,6 +77,8 @@ spec: {{- end }} airflow_create_admin_user + info "Synchronizing internal metadata" + airflow_execute sync-perm true else info "Upgrading database schema" @@ -87,8 +89,6 @@ spec: {{- end }} true fi - info "Synchronizing internal metadata" - airflow_execute sync-perm {{- end }} env: - name: BITNAMI_DEBUG diff --git a/bitnami/airflow/templates/web/deployment.yaml b/bitnami/airflow/templates/web/deployment.yaml index 9e47cd14643e3e..69c867a4a6a9ab 100644 --- a/bitnami/airflow/templates/web/deployment.yaml +++ b/bitnami/airflow/templates/web/deployment.yaml @@ -185,7 +185,11 @@ spec: command: - pgrep - -f - - airflow + {{- if eq (.Values.image.majorVersion | toString) "2" }} + - "airflow webserver" + {{- else }} + - "airflow api-server" + {{- end }} {{- end }} {{- if .Values.web.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customReadinessProbe "context" $) | nindent 12 }} From f68c49ffac7ab713b9060225a808e5eb84fb462d Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 17:10:15 +0200 Subject: [PATCH 08/13] [bitnami/airflow] Feedback Signed-off-by: Gonzalo Gomez Gracia --- .vib/airflow/runtime-parameters.yaml | 1 + bitnami/airflow/templates/config/secret.yaml | 2 +- bitnami/airflow/templates/web/deployment.yaml | 6 +----- bitnami/airflow/templates/web/service.yaml | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.vib/airflow/runtime-parameters.yaml b/.vib/airflow/runtime-parameters.yaml index 9be206d0bf42c8..48f90b88c22346 100644 --- a/.vib/airflow/runtime-parameters.yaml +++ b/.vib/airflow/runtime-parameters.yaml @@ -42,6 +42,7 @@ web: runAsUser: 1002 # Give some extra time for the service to start startupProbe: + enabled: true failureThreshold: 30 periodSeconds: 10 worker: diff --git a/bitnami/airflow/templates/config/secret.yaml b/bitnami/airflow/templates/config/secret.yaml index b56fa92ca24087..73474be80a4a87 100644 --- a/bitnami/airflow/templates/config/secret.yaml +++ b/bitnami/airflow/templates/config/secret.yaml @@ -31,7 +31,7 @@ data: {{- end }} {{- if eq (.Values.image.majorVersion | toString) "3" }} {{- if or (include "common.secrets.exists" (dict "secret" (include "common.names.fullname" .) "context" $)) (not (empty .Values.auth.jwtSecretKey)) }} - airflow-jwt-secret-key: {{ include "common.secrets.passwords.manage" (dict "secret" (include "common.names.fullname" .) "key" "airflow-jwt-secret-key" "providedValues" (list "auth.jwtSecretKey") "context" $) }} + airflow-jwt-secret-key: {{ include "common.secrets.passwords.manage" (dict "secret" (include "common.names.fullname" .) "key" "airflow-jwt-secret-key" "providedValues" (list "auth.jwtSecretKey") "failOnNew" false "context" $) }} {{- else }} airflow-jwt-secret-key: {{ randAlphaNum 32 | b64enc | b64enc | quote }} {{- end }} diff --git a/bitnami/airflow/templates/web/deployment.yaml b/bitnami/airflow/templates/web/deployment.yaml index 69c867a4a6a9ab..9e47cd14643e3e 100644 --- a/bitnami/airflow/templates/web/deployment.yaml +++ b/bitnami/airflow/templates/web/deployment.yaml @@ -185,11 +185,7 @@ spec: command: - pgrep - -f - {{- if eq (.Values.image.majorVersion | toString) "2" }} - - "airflow webserver" - {{- else }} - - "airflow api-server" - {{- end }} + - airflow {{- end }} {{- if .Values.web.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customReadinessProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/web/service.yaml b/bitnami/airflow/templates/web/service.yaml index d83cf2f092844f..04522299cdcbea 100644 --- a/bitnami/airflow/templates/web/service.yaml +++ b/bitnami/airflow/templates/web/service.yaml @@ -38,7 +38,7 @@ spec: - name: http port: {{ .Values.service.ports.http }} protocol: TCP - targetPort: {{ .Values.web.containerPorts.http }} + targetPort: http {{- if (and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.http))) }} nodePort: {{ .Values.service.nodePorts.http }} {{- else if eq .Values.service.type "ClusterIP" }} From 0c80ef570ad24894c8e961133b3a461323b64eda Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 19:56:06 +0200 Subject: [PATCH 09/13] [bitnami/airflow] Add airflow.isImageMajorVersion definitions Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/README.md | 9 ++++- bitnami/airflow/templates/_helpers.tpl | 37 ++++++++++++++++++- .../templates/_init_containers_sidecars.tpl | 2 +- .../airflow/templates/config/configmap.yaml | 8 ++-- bitnami/airflow/templates/config/secret.yaml | 2 +- bitnami/airflow/templates/setup-db-job.yaml | 4 +- bitnami/airflow/templates/web/configmap.yaml | 2 +- bitnami/airflow/templates/web/deployment.yaml | 2 +- bitnami/airflow/values.yaml | 2 - 9 files changed, 52 insertions(+), 16 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index df79f2f2f820e9..fbd2a76be5b512 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -400,7 +400,6 @@ The Bitnami Airflow chart relies on the PostgreSQL chart persistence. This means | Name | Description | Value | | --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | -| `image.majorVersion` | Major version of the Airflow image | `3` | | `image.registry` | Airflow image registry | `REGISTRY_NAME` | | `image.repository` | Airflow image repository | `REPOSITORY_NAME/airflow` | | `image.digest` | Airflow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | `""` | @@ -1291,7 +1290,13 @@ Find more information about how to deal with common errors related to Bitnami's ### To 23.0.0 -This major release adds support for Airflow 3 versions. Additionally, previous Airflow 2 images can be deployed by setting the new `image.majorVersion=2` option and updating the rest of the image related parameters accordingly. +This major release adds support for Airflow 3 versions. Additionally, previous Airflow 2 images can be deployed by setting the corresponding image parameters. The chart logic will detect which image version you are using and it will generate the required Airflow configuration and Kubernetes objects. + +We recommend following the next procedure in order to upgrade from Airflow 2 to Airflow 3 images: + +- Perform a Helm upgrade with the latest chart logic and the latest Airflow 2.x image version. +- Follow the recommended steps for the database backup and the DAGs files verification available at the [official "upgrading to Airflow 3" guide](https://airflow.apache.org/docs/apache-airflow/stable/installation/upgrading_to_airflow3.html). +- Perform a Helm upgrade using the Airflow 3.x image. ### To 22.4.0 diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index 0131091d1b2455..e3bf0481c7999d 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -352,7 +352,7 @@ Add environment variables to configure airflow common values value: "cat /opt/bitnami/airflow/secrets/airflow-fernet-key" - name: AIRFLOW__WEBSERVER__SECRET_KEY_CMD value: "cat /opt/bitnami/airflow/secrets/airflow-secret-key" -{{- if eq (.Values.image.majorVersion | toString) "3" }} +{{- if (include "airflow.isImageMajorVersion3" .) }} - name: AIRFLOW__API_AUTH__JWT_SECRET_CMD value: "cat /opt/bitnami/airflow/secrets/airflow-jwt-secret-key" {{- end }} @@ -367,7 +367,7 @@ Add environment variables to configure airflow common values secretKeyRef: name: {{ include "airflow.secretName" . }} key: airflow-secret-key -{{- if eq (.Values.image.majorVersion | toString) "3" }} +{{- if (include "airflow.isImageMajorVersion3" .) }} - name: AIRFLOW__API_AUTH__JWT_SECRET_CMD valueFrom: secretKeyRef: @@ -527,3 +527,36 @@ Ref: https://github.com/bitnami/charts/pull/6096#issuecomment-856499047 {{- define "airflow.worker.executor" -}} {{- print (ternary "CeleryExecutor" .Values.executor (eq .Values.executor "CeleryKubernetesExecutor")) -}} {{- end -}} + +{{/* +Validates a semver constraint +*/}} +{{- define "airflow.semverCondition" -}} +{{- $constraint := .constraint -}} +{{- $imageVersion := .imageVersion -}} + +{{/* tag 'latest' is an special case, where we fall to .Chart.AppVersion value */}} +{{- if eq "latest" $imageVersion -}} +{{- $imageVersion = .context.Chart.AppVersion -}} +{{- else -}} +{{- $imageVersion = (index (splitList "-" $imageVersion) 0 ) -}} +{{- end -}} + +{{- if semverCompare $constraint $imageVersion -}} +true +{{- end -}} +{{- end -}} + +{{/* +Validates the image tag version is equal or higher than 3.0.0 +*/}} +{{- define "airflow.isImageMajorVersion3" -}} +{{- include "airflow.semverCondition" (dict "constraint" "^3" "imageVersion" .Values.image.tag "context" $) -}} +{{- end -}} + +{{/* +Validates the image tag version is equal or higher than 2.0.0 +*/}} +{{- define "airflow.isImageMajorVersion2" -}} +{{- include "airflow.semverCondition" (dict "constraint" "^2" "imageVersion" .Values.image.tag "context" $) -}} +{{- end -}} diff --git a/bitnami/airflow/templates/_init_containers_sidecars.tpl b/bitnami/airflow/templates/_init_containers_sidecars.tpl index 040c67dca99c4c..1e714dd9117378 100644 --- a/bitnami/airflow/templates/_init_containers_sidecars.tpl +++ b/bitnami/airflow/templates/_init_containers_sidecars.tpl @@ -234,7 +234,7 @@ Returns an init-container that waits for db migrations to be ready airflow_wait_for_db_connection info "Waiting for db migrations to be completed" airflow_wait_for_db_migrations - {{- if eq (.Values.image.majorVersion | toString) "3" }} + {{- if (include "airflow.isImageMajorVersion3" .) }} info "Waiting for the admin user to exist" airflow_wait_for_admin_user {{- end }} diff --git a/bitnami/airflow/templates/config/configmap.yaml b/bitnami/airflow/templates/config/configmap.yaml index 543809908d83a7..d7d0953621ce8c 100644 --- a/bitnami/airflow/templates/config/configmap.yaml +++ b/bitnami/airflow/templates/config/configmap.yaml @@ -14,7 +14,7 @@ ref: https://airflow.apache.org/docs/apache-airflow/stable/configurations-ref.ht core: load_examples: {{ ternary "True" "False" .Values.loadExamples | squote }} executor: {{ .Values.executor | quote }} - {{- if eq (.Values.image.majorVersion | toString) "3" }} + {{- if (include "airflow.isImageMajorVersion3" .) }} execution_api_server_url: {{ printf "http://%s:%s/execution/" (include "airflow.web.fullname" .) (.Values.service.ports.http | toString) | quote }} {{- end }} logging: @@ -27,7 +27,7 @@ metrics: scheduler: standalone_dag_processor: {{ ternary "True" "False" .Values.dagProcessor.enabled | squote }} triggerer: - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} default_capacity: {{ .Values.triggerer.defaultCapacity | quote }} {{- else }} capacity: {{ .Values.triggerer.defaultCapacity | quote }} @@ -39,10 +39,10 @@ webserver: web_server_ssl_cert: "/opt/bitnami/airflow/certs/tls.crt" web_server_ssl_key: "/opt/bitnami/airflow/certs/tls.key" {{- end }} - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} web_server_port: {{ .Values.web.containerPorts.http | quote }} {{- end }} -{{- if eq (.Values.image.majorVersion | toString) "3" }} +{{- if (include "airflow.isImageMajorVersion3" .) }} api: base_url: {{ include "airflow.baseUrl" . | quote }} port: {{ .Values.web.containerPorts.http | quote }} diff --git a/bitnami/airflow/templates/config/secret.yaml b/bitnami/airflow/templates/config/secret.yaml index 73474be80a4a87..1a4170c54b30c9 100644 --- a/bitnami/airflow/templates/config/secret.yaml +++ b/bitnami/airflow/templates/config/secret.yaml @@ -29,7 +29,7 @@ data: {{- else }} airflow-secret-key: {{ randAlphaNum 32 | b64enc | b64enc | quote }} {{- end }} - {{- if eq (.Values.image.majorVersion | toString) "3" }} + {{- if (include "airflow.isImageMajorVersion3" .) }} {{- if or (include "common.secrets.exists" (dict "secret" (include "common.names.fullname" .) "context" $)) (not (empty .Values.auth.jwtSecretKey)) }} airflow-jwt-secret-key: {{ include "common.secrets.passwords.manage" (dict "secret" (include "common.names.fullname" .) "key" "airflow-jwt-secret-key" "providedValues" (list "auth.jwtSecretKey") "failOnNew" false "context" $) }} {{- else }} diff --git a/bitnami/airflow/templates/setup-db-job.yaml b/bitnami/airflow/templates/setup-db-job.yaml index 0bb85d26c1ecc0..e5c576bd98a379 100644 --- a/bitnami/airflow/templates/setup-db-job.yaml +++ b/bitnami/airflow/templates/setup-db-job.yaml @@ -70,7 +70,7 @@ spec: if ! airflow_execute db check-migrations; then info "Populating database" - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} airflow_execute db init {{- else }} airflow_execute db migrate @@ -82,7 +82,7 @@ spec: true else info "Upgrading database schema" - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} airflow_execute db upgrade {{- else }} airflow_execute db migrate diff --git a/bitnami/airflow/templates/web/configmap.yaml b/bitnami/airflow/templates/web/configmap.yaml index 07006b0fb7d72e..530f0bbb1ff76f 100644 --- a/bitnami/airflow/templates/web/configmap.yaml +++ b/bitnami/airflow/templates/web/configmap.yaml @@ -42,7 +42,7 @@ data: AUTH_LDAP_TLS_CACERTFILE = {{ printf "%s/%s" .Values.ldap.tls.certificatesMountPath .Values.ldap.tls.CAFilename | squote }} {{- end }} {{- else }} - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} from airflow.www.fab_security.manager import AUTH_DB {{- else }} from flask_appbuilder.const import AUTH_DB diff --git a/bitnami/airflow/templates/web/deployment.yaml b/bitnami/airflow/templates/web/deployment.yaml index 9e47cd14643e3e..480affbbd06b16 100644 --- a/bitnami/airflow/templates/web/deployment.yaml +++ b/bitnami/airflow/templates/web/deployment.yaml @@ -125,7 +125,7 @@ spec: {{- end }} - name: PYTHONPYCACHEPREFIX value: "/opt/bitnami/airflow/venv/tmp" - {{- if eq (.Values.image.majorVersion | toString) "2" }} + {{- if (include "airflow.isImageMajorVersion2" .) }} - name: AIRFLOW_COMPONENT_TYPE value: "webserver" {{- end }} diff --git a/bitnami/airflow/values.yaml b/bitnami/airflow/values.yaml index 2d0c3f1f87a6d7..5f59ee280c5a71 100644 --- a/bitnami/airflow/values.yaml +++ b/bitnami/airflow/values.yaml @@ -84,7 +84,6 @@ diagnosticMode: ## Bitnami Airflow image version ## ref: https://hub.docker.com/r/bitnami/airflow/tags -## @param image.majorVersion [default: 3] Major version of the Airflow image ## @param image.registry [default: REGISTRY_NAME] Airflow image registry ## @param image.repository [default: REPOSITORY_NAME/airflow] Airflow image repository ## @skip image.tag Airflow image tag (immutable tags are recommended) @@ -93,7 +92,6 @@ diagnosticMode: ## @param image.pullSecrets Airflow image pull secrets ## @param image.debug Enable image debug mode image: - majorVersion: "3" registry: docker.io repository: bitnami/airflow tag: 3.0.0-debian-12-r2 From 6d608a7fa4027adb29b079d865a757676a41aae0 Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 21:30:41 +0200 Subject: [PATCH 10/13] [bitnami/airflow] Update readiness & liveness probes Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/README.md | 15 ++++++++---- .../templates/dag-processor/deployment.yaml | 16 ++++++------- .../templates/scheduler/deployment.yaml | 16 ++++++------- .../templates/triggerer/statefulset.yaml | 16 ++++++------- bitnami/airflow/templates/web/deployment.yaml | 24 +++++++++++-------- .../airflow/templates/worker/statefulset.yaml | 14 +++++------ bitnami/haproxy/README.md | 2 +- 7 files changed, 57 insertions(+), 46 deletions(-) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index fbd2a76be5b512..5af8e36978426e 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -1290,13 +1290,20 @@ Find more information about how to deal with common errors related to Bitnami's ### To 23.0.0 -This major release adds support for Airflow 3 versions. Additionally, previous Airflow 2 images can be deployed by setting the corresponding image parameters. The chart logic will detect which image version you are using and it will generate the required Airflow configuration and Kubernetes objects. +This major release adds support for Airflow `3.x.y` series. Additionally, previous Airflow `2.x.y` series can be deployed by setting the corresponding image parameters. The chart logic will detect which image version you are using, and it will generate the required Airflow configuration and Kubernetes objects. -We recommend following the next procedure in order to upgrade from Airflow 2 to Airflow 3 images: +We recommend following the next procedure in order to upgrade from `22.x.y` chart version to the `23.x.y` series, and also upgrade to Airflow `3.y.z` series: + +- Upgrade your release (maintaining Airflow `2.x.y` series): +```console +helm upgrade airflow oci://REGISTRY_NAME/REPOSITORY_NAME/airflow --set image.tag=2 +``` -- Perform a Helm upgrade with the latest chart logic and the latest Airflow 2.x image version. - Follow the recommended steps for the database backup and the DAGs files verification available at the [official "upgrading to Airflow 3" guide](https://airflow.apache.org/docs/apache-airflow/stable/installation/upgrading_to_airflow3.html). -- Perform a Helm upgrade using the Airflow 3.x image. +- Upgrade your release now using the default Airflow `3.x.y` series: +```console +helm upgrade airflow oci://REGISTRY_NAME/REPOSITORY_NAME/airflow +``` ### To 22.4.0 diff --git a/bitnami/airflow/templates/dag-processor/deployment.yaml b/bitnami/airflow/templates/dag-processor/deployment.yaml index db355a05d48878..ecb3bd117ab6b7 100644 --- a/bitnami/airflow/templates/dag-processor/deployment.yaml +++ b/bitnami/airflow/templates/dag-processor/deployment.yaml @@ -149,11 +149,9 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.dagProcessor.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow dag-processor" {{- end }} {{- if .Values.dagProcessor.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customReadinessProbe "context" $) | trim | nindent 12 }} @@ -161,9 +159,11 @@ spec: readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.dagProcessor.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - pgrep - - -f - - "airflow dag-processor" + - /bin/bash + - -ec + - | + export CONNECTION_CHECK_MAX_COUNT=0 + timeout {{ $livenessTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.dagProcessor.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customStartupProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/scheduler/deployment.yaml b/bitnami/airflow/templates/scheduler/deployment.yaml index be3df7fce98dea..149e2805acf894 100644 --- a/bitnami/airflow/templates/scheduler/deployment.yaml +++ b/bitnami/airflow/templates/scheduler/deployment.yaml @@ -161,11 +161,9 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.scheduler.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow scheduler" {{- end }} {{- if .Values.scheduler.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customReadinessProbe "context" $) | trim | nindent 12 }} @@ -173,9 +171,11 @@ spec: readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.scheduler.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - pgrep - - -f - - "airflow scheduler" + - /bin/bash + - -ec + - | + export CONNECTION_CHECK_MAX_COUNT=0 + timeout {{ $livenessTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.scheduler.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customStartupProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/triggerer/statefulset.yaml b/bitnami/airflow/templates/triggerer/statefulset.yaml index 42245f31ff15cb..4d43d7d4d59898 100644 --- a/bitnami/airflow/templates/triggerer/statefulset.yaml +++ b/bitnami/airflow/templates/triggerer/statefulset.yaml @@ -156,11 +156,9 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.triggerer.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - /bin/bash - - -ec - - | - export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + - pgrep + - -f + - "airflow triggerer" {{- end }} {{- if .Values.triggerer.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customReadinessProbe "context" $) | trim | nindent 12 }} @@ -168,9 +166,11 @@ spec: readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.triggerer.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - - pgrep - - -f - - "airflow triggerer" + - /bin/bash + - -ec + - | + export CONNECTION_CHECK_MAX_COUNT=0 + timeout {{ $livenessTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.triggerer.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customStartupProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/web/deployment.yaml b/bitnami/airflow/templates/web/deployment.yaml index 480affbbd06b16..39e3cc41417a68 100644 --- a/bitnami/airflow/templates/web/deployment.yaml +++ b/bitnami/airflow/templates/web/deployment.yaml @@ -181,19 +181,13 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customLivenessProbe "context" $) | nindent 12 }} {{- else if .Values.web.livenessProbe.enabled }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.web.livenessProbe "enabled") "context" $) | nindent 12 }} - exec: - command: - - pgrep - - -f - - airflow - {{- end }} - {{- if .Values.web.customReadinessProbe }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customReadinessProbe "context" $) | nindent 12 }} - {{- else if .Values.web.readinessProbe.enabled }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.web.readinessProbe "enabled") "context" $) | nindent 12 }} {{- if contains "127.0.0.1" (include "airflow.baseUrl" .) }} httpGet: + {{- if (include "airflow.isImageMajorVersion3" .) }} + path: /api/v2/monitor/health + {{- else }} path: /health + {{- end }} port: http scheme: {{ ternary "HTTPS" "HTTP" .Values.web.tls.enabled }} {{- else }} @@ -201,6 +195,16 @@ spec: port: http {{- end }} {{- end }} + {{- if .Values.web.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.web.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.web.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.web.readinessProbe "enabled") "context" $) | nindent 12 }} + exec: + command: + - pgrep + - -f + - airflow + {{- end }} {{- end }} {{- if .Values.web.lifecycleHooks }} lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.web.lifecycleHooks "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/worker/statefulset.yaml b/bitnami/airflow/templates/worker/statefulset.yaml index 732118dbc2abfe..9d4669309fe04d 100644 --- a/bitnami/airflow/templates/worker/statefulset.yaml +++ b/bitnami/airflow/templates/worker/statefulset.yaml @@ -174,6 +174,13 @@ spec: livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customLivenessProbe "context" $) | nindent 12 }} {{- else if .Values.worker.livenessProbe.enabled }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.livenessProbe "enabled") "context" $) | nindent 12 }} + tcpSocket: + port: worker + {{- end }} + {{- if .Values.worker.customReadinessProbe }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }} + {{- else if .Values.worker.readinessProbe.enabled }} + readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: - sh @@ -182,13 +189,6 @@ spec: . /opt/bitnami/airflow/venv/bin/activate && \ CONNECTION_CHECK_MAX_COUNT=0 python -m celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d celery@$(hostname) {{- end }} - {{- if .Values.worker.customReadinessProbe }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.worker.customReadinessProbe "context" $) | nindent 12 }} - {{- else if .Values.worker.readinessProbe.enabled }} - readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.worker.readinessProbe "enabled") "context" $) | nindent 12 }} - tcpSocket: - port: worker - {{- end }} {{- end }} {{- if .Values.worker.lifecycleHooks }} lifecycle: {{- include "common.tplvalues.render" (dict "value" .Values.worker.lifecycleHooks "context" $) | nindent 12 }} diff --git a/bitnami/haproxy/README.md b/bitnami/haproxy/README.md index aab3517c59ea2c..5a7c754644cb04 100644 --- a/bitnami/haproxy/README.md +++ b/bitnami/haproxy/README.md @@ -400,4 +400,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file +limitations under the License. From 4a9720976ed41f7f37f5260d7ea622dfedd7751c Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 21:35:27 +0200 Subject: [PATCH 11/13] [bitnami/airflow] Typo Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/templates/_helpers.tpl | 2 +- bitnami/airflow/templates/dag-processor/deployment.yaml | 4 ++-- bitnami/airflow/templates/scheduler/deployment.yaml | 4 ++-- bitnami/airflow/templates/triggerer/statefulset.yaml | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bitnami/airflow/templates/_helpers.tpl b/bitnami/airflow/templates/_helpers.tpl index e3bf0481c7999d..7b33fdf5f786d3 100644 --- a/bitnami/airflow/templates/_helpers.tpl +++ b/bitnami/airflow/templates/_helpers.tpl @@ -533,7 +533,7 @@ Validates a semver constraint */}} {{- define "airflow.semverCondition" -}} {{- $constraint := .constraint -}} -{{- $imageVersion := .imageVersion -}} +{{- $imageVersion := (.imageVersion | toString) -}} {{/* tag 'latest' is an special case, where we fall to .Chart.AppVersion value */}} {{- if eq "latest" $imageVersion -}} diff --git a/bitnami/airflow/templates/dag-processor/deployment.yaml b/bitnami/airflow/templates/dag-processor/deployment.yaml index ecb3bd117ab6b7..ea62597029aa87 100644 --- a/bitnami/airflow/templates/dag-processor/deployment.yaml +++ b/bitnami/airflow/templates/dag-processor/deployment.yaml @@ -145,7 +145,6 @@ spec: {{- if .Values.dagProcessor.customLivenessProbe }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customLivenessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.dagProcessor.livenessProbe.enabled }} - {{- $livenessTimeout := sub (int .Values.dagProcessor.livenessProbe.timeoutSeconds) 1 }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.dagProcessor.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -156,6 +155,7 @@ spec: {{- if .Values.dagProcessor.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.dagProcessor.readinessProbe.enabled }} + {{- $readinessTimeout := sub (int .Values.dagProcessor.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.dagProcessor.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -163,7 +163,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $readinessTimeout }} airflow jobs check --job-type DagProcessorJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.dagProcessor.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.dagProcessor.customStartupProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/scheduler/deployment.yaml b/bitnami/airflow/templates/scheduler/deployment.yaml index 149e2805acf894..5b3ba78b4e6b9a 100644 --- a/bitnami/airflow/templates/scheduler/deployment.yaml +++ b/bitnami/airflow/templates/scheduler/deployment.yaml @@ -157,7 +157,6 @@ spec: {{- if .Values.scheduler.customLivenessProbe }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customLivenessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.scheduler.livenessProbe.enabled }} - {{- $livenessTimeout := sub (int .Values.scheduler.livenessProbe.timeoutSeconds) 1 }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.scheduler.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -168,6 +167,7 @@ spec: {{- if .Values.scheduler.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.scheduler.readinessProbe.enabled }} + {{- $readinessTimeout := sub (int .Values.scheduler.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.scheduler.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -175,7 +175,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $readinessTimeout }} airflow jobs check --job-type SchedulerJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.scheduler.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.scheduler.customStartupProbe "context" $) | nindent 12 }} diff --git a/bitnami/airflow/templates/triggerer/statefulset.yaml b/bitnami/airflow/templates/triggerer/statefulset.yaml index 4d43d7d4d59898..b89a7300076c90 100644 --- a/bitnami/airflow/templates/triggerer/statefulset.yaml +++ b/bitnami/airflow/templates/triggerer/statefulset.yaml @@ -152,7 +152,6 @@ spec: {{- if .Values.triggerer.customLivenessProbe }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customLivenessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.triggerer.livenessProbe.enabled }} - {{- $livenessTimeout := sub (int .Values.triggerer.livenessProbe.timeoutSeconds) 1 }} livenessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.triggerer.livenessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -163,6 +162,7 @@ spec: {{- if .Values.triggerer.customReadinessProbe }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customReadinessProbe "context" $) | trim | nindent 12 }} {{- else if .Values.triggerer.readinessProbe.enabled }} + {{- $readinessTimeout := sub (int .Values.triggerer.readinessProbe.timeoutSeconds) 1 }} readinessProbe: {{- include "common.tplvalues.render" (dict "value" (omit .Values.triggerer.readinessProbe "enabled") "context" $) | nindent 12 }} exec: command: @@ -170,7 +170,7 @@ spec: - -ec - | export CONNECTION_CHECK_MAX_COUNT=0 - timeout {{ $livenessTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} + timeout {{ $readinessTimeout }} airflow jobs check --job-type TriggererJob --local --limit 0 {{- if not .Values.diagnosticMode.enabled }} 2>/dev/null {{- end }} {{- end }} {{- if .Values.triggerer.customStartupProbe }} startupProbe: {{- include "common.tplvalues.render" (dict "value" .Values.triggerer.customStartupProbe "context" $) | nindent 12 }} From 201fb0705eeae3cafdd810f0dc6730438bcae9c8 Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 22:00:41 +0200 Subject: [PATCH 12/13] [bitnami/airflow] Markdown linter Signed-off-by: Gonzalo Gomez Gracia --- bitnami/airflow/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bitnami/airflow/README.md b/bitnami/airflow/README.md index 5af8e36978426e..5adcbd1cb19274 100644 --- a/bitnami/airflow/README.md +++ b/bitnami/airflow/README.md @@ -1295,12 +1295,15 @@ This major release adds support for Airflow `3.x.y` series. Additionally, previo We recommend following the next procedure in order to upgrade from `22.x.y` chart version to the `23.x.y` series, and also upgrade to Airflow `3.y.z` series: - Upgrade your release (maintaining Airflow `2.x.y` series): + ```console helm upgrade airflow oci://REGISTRY_NAME/REPOSITORY_NAME/airflow --set image.tag=2 ``` - Follow the recommended steps for the database backup and the DAGs files verification available at the [official "upgrading to Airflow 3" guide](https://airflow.apache.org/docs/apache-airflow/stable/installation/upgrading_to_airflow3.html). + - Upgrade your release now using the default Airflow `3.x.y` series: + ```console helm upgrade airflow oci://REGISTRY_NAME/REPOSITORY_NAME/airflow ``` From ae10a0efab547b1a22d21861d7b6cfcdfe129e14 Mon Sep 17 00:00:00 2001 From: Gonzalo Gomez Gracia Date: Tue, 29 Apr 2025 22:02:58 +0200 Subject: [PATCH 13/13] [bitnami/airflow] Undo haproxy changes Signed-off-by: Gonzalo Gomez Gracia --- bitnami/haproxy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitnami/haproxy/README.md b/bitnami/haproxy/README.md index 5a7c754644cb04..aab3517c59ea2c 100644 --- a/bitnami/haproxy/README.md +++ b/bitnami/haproxy/README.md @@ -400,4 +400,4 @@ Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. +limitations under the License. \ No newline at end of file