From 422fd2cfc86b1bfdcda142efbbfa40abda0f213c Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Tue, 20 May 2025 16:19:20 +0100 Subject: [PATCH] Fix Airflow dependencies in the CI Our CI tests have been quite unstable lately, this PR aims to fix the most recent issues. --- scripts/test/pre-install-airflow.sh | 32 +++++++++++++++++------------ 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index edc87c9dc0..e56cd79903 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -26,9 +26,7 @@ mv /tmp/constraint.txt.tmp /tmp/constraint.txt pip install uv uv pip install pip --upgrade -uv pip install "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt -uv pip install apache-airflow-providers-docker --constraint /tmp/constraint.txt -uv pip install apache-airflow-providers-postgres --constraint /tmp/constraint.txt +uv pip install "apache-airflow==$AIRFLOW_VERSION" apache-airflow-providers-docker apache-airflow-providers-postgres --constraint /tmp/constraint.txt # Due to issue https://github.com/fsspec/gcsfs/issues/664 uv pip install "gcsfs<2025.3.0" @@ -44,10 +42,21 @@ elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt -else +elif [ "$AIRFLOW_VERSION" = "2.8" ] ; then uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - + uv pip install "apache-airflow-providers-google<=10.26" "apache-airflow==$AIRFLOW_VERSION" + # The Airflow 2.8 constraints file at + # https://raw.githubusercontent.com/apache/airflow/constraints-2.8.0/constraints-3.11.txt + # specifies apache-airflow-providers-microsoft-azure==8.4.0. However, our Azure connection setup in the CI, + # previously led to authentication issues with this version. This issue got resolved in + # apache-airflow-providers-microsoft-azure==8.5.0. Hence, we are using apache-airflow-providers-microsoft-azure>=8.5.0 + # and skipping installation with constraints, as the specified version does not meet our requirements. + uv pip install "apache-airflow-providers-microsoft-azure>=8.5.0" "apache-airflow==$AIRFLOW_VERSION" +elif [ "$AIRFLOW_VERSION" = "2.9" ] ; then + uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-microsoft-azure" --constraint /tmp/constraint.txt # The Airflow 2.9 constraints file at # https://raw.githubusercontent.com/apache/airflow/constraints-2.9.0/constraints-3.11.txt # specifies apache-airflow-providers-google==10.16.0. However, our CI setup uses a Google connection without a token, @@ -56,14 +65,11 @@ else # we are using apache-airflow-providers-google>=10.17.0 and skipping constraints installation, as the specified # version does not meet our requirements. uv pip install "apache-airflow-providers-google>=10.17.0" "apache-airflow==$AIRFLOW_VERSION" - - # The Airflow 2.8 constraints file at - # https://raw.githubusercontent.com/apache/airflow/constraints-2.8.0/constraints-3.11.txt - # specifies apache-airflow-providers-microsoft-azure==8.4.0. However, our Azure connection setup in the CI, - # previously led to authentication issues with this version. This issue got resolved in - # apache-airflow-providers-microsoft-azure==8.5.0. Hence, we are using apache-airflow-providers-microsoft-azure>=8.5.0 - # and skipping installation with constraints, as the specified version does not meet our requirements. - uv pip install "apache-airflow-providers-microsoft-azure>=8.5.0" "apache-airflow==$AIRFLOW_VERSION" +else + uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-microsoft-azure" --constraint /tmp/constraint.txt fi rm /tmp/constraint.txt