diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b447fe3fee..d75be38310 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,17 +55,13 @@ jobs: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] - airflow-version: ["2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] + airflow-version: ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] dbt-version: ["1.10"] exclude: - - python-version: "3.11" - airflow-version: "2.5" # Apache Airflow versions prior to 2.9.0 have not been tested with Python 3.12. # Official support for Python 3.12 and the corresponding constraints.txt are available only for Apache Airflow >= 2.9.0. # See: https://github.com/apache/airflow/tree/2.9.0?tab=readme-ov-file#requirements # See: https://github.com/apache/airflow/tree/2.8.4?tab=readme-ov-file#requirements - - python-version: "3.12" - airflow-version: "2.5" - python-version: "3.12" airflow-version: "2.6" - python-version: "3.12" @@ -73,8 +69,6 @@ jobs: - python-version: "3.12" airflow-version: "2.8" # Apache Airflow versions prior to 3.1.0 have not been tested with Python 3.13. - - python-version: "3.13" - airflow-version: "2.5" - python-version: "3.13" airflow-version: "2.6" - python-version: "3.13" @@ -133,14 +127,10 @@ jobs: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.13"] - airflow-version: ["2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] + airflow-version: ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] dbt-version: ["1.10"] exclude: - - python-version: "3.11" - airflow-version: "2.5" # Apache Airflow versions prior to 3.1.0 have not been tested with Python 3.13. - - python-version: "3.13" - airflow-version: "2.5" - python-version: "3.13" airflow-version: "2.6" - python-version: "3.13" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3782ce1b98..e9cf24ad60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,7 @@ Breaking changes - ``create_task_metadata`` now expects the ``node_converters`` argument * Drop Python 3.9 Support by @pankajastro in #2118 * Drop Airflow 2.4 support by @pankajastro in #2161 +* Drop Airflow 2.5 support by @pankajastro in #2165 Features diff --git a/pyproject.toml b/pyproject.toml index 98833da545..f985253ec0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ dependencies = [ "aenum", "attrs", - "apache-airflow>=2.5.0", + "apache-airflow>=2.6.0", "deprecation", # Python 3.13 exposes a deprecated operator, we can remove this dependency in the future "Jinja2>=3.0.0", "msgpack", @@ -169,7 +169,7 @@ pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} [[tool.hatch.envs.tests.matrix]] python = ["3.10", "3.11", "3.12", "3.13"] -airflow = ["2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] +airflow = ["2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0", "3.1"] dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index 552c398c93..dd576db1fe 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -41,7 +41,7 @@ uv pip install "apache-airflow==$AIRFLOW_VERSION" apache-airflow-providers-docke uv pip install "gcsfs<2025.3.0" -if [ "$AIRFLOW_VERSION" = "2.5" ] || [ "$AIRFLOW_VERSION" = "2.6" ] ; then +if [ "$AIRFLOW_VERSION" = "2.6" ] ; then uv pip install "apache-airflow-providers-amazon" "apache-airflow==$AIRFLOW_VERSION" "urllib3<2" uv pip install "apache-airflow-providers-cncf-kubernetes" "apache-airflow==$AIRFLOW_VERSION" uv pip install "apache-airflow-providers-google<10.11" "apache-airflow==$AIRFLOW_VERSION" diff --git a/tests/_triggers/test_watcher.py b/tests/_triggers/test_watcher.py index 5a2b344ac5..be37d088f6 100644 --- a/tests/_triggers/test_watcher.py +++ b/tests/_triggers/test_watcher.py @@ -98,7 +98,7 @@ async def test_parse_node_status(self, use_event, xcom_val, expected_status): @pytest.mark.parametrize( "airflow_version, expected_val", [ - (Version("2.5.0"), "af2"), # Airflow < 3 uses get_xcom_val_af2 + (Version("2.11.0"), "af2"), # Airflow < 3 uses get_xcom_val_af2 (Version("3.0.0"), "af3"), # Airflow >= 3 uses get_xcom_val_af3 ], ) diff --git a/tests/operators/test_virtualenv.py b/tests/operators/test_virtualenv.py index 560c75999c..f2eaa352a3 100644 --- a/tests/operators/test_virtualenv.py +++ b/tests/operators/test_virtualenv.py @@ -394,10 +394,6 @@ def test__release_venv_lock_current_process(tmpdir): assert not lockfile.exists() -@pytest.mark.skipif( - AIRFLOW_VERSION < Version("2.5"), - reason="This error is only reproducible with dag.test, which was introduced in Airflow 2.5", -) @pytest.mark.integration def test_integration_virtualenv_operator(caplog): """ diff --git a/tests/utils.py b/tests/utils.py index 02bf6bc29c..7c30e02666 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -74,42 +74,35 @@ def test_dag( dr = None if custom_tester: dr = test_old_dag(dag, conn_file_path) + elif AIRFLOW_VERSION not in (Version("2.10.0"), Version("2.10.1"), Version("2.10.2"), Version("2.11.0")): + dr = new_test_dag(dag) assert check_dag_success(dr, expect_success), f"Dag {dag.dag_id} did not run successfully. State: {dr.state}. " - elif AIRFLOW_VERSION >= version.Version("2.5"): - if AIRFLOW_VERSION not in (Version("2.10.0"), Version("2.10.1"), Version("2.10.2"), Version("2.11.0")): + else: + # This is a work around until we fix the issue in Airflow: + # https://github.com/apache/airflow/issues/42495 + """ + FAILED tests/test_example_dags.py::test_example_dag[example_model_version] - sqlalchemy.exc.PendingRollbackError: + This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). + Original exception was: Can't flush None value found in collection DatasetModel.aliases (Background on this error at: https://sqlalche.me/e/14/7s2a) + FAILED tests/test_example_dags.py::test_example_dag[basic_cosmos_dag] + FAILED tests/test_example_dags.py::test_example_dag[cosmos_profile_mapping] + FAILED tests/test_example_dags.py::test_example_dag[user_defined_profile] + """ + try: dr = new_test_dag(dag) assert check_dag_success( dr, expect_success ), f"Dag {dag.dag_id} did not run successfully. State: {dr.state}. " - else: - # This is a work around until we fix the issue in Airflow: - # https://github.com/apache/airflow/issues/42495 - """ - FAILED tests/test_example_dags.py::test_example_dag[example_model_version] - sqlalchemy.exc.PendingRollbackError: - This Session's transaction has been rolled back due to a previous exception during flush. To begin a new transaction with this Session, first issue Session.rollback(). - Original exception was: Can't flush None value found in collection DatasetModel.aliases (Background on this error at: https://sqlalche.me/e/14/7s2a) - FAILED tests/test_example_dags.py::test_example_dag[basic_cosmos_dag] - FAILED tests/test_example_dags.py::test_example_dag[cosmos_profile_mapping] - FAILED tests/test_example_dags.py::test_example_dag[user_defined_profile] - """ - try: - dr = new_test_dag(dag) - assert check_dag_success( - dr, expect_success - ), f"Dag {dag.dag_id} did not run successfully. State: {dr.state}. " - except sqlalchemy.exc.PendingRollbackError: - warnings.warn( - "Early versions of Airflow 2.10 and Airflow 2.11 have issues when running the test command with DatasetAlias / Datasets" - ) - else: - dr = test_old_dag(dag, conn_file_path) - assert check_dag_success(dr), f"Dag {dag.dag_id} did not run successfully. State: {dr.state}. " - + except sqlalchemy.exc.PendingRollbackError: + warnings.warn( + "Early versions of Airflow 2.10 and Airflow 2.11 have issues when running the test command with DatasetAlias / Datasets" + ) return dr -# DAG.test() was added in Airflow version 2.5.0. And to test on older Airflow versions, we need to copy the -# implementation here. +# TODO: Test operators/test_local.py::test_run_operator_dataset_inlets_and_outlets_airflow_210 +# still depends on this utility. Remove this once that test is fixed. +# https://github.com/astronomer/astronomer-cosmos/issues/2166 @provide_session def test_old_dag( dag,