From cba8123355d8dff7d26ed8e7c28745a556a3bd4e Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 18:39:46 +0530 Subject: [PATCH 01/19] Enable dbt-1.9 and python-3.9 tests --- .github/workflows/test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34d6a73d8a..e1638f511d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main] + branches: [main, enable_dbt-1.9-py-3.9-test] # Also run on pull requests originating from forks. Although this is insecure by default, we need it to run # integration tests on forked PRs. As a guardrail, we’ve added an Authorize step to each job, which requires manually # approving the workflow run for each pushed commit. Approval only happens after a careful code review of the changes. @@ -139,10 +139,7 @@ jobs: dbt-version: "1.9" - python-version: "3.8" dbt-version: "1.10" - # To be fixed in https://github.com/astronomer/astronomer-cosmos/issues/1817 - - python-version: "3.9" - airflow-version: "3.0" - dbt-version: "1.9" + services: postgres: image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 From 4c9637e6e38e8b141e4dee09edcb0da61dba33e3 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 18:51:04 +0530 Subject: [PATCH 02/19] comment out typing_extensions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6d105a233f..97840f967a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,12 +167,12 @@ python = ["3.9", "3.10", "3.11", "3.12"] airflow = ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0"] dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] -[tool.hatch.envs.tests.overrides] -matrix.airflow.dependencies = [ - { value = "typing_extensions<4.6", if = ["2.6"] }, - # Known issue: https://github.com/zmievsa/cadwyn/issues/283 - { value = "typing-extensions!=4.14.0", if = ["3.0"] } -] +#[tool.hatch.envs.tests.overrides] +#matrix.airflow.dependencies = [ +# { value = "typing_extensions<4.6", if = ["2.6"] }, +# # Known issue: https://github.com/zmievsa/cadwyn/issues/283 +# { value = "typing_extensions!=4.14.0", if = ["3.0"] } +#] [tool.hatch.envs.tests.scripts] freeze = "pip freeze" From e6ab1dd4d6921262a3fd4a00d55b0259ece33c93 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 18:54:19 +0530 Subject: [PATCH 03/19] Restrict typing_extensions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97840f967a..7e9effed46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,12 +167,12 @@ python = ["3.9", "3.10", "3.11", "3.12"] airflow = ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9", "2.10", "2.11", "3.0"] dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] -#[tool.hatch.envs.tests.overrides] -#matrix.airflow.dependencies = [ -# { value = "typing_extensions<4.6", if = ["2.6"] }, -# # Known issue: https://github.com/zmievsa/cadwyn/issues/283 -# { value = "typing_extensions!=4.14.0", if = ["3.0"] } -#] +[tool.hatch.envs.tests.overrides] +matrix.airflow.dependencies = [ + { value = "typing_extensions<4.6", if = ["2.6"] }, + # Known issue: https://github.com/zmievsa/cadwyn/issues/283 + { value = "typing_extensions!=4.14.0" } +] [tool.hatch.envs.tests.scripts] freeze = "pip freeze" From 45a1efcccd0ca6290d5bc990313429b3d4a243c5 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 19:03:13 +0530 Subject: [PATCH 04/19] Restrict cadwyn --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e9effed46..e4fa72e24b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,7 @@ dependencies = [ "types-python-dateutil", "Werkzeug<3.0.0", "methodtools", + "cadwyn!=4.14.0" ] pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] @@ -170,8 +171,6 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ { value = "typing_extensions<4.6", if = ["2.6"] }, - # Known issue: https://github.com/zmievsa/cadwyn/issues/283 - { value = "typing_extensions!=4.14.0" } ] [tool.hatch.envs.tests.scripts] From c0ab6400ee7063974396c074bfa5b46cd2fc7a90 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 19:09:08 +0530 Subject: [PATCH 05/19] Revert some changes --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e4fa72e24b..fbbeb44c02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,8 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ { value = "typing_extensions<4.6", if = ["2.6"] }, + # Known issue: https://github.com/zmievsa/cadwyn/issues/283 + { value = "typing_extensions!=4.14.0", if = ["3.0"] } ] [tool.hatch.envs.tests.scripts] From 5a8169ea1bb90b4a4bc9a2de26a3a3403032ef76 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 19:26:55 +0530 Subject: [PATCH 06/19] Disable typing_extensions restriction for AF2.6 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fbbeb44c02..9761dcf876 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,7 +170,7 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ - { value = "typing_extensions<4.6", if = ["2.6"] }, + # { value = "typing_extensions<4.6", if = ["2.6"] }, # Known issue: https://github.com/zmievsa/cadwyn/issues/283 { value = "typing_extensions!=4.14.0", if = ["3.0"] } ] From d83734aec601d9ff96194b97fde1ee446a2e6987 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Mon, 4 Aug 2025 19:31:45 +0530 Subject: [PATCH 07/19] Update path --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6f7991c6d3..9adaef944f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ from unittest.mock import patch import pytest -from airflow.models.connection import Connection +from airflow.models import Connection @pytest.fixture() From 4047fc9e7829d29451bcf7aa9af9bcb937a03d78 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 14:38:44 +0530 Subject: [PATCH 08/19] Fix --- .github/workflows/test.yml | 1 - tests/conftest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1638f511d..f086298582 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,7 +139,6 @@ jobs: dbt-version: "1.9" - python-version: "3.8" dbt-version: "1.10" - services: postgres: image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 diff --git a/tests/conftest.py b/tests/conftest.py index 9adaef944f..6f7991c6d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ from unittest.mock import patch import pytest -from airflow.models import Connection +from airflow.models.connection import Connection @pytest.fixture() From ea43a348dd59f7d9692c553de8f9b88a6621d751 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 14:41:35 +0530 Subject: [PATCH 09/19] Remove typing_extensions limitation --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9761dcf876..ede8e31f16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,9 +170,9 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ - # { value = "typing_extensions<4.6", if = ["2.6"] }, + { value = "typing-extensions<4.6", if = ["2.6"] }, # Known issue: https://github.com/zmievsa/cadwyn/issues/283 - { value = "typing_extensions!=4.14.0", if = ["3.0"] } + # { value = "typing_extensions!=4.14.0", if = ["3.0"] } ] [tool.hatch.envs.tests.scripts] From d877af2587c1c25126576dd931f2e345dc1747e6 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 14:50:59 +0530 Subject: [PATCH 10/19] pin dbt-databricks --- scripts/test/integration-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/integration-setup.sh b/scripts/test/integration-setup.sh index fbaeadd86d..27ba27b6b1 100644 --- a/scripts/test/integration-setup.sh +++ b/scripts/test/integration-setup.sh @@ -31,7 +31,7 @@ else fi if [ "$DBT_VERSION" = "1.10" ]; then - uv pip install -U "dbt-core~=$DBT_VERSION" dbt-postgres dbt-bigquery dbt-vertica "dbt-databricks>=1.10.8" pyspark + uv pip install -U "dbt-core~=$DBT_VERSION" dbt-postgres dbt-bigquery dbt-vertica "dbt-databricks!=1.10.8" pyspark else uv pip install -U "dbt-core~=$DBT_VERSION" dbt-postgres dbt-bigquery dbt-vertica dbt-databricks pyspark fi From 74610da000d41193d75333065f9dfaac2ed460c8 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 17:56:50 +0530 Subject: [PATCH 11/19] Remove typing-extensions limitation for AF2.6 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ede8e31f16..094e263b12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,9 +170,9 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ - { value = "typing-extensions<4.6", if = ["2.6"] }, + # { value = "typing-extensions<4.6", if = ["2.6"] }, # Known issue: https://github.com/zmievsa/cadwyn/issues/283 - # { value = "typing_extensions!=4.14.0", if = ["3.0"] } + { value = "typing_extensions!=4.14.0", if = ["3.0"] } ] [tool.hatch.envs.tests.scripts] From 0cdf2db14118995cbcc63af48b6b9576f892ff0a Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 18:13:12 +0530 Subject: [PATCH 12/19] Pass contriant for google provider and airflow 2.7 --- scripts/test/pre-install-airflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index 3553894594..0586bd03b7 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -53,7 +53,7 @@ elif [ "$AIRFLOW_VERSION" = "2.6" ] ; then elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then uv pip install "apache-airflow-providers-amazon" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" + uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt elif [ "$AIRFLOW_VERSION" = "2.8" ] ; then uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt From 0bd5628ee063ec1abdb7f3cb984c253f1f5bd788 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 18:17:04 +0530 Subject: [PATCH 13/19] Remove apache-airflow-providers-google version restriction for AF2.7 tests --- scripts/test/pre-install-airflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index 0586bd03b7..d10d238912 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -53,7 +53,7 @@ elif [ "$AIRFLOW_VERSION" = "2.6" ] ; then elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then uv pip install "apache-airflow-providers-amazon" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt + # uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt elif [ "$AIRFLOW_VERSION" = "2.8" ] ; then uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt From 0afb1d71f3d973f4eaba41ff39c41d1205474a69 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 18:45:12 +0530 Subject: [PATCH 14/19] Unpin google provider --- scripts/test/pre-install-airflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index d10d238912..c067bb7f8d 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -53,7 +53,7 @@ elif [ "$AIRFLOW_VERSION" = "2.6" ] ; then elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then uv pip install "apache-airflow-providers-amazon" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - # uv pip install "apache-airflow-providers-google>10.11" "apache-airflow==$AIRFLOW_VERSION" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt elif [ "$AIRFLOW_VERSION" = "2.8" ] ; then uv pip install "apache-airflow-providers-amazon[s3fs]" --constraint /tmp/constraint.txt From fd0ef344628f49fc7d709c7bd5fe1d5e9a8df0c3 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 18:48:18 +0530 Subject: [PATCH 15/19] Remove cadwyn --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 094e263b12..57911f2550 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,8 +158,7 @@ dependencies = [ "types-requests", "types-python-dateutil", "Werkzeug<3.0.0", - "methodtools", - "cadwyn!=4.14.0" + "methodtools" ] pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] From 7040de7a05effee88d46aa25870315ab8475e0a1 Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 18:51:00 +0530 Subject: [PATCH 16/19] Restore --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 57911f2550..248615a16e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,7 +169,7 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] [tool.hatch.envs.tests.overrides] matrix.airflow.dependencies = [ - # { value = "typing-extensions<4.6", if = ["2.6"] }, + { value = "typing-extensions<4.6", if = ["2.6"] }, # Known issue: https://github.com/zmievsa/cadwyn/issues/283 { value = "typing_extensions!=4.14.0", if = ["3.0"] } ] From 21cf8d0b82ae10b4d722f12db062ac4a7ea4233c Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 19:23:22 +0530 Subject: [PATCH 17/19] Wrap test with skip_on_empty_operator --- scripts/test/pre-install-airflow.sh | 2 +- tests/operators/test_gcp_cloud_run_job.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test/pre-install-airflow.sh b/scripts/test/pre-install-airflow.sh index c067bb7f8d..1716a2fafa 100755 --- a/scripts/test/pre-install-airflow.sh +++ b/scripts/test/pre-install-airflow.sh @@ -47,7 +47,7 @@ if [ "$AIRFLOW_VERSION" = "2.4" ] || [ "$AIRFLOW_VERSION" = "2.5" ] || [ "$AIRFL elif [ "$AIRFLOW_VERSION" = "2.6" ] ; then uv pip install "apache-airflow-providers-amazon" --constraint /tmp/constraint.txt uv pip install "apache-airflow-providers-cncf-kubernetes" --constraint /tmp/constraint.txt - uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt + uv pip install "apache-airflow-providers-google" --constraint /tmp/constraint.txt uv pip install apache-airflow-providers-microsoft-azure --constraint /tmp/constraint.txt uv pip install "pydantic<2.0" elif [ "$AIRFLOW_VERSION" = "2.7" ] ; then diff --git a/tests/operators/test_gcp_cloud_run_job.py b/tests/operators/test_gcp_cloud_run_job.py index 16f30db32e..1a36ac74dc 100644 --- a/tests/operators/test_gcp_cloud_run_job.py +++ b/tests/operators/test_gcp_cloud_run_job.py @@ -61,6 +61,7 @@ def skip_on_empty_operator(test_func): )(test_func) +@skip_on_empty_operator def test_overrides_missing(): """ The overrides parameter needed to pass the dbt command was added in apache-airflow-providers-google==10.11.0. From c095bb3114ee88dd4a95edbd43c7effef61aed9c Mon Sep 17 00:00:00 2001 From: pankajastro Date: Thu, 7 Aug 2025 19:24:18 +0530 Subject: [PATCH 18/19] Fix --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 248615a16e..d1a691631d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,7 +158,7 @@ dependencies = [ "types-requests", "types-python-dateutil", "Werkzeug<3.0.0", - "methodtools" + "methodtools", ] pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] @@ -171,7 +171,7 @@ dbt = ["1.5", "1.6", "1.7", "1.8", "1.9", "1.10", "2.0"] matrix.airflow.dependencies = [ { value = "typing-extensions<4.6", if = ["2.6"] }, # Known issue: https://github.com/zmievsa/cadwyn/issues/283 - { value = "typing_extensions!=4.14.0", if = ["3.0"] } + { value = "typing-extensions!=4.14.0", if = ["3.0"] } ] [tool.hatch.envs.tests.scripts] From 4e3ff0d48927ebfc892b555f139898a473bd484f Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Thu, 7 Aug 2025 20:37:47 +0530 Subject: [PATCH 19/19] Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f086298582..24ac1988c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main, enable_dbt-1.9-py-3.9-test] + branches: [main] # Also run on pull requests originating from forks. Although this is insecure by default, we need it to run # integration tests on forked PRs. As a guardrail, we’ve added an Authorize step to each job, which requires manually # approving the workflow run for each pushed commit. Approval only happens after a careful code review of the changes.