From 14cc9f9ee538b42c8d17d98dfd12b1eb6e8d7640 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 28 Apr 2025 22:57:45 +0530 Subject: [PATCH 1/6] Use latest minor versions for dbt adapters to get in compatibility fixes --- scripts/test/integration-dbt-async.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/test/integration-dbt-async.sh b/scripts/test/integration-dbt-async.sh index eefce0bb97..f4c4215091 100644 --- a/scripts/test/integration-dbt-async.sh +++ b/scripts/test/integration-dbt-async.sh @@ -10,7 +10,7 @@ echo "$DBT_VERSION" pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y -pip install "dbt-postgres==$DBT_VERSION" "dbt-databricks==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" +pip install -U "dbt-postgres~=$DBT_VERSION" "dbt-databricks~=$DBT_VERSION" "dbt-bigquery~=$DBT_VERSION" # apache-airflow-core 3.0.0 requires pydantic>=2.11.0, but the above dbt adapters in case of version 1.6 and 1.9 install # pydantic 1.10.22 which make it incompatible. @@ -37,13 +37,6 @@ else airflow db init fi -if [ "$DBT_VERSION" = "1.7" ]; then - # Otherwise, we will get the following error: - # stderr: MessageToJson() got an unexpected keyword argument 'including_default_value_fields' - echo "DBT version is 1.7 — Installing protobuf==4.25.6..." - pip install protobuf==4.25.6 -fi - rm -rf dbt/jaffle_shop/dbt_packages pytest -vv \ --cov=cosmos \ From 0772c2a54b267ac50900316f4e448dab203dfeee Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 28 Apr 2025 23:04:30 +0530 Subject: [PATCH 2/6] Run tests in CI --- .github/workflows/test.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8a1a35eb0..998080bcc6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main] + branches: [main,enable-ci-dbt-async-1.7-af3] pull_request_target: # Also run on pull requests originated from forks branches: [main] @@ -356,12 +356,6 @@ jobs: python-version: [ "3.11" ] airflow-version: [ "2.10", "3.0" ] dbt-version: ["1.5", "1.6", "1.7", "1.8", "1.9"] - # TODO: Add support for dbt 1.7 for Airflow 3.0. - # Issue dbt 1.7: https://github.com/astronomer/astronomer-cosmos/issues/1709 - exclude: - - python-version: "3.11" - airflow-version: "3.0" - dbt-version: "1.7" services: postgres: image: postgres From 2b56397d1b211a78c36514ecbba1de809fc4f653 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 28 Apr 2025 23:20:40 +0530 Subject: [PATCH 3/6] Correct installation with upper bounds for minor versions of dbt adapters --- scripts/test/integration-dbt-async.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/test/integration-dbt-async.sh b/scripts/test/integration-dbt-async.sh index f4c4215091..c13bff893e 100644 --- a/scripts/test/integration-dbt-async.sh +++ b/scripts/test/integration-dbt-async.sh @@ -8,9 +8,17 @@ DBT_VERSION="$1" echo "DBT_VERSION:" echo "$DBT_VERSION" +# Calculate next minor version +NEXT_MINOR_VERSION=$(echo "$DBT_VERSION" | awk -F. '{print $1"."$2+1}') + +echo "Installing dbt adapters for DBT_VERSION=$DBT_VERSION (<$NEXT_MINOR_VERSION)" pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y -pip install -U "dbt-postgres~=$DBT_VERSION" "dbt-databricks~=$DBT_VERSION" "dbt-bigquery~=$DBT_VERSION" +#pip install -U "dbt-postgres~=$DBT_VERSION" "dbt-databricks~=$DBT_VERSION" "dbt-bigquery~=$DBT_VERSION" +pip install -U \ + "dbt-postgres>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \ + "dbt-databricks>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \ + "dbt-bigquery>=$DBT_VERSION,<$NEXT_MINOR_VERSION" # apache-airflow-core 3.0.0 requires pydantic>=2.11.0, but the above dbt adapters in case of version 1.6 and 1.9 install # pydantic 1.10.22 which make it incompatible. From 86c3751647c5fd0e67e91464a73c2c90a8a6f1f4 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Mon, 28 Apr 2025 23:30:18 +0530 Subject: [PATCH 4/6] Restrict google-cloud-bigquery<3.31.0 for dbt-bigquery 1.6 for compatibility --- scripts/test/integration-dbt-async.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/test/integration-dbt-async.sh b/scripts/test/integration-dbt-async.sh index c13bff893e..f55b7e49db 100644 --- a/scripts/test/integration-dbt-async.sh +++ b/scripts/test/integration-dbt-async.sh @@ -32,6 +32,17 @@ if [ "$DBT_VERSION" = "1.6" ] || [ "$DBT_VERSION" = "1.9" ]; then pip install "pydantic>2.11.0" fi +# As on 28th April, 2025, the latest patch dbt-bigquery 1.6.13 on the 1.6 minor is not yet compatible with the latest +# release of google-cloud-bigquery 3.31.0 as it tries to access a protected attribute which no longer exists in that +# latest version for google-cloud-bigquery and gives the below error: +# stderr: module 'google.cloud.bigquery._helpers' has no attribute '_CELLDATA_FROM_JSON'. +# Hence, we need to install the previous version of google-cloud-bigquery <3.31.0 that still has the protected attribute +# 'google.cloud.bigquery._helpers._CELLDATA_FROM_JSON' available to make it compatible with dbt-bigquery 1.6.13. +if [ "$DBT_VERSION" = "1.6" ]; then + echo "DBT_VERSION is $DBT_VERSION, installing google-cloud-bigquery<3.31.0 for compatibility issue." + pip install "google-cloud-bigquery<3.31.0" +fi + export SOURCE_RENDERING_BEHAVIOR=all rm -rf airflow.* From bf66c50a33df5c3017c3af3baa1123999acf56b1 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Tue, 29 Apr 2025 11:00:02 +0530 Subject: [PATCH 5/6] Update .github/workflows/test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 998080bcc6..93d4bebdbf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: test on: push: # Run on pushes to the default branch - branches: [main,enable-ci-dbt-async-1.7-af3] + branches: [main] pull_request_target: # Also run on pull requests originated from forks branches: [main] From 7d56029efbe183af27e479665ac435614a6d5d2e Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Tue, 29 Apr 2025 11:00:11 +0530 Subject: [PATCH 6/6] Update scripts/test/integration-dbt-async.sh --- scripts/test/integration-dbt-async.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test/integration-dbt-async.sh b/scripts/test/integration-dbt-async.sh index f55b7e49db..2dbd2a247e 100644 --- a/scripts/test/integration-dbt-async.sh +++ b/scripts/test/integration-dbt-async.sh @@ -14,7 +14,6 @@ NEXT_MINOR_VERSION=$(echo "$DBT_VERSION" | awk -F. '{print $1"."$2+1}') echo "Installing dbt adapters for DBT_VERSION=$DBT_VERSION (<$NEXT_MINOR_VERSION)" pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y -#pip install -U "dbt-postgres~=$DBT_VERSION" "dbt-databricks~=$DBT_VERSION" "dbt-bigquery~=$DBT_VERSION" pip install -U \ "dbt-postgres>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \ "dbt-databricks>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \