diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b8a1a35eb0..93d4bebdbf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -356,12 +356,6 @@ jobs: python-version: [ "3.11" ] airflow-version: [ "2.10", "3.0" ] dbt-version: ["1.5", "1.6", "1.7", "1.8", "1.9"] - # TODO: Add support for dbt 1.7 for Airflow 3.0. - # Issue dbt 1.7: https://github.com/astronomer/astronomer-cosmos/issues/1709 - exclude: - - python-version: "3.11" - airflow-version: "3.0" - dbt-version: "1.7" services: postgres: image: postgres diff --git a/scripts/test/integration-dbt-async.sh b/scripts/test/integration-dbt-async.sh index eefce0bb97..2dbd2a247e 100644 --- a/scripts/test/integration-dbt-async.sh +++ b/scripts/test/integration-dbt-async.sh @@ -8,9 +8,16 @@ DBT_VERSION="$1" echo "DBT_VERSION:" echo "$DBT_VERSION" +# Calculate next minor version +NEXT_MINOR_VERSION=$(echo "$DBT_VERSION" | awk -F. '{print $1"."$2+1}') + +echo "Installing dbt adapters for DBT_VERSION=$DBT_VERSION (<$NEXT_MINOR_VERSION)" pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y -pip install "dbt-postgres==$DBT_VERSION" "dbt-databricks==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" +pip install -U \ + "dbt-postgres>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \ + "dbt-databricks>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \ + "dbt-bigquery>=$DBT_VERSION,<$NEXT_MINOR_VERSION" # apache-airflow-core 3.0.0 requires pydantic>=2.11.0, but the above dbt adapters in case of version 1.6 and 1.9 install # pydantic 1.10.22 which make it incompatible. @@ -24,6 +31,17 @@ if [ "$DBT_VERSION" = "1.6" ] || [ "$DBT_VERSION" = "1.9" ]; then pip install "pydantic>2.11.0" fi +# As on 28th April, 2025, the latest patch dbt-bigquery 1.6.13 on the 1.6 minor is not yet compatible with the latest +# release of google-cloud-bigquery 3.31.0 as it tries to access a protected attribute which no longer exists in that +# latest version for google-cloud-bigquery and gives the below error: +# stderr: module 'google.cloud.bigquery._helpers' has no attribute '_CELLDATA_FROM_JSON'. +# Hence, we need to install the previous version of google-cloud-bigquery <3.31.0 that still has the protected attribute +# 'google.cloud.bigquery._helpers._CELLDATA_FROM_JSON' available to make it compatible with dbt-bigquery 1.6.13. +if [ "$DBT_VERSION" = "1.6" ]; then + echo "DBT_VERSION is $DBT_VERSION, installing google-cloud-bigquery<3.31.0 for compatibility issue." + pip install "google-cloud-bigquery<3.31.0" +fi + export SOURCE_RENDERING_BEHAVIOR=all rm -rf airflow.* @@ -37,13 +55,6 @@ else airflow db init fi -if [ "$DBT_VERSION" = "1.7" ]; then - # Otherwise, we will get the following error: - # stderr: MessageToJson() got an unexpected keyword argument 'including_default_value_fields' - echo "DBT version is 1.7 — Installing protobuf==4.25.6..." - pip install protobuf==4.25.6 -fi - rm -rf dbt/jaffle_shop/dbt_packages pytest -vv \ --cov=cosmos \