Use latest minor versions for dbt adapters to get in compatibility fixes

.github/workflows/test.yml

@@ -356,12 +356,6 @@ jobs:
         python-version: [ "3.11" ]
         airflow-version: [ "2.10", "3.0" ]
         dbt-version: ["1.5", "1.6", "1.7", "1.8", "1.9"]
-        # TODO: Add support for dbt 1.7 for Airflow 3.0.
-        #       Issue dbt 1.7: https://github.com/astronomer/astronomer-cosmos/issues/1709
-        exclude:
-          - python-version: "3.11"
-            airflow-version: "3.0"
-            dbt-version: "1.7"
     services:
       postgres:
         image: postgres

scripts/test/integration-dbt-async.sh

@@ -8,9 +8,16 @@ DBT_VERSION="$1"
 echo "DBT_VERSION:"
 echo "$DBT_VERSION"
 
+# Calculate next minor version
+NEXT_MINOR_VERSION=$(echo "$DBT_VERSION" | awk -F. '{print $1"."$2+1}')
+
+echo "Installing dbt adapters for DBT_VERSION=$DBT_VERSION (<$NEXT_MINOR_VERSION)"
 
 pip uninstall dbt-adapters dbt-common dbt-core dbt-extractor dbt-postgres dbt-semantic-interfaces -y
-pip install "dbt-postgres==$DBT_VERSION"  "dbt-databricks==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION"
+pip install -U \
+  "dbt-postgres>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \
+  "dbt-databricks>=$DBT_VERSION,<$NEXT_MINOR_VERSION" \
+  "dbt-bigquery>=$DBT_VERSION,<$NEXT_MINOR_VERSION"
 
 # apache-airflow-core 3.0.0 requires pydantic>=2.11.0, but the above dbt adapters in case of version 1.6 and 1.9 install
 # pydantic 1.10.22 which make it incompatible.
@@ -24,6 +31,17 @@ if [ "$DBT_VERSION" = "1.6" ] || [ "$DBT_VERSION" = "1.9" ]; then
     pip install "pydantic>2.11.0"
 fi
 
+# As on 28th April, 2025, the latest patch dbt-bigquery 1.6.13 on the 1.6 minor is not yet compatible with the latest
+# release of google-cloud-bigquery 3.31.0 as it tries to access a protected attribute which no longer exists in that
+# latest version for google-cloud-bigquery and gives the below error:
+# stderr: module 'google.cloud.bigquery._helpers' has no attribute '_CELLDATA_FROM_JSON'.
+# Hence, we need to install the previous version of google-cloud-bigquery <3.31.0 that still has the protected attribute
+# 'google.cloud.bigquery._helpers._CELLDATA_FROM_JSON' available to make it compatible with dbt-bigquery 1.6.13.
+if [ "$DBT_VERSION" = "1.6" ]; then
+    echo "DBT_VERSION is $DBT_VERSION, installing google-cloud-bigquery<3.31.0 for compatibility issue."
+    pip install "google-cloud-bigquery<3.31.0"
+fi
+
 export SOURCE_RENDERING_BEHAVIOR=all
 rm -rf airflow.*
 
@@ -37,13 +55,6 @@ else
     airflow db init
 fi
 
-if [ "$DBT_VERSION" = "1.7" ]; then
-    # Otherwise, we will get the following error:
-    # stderr: MessageToJson() got an unexpected keyword argument 'including_default_value_fields'
-    echo "DBT version is 1.7 — Installing protobuf==4.25.6..."
-    pip install protobuf==4.25.6
-fi
-
 rm -rf dbt/jaffle_shop/dbt_packages
 pytest -vv \
     --cov=cosmos \