From df02e6ec994bbefd4197642ca2622e41188b6dc5 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:31 -0700 Subject: [PATCH 1/6] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 633cebe92..3ee7047d3 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,6 +87,14 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: + @pytest.fixture(scope="class", autouse=True) + def setup(self, project): + with project.adapter.connection_named("__test_grants"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + yield relation + project.adapter.drop_relation(relation) @pytest.fixture(scope="class") def models(self): return { From 84ab513f0688fe86bc5d02be0ff600207458d569 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 21 Jul 2023 15:58:51 -0700 Subject: [PATCH 2/6] use dynamic schema in test_grant_access_to.py --- tests/functional/adapter/test_grant_access_to.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 3ee7047d3..77213bf26 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -95,6 +95,7 @@ def setup(self, project): ) yield relation project.adapter.drop_relation(relation) + @pytest.fixture(scope="class") def models(self): return { From 11de428e14a718e4a928e89d7d42eb1cf2955913 Mon Sep 17 00:00:00 2001 From: Colin Date: Fri, 22 Sep 2023 11:45:24 -0700 Subject: [PATCH 3/6] revert setup --- tests/functional/adapter/test_grant_access_to.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/functional/adapter/test_grant_access_to.py b/tests/functional/adapter/test_grant_access_to.py index 77213bf26..633cebe92 100644 --- a/tests/functional/adapter/test_grant_access_to.py +++ b/tests/functional/adapter/test_grant_access_to.py @@ -87,15 +87,6 @@ def test_grant_access_succeeds(self, project, setup_grant_schema, teardown_grant class TestAccessGrantFails: - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - with project.adapter.connection_named("__test_grants"): - relation = project.adapter.Relation.create( - database=project.database, schema=f"{project.test_schema}_seeds" - ) - yield relation - project.adapter.drop_relation(relation) - @pytest.fixture(scope="class") def models(self): return { From 8bc50ac485103eff4c1b770c4fc7f862c68b6bfc Mon Sep 17 00:00:00 2001 From: Colin Date: Tue, 19 Dec 2023 15:37:05 -0800 Subject: [PATCH 4/6] remove _json_from_json patch --- .changes/unreleased/Fixes-20231219-153446.yaml | 6 ++++++ dbt/adapters/bigquery/connections.py | 11 ----------- dev-requirements.txt | 2 +- tests/functional/adapter/dbt_show/test_dbt_show.py | 8 ++++++++ 4 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 .changes/unreleased/Fixes-20231219-153446.yaml diff --git a/.changes/unreleased/Fixes-20231219-153446.yaml b/.changes/unreleased/Fixes-20231219-153446.yaml new file mode 100644 index 000000000..44a858eb8 --- /dev/null +++ b/.changes/unreleased/Fixes-20231219-153446.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: remove json patch to leverage bigquery-python improvement +time: 2023-12-19T15:34:46.843931-08:00 +custom: + Author: colin-rogers-dbt + Issue: "1055" diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index de84e4bf8..7d179a3db 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -15,7 +15,6 @@ import google.auth import google.auth.exceptions -import google.cloud.bigquery as bigquery import google.cloud.exceptions from google.api_core import retry, client_info from google.auth import impersonated_credentials @@ -63,16 +62,6 @@ ) -# Override broken json deserializer for dbt show --inline -# can remove once this is fixed: https://github.com/googleapis/python-bigquery/issues/1500 -def _json_from_json(value, _): - """NOOP string -> string coercion""" - return json.loads(value) - - -bigquery._helpers._CELLDATA_FROM_JSON["JSON"] = _json_from_json - - @lru_cache() def get_bigquery_defaults(scopes=None) -> Tuple[Any, Optional[str]]: """ diff --git a/dev-requirements.txt b/dev-requirements.txt index a101e8b3e..37fee930d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter - +git+https://github.com/googleapis/python-bigquery.git#egg=google-cloud-bigquery # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black~=23.12 diff --git a/tests/functional/adapter/dbt_show/test_dbt_show.py b/tests/functional/adapter/dbt_show/test_dbt_show.py index 203d7031b..acb54cc47 100644 --- a/tests/functional/adapter/dbt_show/test_dbt_show.py +++ b/tests/functional/adapter/dbt_show/test_dbt_show.py @@ -24,6 +24,10 @@ limit 5 """ +model_with_null_json_struct = """ + select (struct(null)) as null_struct +""" + class TestBigQueryShowLimit(BaseShowLimit): pass @@ -40,7 +44,11 @@ class TestBigQueryShowSqlWorksWithJSONStruct: def models(self): return { "json_struct_model.sql": model_with_json_struct, + "null_json_struct_model.sql": model_with_null_json_struct, } def test_sql_header(self, project): run_dbt(["show", "--select", "json_struct_model"]) + + def test_show_with_null_json_struct(self, project): + run_dbt(["show", "--select", "null_json_struct_model"]) From d484b069a237666b0d5d6836419d4213ad461e46 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 20 Dec 2023 09:14:05 -0800 Subject: [PATCH 5/6] fix import --- dbt/adapters/bigquery/connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/bigquery/connections.py b/dbt/adapters/bigquery/connections.py index 7d179a3db..f8e9ee168 100644 --- a/dbt/adapters/bigquery/connections.py +++ b/dbt/adapters/bigquery/connections.py @@ -15,6 +15,7 @@ import google.auth import google.auth.exceptions +import google.cloud.bigquery import google.cloud.exceptions from google.api_core import retry, client_info from google.auth import impersonated_credentials @@ -39,7 +40,6 @@ from dbt.events.functions import fire_event from dbt.events.types import SQLQuery from dbt.version import __version__ as dbt_version - from dbt.dataclass_schema import ExtensibleDbtClassMixin, StrEnum logger = AdapterLogger("BigQuery") From f19cd9e76de115e45ab75d1c9594e129f53c4446 Mon Sep 17 00:00:00 2001 From: Colin Date: Wed, 7 Feb 2024 16:11:15 -0800 Subject: [PATCH 6/6] remove google-apis install from dev-requirements.txt --- dev-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 37fee930d..bd45c99a1 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,6 @@ # TODO: how to automate switching from develop to version branches? git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter -git+https://github.com/googleapis/python-bigquery.git#egg=google-cloud-bigquery # if version 1.x or greater -> pin to major version # if version 0.x -> pin to minor black~=23.12