From 69f5cbf0f8dbf8d70bdbe1a0b9273f0f0cf286ce Mon Sep 17 00:00:00 2001 From: Yu Ishikawa Date: Sat, 20 Nov 2021 00:03:15 +0900 Subject: [PATCH] Don't apply `require_partition_filter` to a temporary table (#65) * Don't apply `require_partition_filter` to a temporary table * Correct the partition_expiration_days option * Update CHANGELOG.md * Move up changelog entry, add contributor Co-authored-by: Jeremy Cohen --- CHANGELOG.md | 4 ++++ dbt/adapters/bigquery/impl.py | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4bbefcb..93fef4953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,12 +3,16 @@ ### Features - Add optional `scopes` profile configuration argument to reduce the BigQuery OAuth scopes down to the minimal set needed. ([#23](https://github.com/dbt-labs/dbt-bigquery/issues/23), [#63](https://github.com/dbt-labs/dbt-bigquery/pull/63)) +### Fixes +- Don't apply `require_partition_filter` to temporary tables, thereby fixing `insert_overwrite` strategy when partition filter is required ([#64](https://github.com/dbt-labs/dbt-bigquery/issues/64)), ([#65](https://github.com/dbt-labs/dbt-bigquery/pull/65)) + ### Under the hood - Adding `execution_project` to `target` object ([#66](https://github.com/dbt-labs/dbt-bigquery/issues/66)) ### Contributors - [@pgoslatara](https://github.com/pgoslatara) ([#66](https://github.com/dbt-labs/dbt-bigquery/issues/66)) - [@bborysenko](https://github.com/bborysenko) ([#63](https://github.com/dbt-labs/dbt-bigquery/pull/63)) +- [@yu-iskw](https://github.com/yu-iskw) ([#65](https://github.com/dbt-labs/dbt-bigquery/pull/65)) ## dbt-bigquery 1.0.0rc1 (November 10, 2021) diff --git a/dbt/adapters/bigquery/impl.py b/dbt/adapters/bigquery/impl.py index 8e3384dd1..4007ba0e8 100644 --- a/dbt/adapters/bigquery/impl.py +++ b/dbt/adapters/bigquery/impl.py @@ -757,20 +757,22 @@ def get_table_options( ) -> Dict[str, Any]: opts = self.get_common_options(config, node, temporary) - if temporary: - expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)' - opts['expiration_timestamp'] = expiration - if config.get('kms_key_name') is not None: opts['kms_key_name'] = "'{}'".format(config.get('kms_key_name')) - if config.get('require_partition_filter'): - opts['require_partition_filter'] = config.get( - 'require_partition_filter') - - if config.get('partition_expiration_days') is not None: - opts['partition_expiration_days'] = config.get( - 'partition_expiration_days') + if temporary: + expiration = 'TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 12 hour)' + opts['expiration_timestamp'] = expiration + else: + # It doesn't apply the `require_partition_filter` option for a temporary table + # so that we avoid the error by not specifying a partition with a temporary table + # in the incremental model. + if config.get('require_partition_filter') is not None: + opts['require_partition_filter'] = config.get( + 'require_partition_filter') + if config.get('partition_expiration_days') is not None: + opts['partition_expiration_days'] = config.get( + 'partition_expiration_days') return opts