Merge pull request #33 from fivetran/MagicBot/dbt-utils-cross-db-migr…

…ation Updates for dbt-utils to dbt-core cross-db macro migration
fivetran · Dec 21, 2022 · 69c8873 · 69c8873
2 parents db9bb31 + bc4e50f
commit 69c8873
Show file tree

Hide file tree

Showing 22 changed files with 164 additions and 125 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,4 +1,3 @@
-Pull Request
 **Are you a current Fivetran customer?** 
 <!--- Please tell us your name, title and company -->
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,42 @@
+# dbt_apple_search_ads_source v0.2.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #33](https://github.com/fivetran/dbt_apple_search_ads_source/pull/33) includes the following breaking changes:
+- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
+    - `any_value`
+    - `bool_or`
+    - `cast_bool_to_text`
+    - `concat`
+    - `date_trunc`
+    - `dateadd`
+    - `datediff`
+    - `escape_single_quotes`
+    - `except`
+    - `hash`
+    - `intersect`
+    - `last_day`
+    - `length`
+    - `listagg`
+    - `position`
+    - `replace`
+    - `right`
+    - `safe_cast`
+    - `split_part`
+    - `string_literal`
+    - `type_bigint`
+    - `type_float`
+    - `type_int`
+    - `type_numeric`
+    - `type_string`
+    - `type_timestamp`
+    - `array_append`
+    - `array_concat`
+    - `array_construct`
+- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
+    - `dbt.current_timestamp_backcompat`
+    - `dbt.current_timestamp_in_utc_backcompat`
+- `packages.yml` has been updated to reflect new default `fivetran/fivetran_utils` version, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+
 # dbt_apple_search_ads_source v0.1.3
 
 ## 🕷️ Bugfixes 🕷️

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
         href="https://github.com/fivetran/dbt_apple_search_ads_source/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
     <a alt="dbt-core">
-        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.0.0_<2.0.0-orange.svg" /></a>
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_,<2.0.0-orange.svg" /></a>
     <a alt="Maintained?">
         <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
     <a alt="PRs">
@@ -39,7 +39,7 @@ Include the following apple_search_ads_source package version in your `packages.
 ```yaml
 packages:
   - package: fivetran/apple_search_ads_source
-    version: [">=0.1.0", "<0.2.0"]
+    version: [">=0.2.0", "<0.3.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `apple_search_ads` schema. If this is not where your Apple Search Ads data is (for example, if your Apple Search Ads schema is named `apple_search_ads_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -123,10 +123,13 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/fivetran_utils
-      version: [">=0.3.0", "<0.4.0"]
+      version: [">=0.4.0", "<0.5.0"]
 
     - package: dbt-labs/dbt_utils
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
 ```
 
 # 🙌 How is this package maintained and can I contribute?

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,9 +1,8 @@
 name: 'apple_search_ads_source'
-version: '0.1.3'
-config-version: 2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
-
+version: '0.2.0'
 
+config-version: 2
+require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:
   apple_search_ads_source:
     organization: "{{ source('apple_search_ads', 'organization') }}"
@@ -16,13 +15,11 @@ vars:
     keyword_history: "{{ source('apple_search_ads', 'keyword_history') }}"
     keyword_report: "{{ source('apple_search_ads', 'keyword_report') }}"
     search_term_report: "{{ source('apple_search_ads', 'search_term_report') }}"
-
   apple_search_ads__ad_group_passthrough_metrics: []
   apple_search_ads__campaign_passthrough_metrics: []
   apple_search_ads__ad_passthrough_metrics: []
-  apple_search_ads__keyword_passthrough_metrics: [] 
+  apple_search_ads__keyword_passthrough_metrics: []
   apple_search_ads__search_term_passthrough_metrics: []
-
 models:
   apple_search_ads_source:
     +schema: apple_search_ads_source

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'apple_search_ads_source_integration_tests'
-version: '0.1.3'
+version: '0.2.0'
 
 profile: 'integration_tests' 
 

diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -1,7 +1,8 @@
-dbt-snowflake>=1.0.0,<1.3.0
-dbt-bigquery>=1.0.0,<1.3.0
-dbt-redshift>=1.0.0,<1.3.0
-dbt-postgres>=1.0.0,<1.3.0
-dbt-spark>=1.0.0,<1.3.0
-dbt-spark[PyHive]>=1.0.0,<1.3.0
-dbt-databricks>=1.0.0,<1.3.0
+dbt-snowflake>=1.3.0,<2.0.0
+dbt-bigquery>=1.3.0,<2.0.0
+dbt-redshift>=1.3.0,<2.0.0
+dbt-postgres>=1.3.0,<2.0.0
+dbt-spark>=1.3.0,<2.0.0
+dbt-spark[PyHive]>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+
diff --git a/macros/get_ad_group_history_columns.sql b/macros/get_ad_group_history_columns.sql
@@ -1,14 +1,14 @@
 {% macro get_ad_group_history_columns() %}
 
 {% set columns = [
-    {"name": "campaign_id", "datatype": dbt_utils.type_int()},
-    {"name": "end_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "modification_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "name", "datatype": dbt_utils.type_string()},
-    {"name": "organization_id", "datatype": dbt_utils.type_int()},
-    {"name": "start_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "status", "datatype": dbt_utils.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "end_time", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "modification_time", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "organization_id", "datatype": dbt.type_int()},
+    {"name": "start_time", "datatype": dbt.type_timestamp()},
+    {"name": "status", "datatype": dbt.type_string()},
 ] %}
 
 {{ return(columns) }}

diff --git a/macros/get_ad_group_report_columns.sql b/macros/get_ad_group_report_columns.sql
@@ -1,14 +1,14 @@
 {% macro get_ad_group_report_columns() %}
 
 {% set columns = [
-    {"name": "ad_group_id", "datatype": dbt_utils.type_int()},
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "date", "datatype": "date"},
-    {"name": "impressions", "datatype": dbt_utils.type_int()},
-    {"name": "local_spend_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "local_spend_currency", "datatype": dbt_utils.type_string()},
-    {"name": "new_downloads", "datatype": dbt_utils.type_int()},
-    {"name": "redownloads", "datatype": dbt_utils.type_int()},
-    {"name": "taps", "datatype": dbt_utils.type_int()}
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "local_spend_amount", "datatype": dbt.type_numeric()},
+    {"name": "local_spend_currency", "datatype": dbt.type_string()},
+    {"name": "new_downloads", "datatype": dbt.type_int()},
+    {"name": "redownloads", "datatype": dbt.type_int()},
+    {"name": "taps", "datatype": dbt.type_int()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('apple_search_ads__ad_group_passthrough_metrics')) }}

diff --git a/macros/get_ad_history_columns.sql b/macros/get_ad_history_columns.sql
@@ -1,14 +1,14 @@
 {% macro get_ad_history_columns() %}
 
 {% set columns = [
-    {"name": "ad_group_id", "datatype": dbt_utils.type_int()},
-    {"name": "campaign_id", "datatype": dbt_utils.type_int()},
-    {"name": "creation_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "modification_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "name", "datatype": dbt_utils.type_string()},
-    {"name": "org_id", "datatype": dbt_utils.type_int()},
-    {"name": "status", "datatype": dbt_utils.type_string()}
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "creation_time", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "modification_time", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "org_id", "datatype": dbt.type_int()},
+    {"name": "status", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

diff --git a/macros/get_ad_level_report_columns.sql b/macros/get_ad_level_report_columns.sql
@@ -1,16 +1,16 @@
 {% macro get_ad_level_report_columns() %}
 
 {% set columns = [
-    {"name": "ad_group_id", "datatype": dbt_utils.type_int()},
-    {"name": "ad_id", "datatype": dbt_utils.type_int()},
-    {"name": "campaign_id", "datatype": dbt_utils.type_int()},
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "ad_id", "datatype": dbt.type_int()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
     {"name": "date", "datatype": "date"},
-    {"name": "impressions", "datatype": dbt_utils.type_int()},
-    {"name": "local_spend_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "local_spend_currency", "datatype": dbt_utils.type_string()},
-    {"name": "new_downloads", "datatype": dbt_utils.type_int()},
-    {"name": "redownloads", "datatype": dbt_utils.type_int()},
-    {"name": "taps", "datatype": dbt_utils.type_int()}
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "local_spend_amount", "datatype": dbt.type_numeric()},
+    {"name": "local_spend_currency", "datatype": dbt.type_string()},
+    {"name": "new_downloads", "datatype": dbt.type_int()},
+    {"name": "redownloads", "datatype": dbt.type_int()},
+    {"name": "taps", "datatype": dbt.type_int()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('apple_search_ads__ad_passthrough_metrics')) }}

diff --git a/macros/get_campaign_history_columns.sql b/macros/get_campaign_history_columns.sql
@@ -1,13 +1,13 @@
 {% macro get_campaign_history_columns() %}
 
 {% set columns = [
-    {"name": "end_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "modification_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "name", "datatype": dbt_utils.type_string()},
-    {"name": "organization_id", "datatype": dbt_utils.type_int()},
-    {"name": "start_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "status", "datatype": dbt_utils.type_string()}
+    {"name": "end_time", "datatype": dbt.type_timestamp()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "modification_time", "datatype": dbt.type_timestamp()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "organization_id", "datatype": dbt.type_int()},
+    {"name": "start_time", "datatype": dbt.type_timestamp()},
+    {"name": "status", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

diff --git a/macros/get_campaign_report_columns.sql b/macros/get_campaign_report_columns.sql
@@ -2,13 +2,13 @@
 
 {% set columns = [
     {"name": "date", "datatype": "date"},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "impressions", "datatype": dbt_utils.type_int()},
-    {"name": "local_spend_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "local_spend_currency", "datatype": dbt_utils.type_string()},
-    {"name": "new_downloads", "datatype": dbt_utils.type_int()},
-    {"name": "redownloads", "datatype": dbt_utils.type_int()},
-    {"name": "taps", "datatype": dbt_utils.type_int()}
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "local_spend_amount", "datatype": dbt.type_numeric()},
+    {"name": "local_spend_currency", "datatype": dbt.type_string()},
+    {"name": "new_downloads", "datatype": dbt.type_int()},
+    {"name": "redownloads", "datatype": dbt.type_int()},
+    {"name": "taps", "datatype": dbt.type_int()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('apple_search_ads__campaign_passthrough_metrics')) }}

diff --git a/macros/get_keyword_history_columns.sql b/macros/get_keyword_history_columns.sql
@@ -1,15 +1,15 @@
 {% macro get_keyword_history_columns() %}
 
 {% set columns = [
-    {"name": "ad_group_id", "datatype": dbt_utils.type_int()},
-    {"name": "bid_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "bid_currency", "datatype": dbt_utils.type_string()},
-    {"name": "campaign_id", "datatype": dbt_utils.type_int()},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "match_type", "datatype": dbt_utils.type_string()},
-    {"name": "modification_time", "datatype": dbt_utils.type_timestamp()},
-    {"name": "status", "datatype": dbt_utils.type_string()},
-    {"name": "text", "datatype": dbt_utils.type_string()}
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "bid_amount", "datatype": dbt.type_numeric()},
+    {"name": "bid_currency", "datatype": dbt.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "match_type", "datatype": dbt.type_string()},
+    {"name": "modification_time", "datatype": dbt.type_timestamp()},
+    {"name": "status", "datatype": dbt.type_string()},
+    {"name": "text", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

diff --git a/macros/get_keyword_report_columns.sql b/macros/get_keyword_report_columns.sql
@@ -2,13 +2,13 @@
 
 {% set columns = [
     {"name": "date", "datatype": "date"},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "impressions", "datatype": dbt_utils.type_int()},
-    {"name": "local_spend_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "local_spend_currency", "datatype": dbt_utils.type_string()},
-    {"name": "new_downloads", "datatype": dbt_utils.type_int()},
-    {"name": "redownloads", "datatype": dbt_utils.type_int()},
-    {"name": "taps", "datatype": dbt_utils.type_int()}
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "local_spend_amount", "datatype": dbt.type_numeric()},
+    {"name": "local_spend_currency", "datatype": dbt.type_string()},
+    {"name": "new_downloads", "datatype": dbt.type_int()},
+    {"name": "redownloads", "datatype": dbt.type_int()},
+    {"name": "taps", "datatype": dbt.type_int()}
 ] %}
 
 {{ fivetran_utils.add_pass_through_columns(columns, var('apple_search_ads__keyword_passthrough_metrics')) }}

diff --git a/macros/get_organization_columns.sql b/macros/get_organization_columns.sql
@@ -1,11 +1,11 @@
 {% macro get_organization_columns() %}
 
 {% set columns = [
-    {"name": "currency", "datatype": dbt_utils.type_string()},
-    {"name": "id", "datatype": dbt_utils.type_int()},
-    {"name": "name", "datatype": dbt_utils.type_string()},
-    {"name": "payment_model", "datatype": dbt_utils.type_string()},
-    {"name": "time_zone", "datatype": dbt_utils.type_string()}
+    {"name": "currency", "datatype": dbt.type_string()},
+    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "name", "datatype": dbt.type_string()},
+    {"name": "payment_model", "datatype": dbt.type_string()},
+    {"name": "time_zone", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

diff --git a/macros/get_search_term_report_columns.sql b/macros/get_search_term_report_columns.sql
@@ -1,27 +1,27 @@
 {% macro get_search_term_report_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_id", "datatype": dbt_utils.type_string()},
+    {"name": "_fivetran_id", "datatype": dbt.type_string()},
     {"name": "ad_group_deleted", "datatype": "boolean"},
-    {"name": "ad_group_id", "datatype": dbt_utils.type_int()},
-    {"name": "ad_group_name", "datatype": dbt_utils.type_string()},
-    {"name": "bid_amount_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "bid_amount_currency", "datatype": dbt_utils.type_string()},
-    {"name": "campaign_id", "datatype": dbt_utils.type_int()},
+    {"name": "ad_group_id", "datatype": dbt.type_int()},
+    {"name": "ad_group_name", "datatype": dbt.type_string()},
+    {"name": "bid_amount_amount", "datatype": dbt.type_numeric()},
+    {"name": "bid_amount_currency", "datatype": dbt.type_string()},
+    {"name": "campaign_id", "datatype": dbt.type_int()},
     {"name": "date", "datatype": "date"},
     {"name": "deleted", "datatype": "boolean"},
-    {"name": "impressions", "datatype": dbt_utils.type_int()},
-    {"name": "keyword", "datatype": dbt_utils.type_string()},
-    {"name": "keyword_display_status", "datatype": dbt_utils.type_string()},
-    {"name": "keyword_id", "datatype": dbt_utils.type_int()},
-    {"name": "local_spend_amount", "datatype": dbt_utils.type_numeric()},
-    {"name": "local_spend_currency", "datatype": dbt_utils.type_string()},
-    {"name": "match_type", "datatype": dbt_utils.type_string()},
-    {"name": "new_downloads", "datatype": dbt_utils.type_int()},
-    {"name": "redownloads", "datatype": dbt_utils.type_int()},
-    {"name": "search_term_source", "datatype": dbt_utils.type_string()},
-    {"name": "search_term_text", "datatype": dbt_utils.type_string()},
-    {"name": "taps", "datatype": dbt_utils.type_int()}
+    {"name": "impressions", "datatype": dbt.type_int()},
+    {"name": "keyword", "datatype": dbt.type_string()},
+    {"name": "keyword_display_status", "datatype": dbt.type_string()},
+    {"name": "keyword_id", "datatype": dbt.type_int()},
+    {"name": "local_spend_amount", "datatype": dbt.type_numeric()},
+    {"name": "local_spend_currency", "datatype": dbt.type_string()},
+    {"name": "match_type", "datatype": dbt.type_string()},
+    {"name": "new_downloads", "datatype": dbt.type_int()},
+    {"name": "redownloads", "datatype": dbt.type_int()},
+    {"name": "search_term_source", "datatype": dbt.type_string()},
+    {"name": "search_term_text", "datatype": dbt.type_string()},
+    {"name": "taps", "datatype": dbt.type_int()}
 ] %}
 
 {{ return(columns) }}

diff --git a/models/src_apple_search_ads.yml b/models/src_apple_search_ads.yml
@@ -4,7 +4,7 @@ version: 2
 sources:
   - name: apple_search_ads
     schema: "{{ var('apple_search_ads_schema', 'apple_search_ads') }}"
-    database: "{% if target.type != 'spark'%}{{ var('apple_search_ads_database', target.database) }}{% endif %}"
+    database: "{% if target.type != 'spark' %}{{ var('apple_search_ads_database', target.database) }}{% endif %}"
 
     loader: Fivetran
     loaded_at_field: _fivetran_synced