Merge pull request #12 from fivetran/update/v2-updates

update/v2-updates
fivetran · Sep 7, 2022 · fd933a5 · fd933a5
2 parents d0daa47 + 4e04d6d
commit fd933a5
Show file tree

Hide file tree

Showing 32 changed files with 883 additions and 671 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -31,7 +31,7 @@ jobs:
             dbt deps
             dbt seed --target spark --full-refresh
             dbt run --target spark --full-refresh
-            dbt run --vars '{apple_search_ads__using_search_terms: false}' --target spark
+            dbt run --vars '{apple_search_ads__using_search_terms: true}' --target spark
             dbt test --target spark
       - run:
           name: "Run Tests - Redshift"
@@ -42,19 +42,8 @@ jobs:
             dbt deps
             dbt seed --target redshift --full-refresh
             dbt run --target redshift --full-refresh
-            dbt run --vars '{apple_search_ads__using_search_terms: false}' --target redshift
+            dbt run --vars '{apple_search_ads__using_search_terms: true}' --target redshift
             dbt test --target redshift
-      - run:
-          name: "Run Tests - Postgres"
-          command: |
-            . venv/bin/activate
-            echo `pwd`
-            cd integration_tests
-            dbt deps
-            dbt seed --target postgres --full-refresh
-            dbt run --target postgres --full-refresh
-            dbt run --vars '{apple_search_ads__using_search_terms: false}' --target postgres --full-refresh
-            dbt test --target postgres
       - run:
           name: "Run Tests - Snowflake"
           command: |
@@ -64,7 +53,7 @@ jobs:
             dbt deps
             dbt seed --target snowflake --full-refresh
             dbt run --target snowflake --full-refresh
-            dbt run --vars '{apple_search_ads__using_search_terms: false}' --target snowflake --full-refresh
+            dbt run --vars '{apple_search_ads__using_search_terms: true}' --target snowflake --full-refresh
             dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
@@ -78,5 +67,16 @@ jobs:
             dbt deps
             dbt seed --target bigquery --full-refresh
             dbt run --target bigquery --full-refresh
-            dbt run --vars '{apple_search_ads__using_search_terms: false}' --target bigquery --full-refresh
-            dbt test --target bigquery
+            dbt run --vars '{apple_search_ads__using_search_terms: true}' --target bigquery --full-refresh
+            dbt test --target bigquery
+      - run:
+          name: "Run Tests - Postgres"
+          command: |
+            . venv/bin/activate
+            echo `pwd`
+            cd integration_tests
+            dbt deps
+            dbt seed --target postgres --full-refresh
+            dbt run --target postgres --full-refresh
+            dbt run --vars '{apple_search_ads__using_search_terms: true}' --target postgres --full-refresh
+            dbt test --target postgres
diff --git a/README.md b/README.md
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -10,16 +10,19 @@ vars:
     campaign_report: "{{ ref('stg_apple_search_ads__campaign_report') }}"
     ad_group_history: "{{ ref('stg_apple_search_ads__ad_group_history') }}"
     ad_group_report: "{{ ref('stg_apple_search_ads__ad_group_report') }}"
+    ad_history: "{{ ref('stg_apple_search_ads__ad_history') }}"
+    ad_report: "{{ ref('stg_apple_search_ads__ad_report') }}"
     keyword_history: "{{ ref('stg_apple_search_ads__keyword_history') }}"
     keyword_report: "{{ ref('stg_apple_search_ads__keyword_report') }}"
     search_term_report: "{{ ref('stg_apple_search_ads__search_term_report') }}"
+
+  apple_search_ads__ad_group_passthrough_metrics: []
+  apple_search_ads__ad_passthrough_metrics: []
+  apple_search_ads__campaign_passthrough_metrics: []
+  apple_search_ads__keyword_passthrough_metrics: [] 
+  apple_search_ads__search_term_passthrough_metrics: []
 
-    apple_search_ads__using_search_terms: true
-    apple_search_ads__adapter: false ## placeholder for ad_reporting package
-
 models:
   apple_search_ads:
-    materialized: table
     +schema: apple_search_ads
-    intermediate:
-      +materialized: ephemeral
+    +materialized: table
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -6,42 +6,29 @@ version: '0.1.0'
 profile: 'integration_tests' 
 
 vars:
-  apple_search_ads_source:
-    apple_search_ads__using_search_terms: true
-    apple_search_ads__adapter: false ## ad reporting ad adapter
-    organization: "{{ ref('organization_data') }}"
-    campaign_history: "{{ ref('campaign_history_data') }}"
-    campaign_report: "{{ ref('campaign_report_data') }}"
-    ad_group_history: "{{ ref('ad_group_history_data') }}"
-    ad_group_report: "{{ ref('ad_group_report_data') }}"
-    keyword_history: "{{ ref('keyword_history_data') }}"
-    keyword_report: "{{ ref('keyword_report_data') }}"
-    search_term_report: "{{ ref('search_term_report_data') }}"
-    apple_search_ads_schema: apple_search_ads_source_integration_tests
+  apple_search_ads_schema: apple_search_ads_integration_tests
+  apple_search_ads_organization_identifier: "organization_data"
+  apple_search_ads_campaign_history_identifier: "campaign_history_data"
+  apple_search_ads_campaign_report_identifier: "campaign_report_data"
+  apple_search_ads_ad_group_history_identifier: "ad_group_history_data"
+  apple_search_ads_ad_group_report_identifier: "ad_group_report_data"
+  apple_search_ads_ad_history_identifier: "ad_history_data"
+  apple_search_ads_ad_report_identifier: "ad_report_data"
+  apple_search_ads_keyword_history_identifier: "keyword_history_data"
+  apple_search_ads_keyword_report_identifier: "keyword_report_data"
+  apple_search_ads_search_term_report_identifier: "search_term_report_data"
+  apple_search_ads__using_search_terms: True
 
 seeds:
   +quote_columns: "{{ true if target.type == 'redshift' else false }}"
   apple_search_ads_integration_tests:
     +column_types:
-      _fivetran_synced: timestamp
-    ad_group_history_data: 
-      +column_types:
-        modification_time: timestamp
-        start_time: timestamp
-        end_time: timestamp
-    campaign_history_data:
-      +column_types:
-        modification_time: timestamp
-        start_time: timestamp
-        end_time: timestamp
-    keyword_history_data:
-      +column_types:
-        modification_time: timestamp
-    creative_set_report_data:
-      +column_types:
-        modification_time: timestamp
-        creation_time: timestamp
-
+      modification_time: "timestamp"
+      start_time: "timestamp"
+      end_time: "timestamp"
+      creation_time: "timestamp"
+      _fivetran_synced: "timestamp"
+
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml
@@ -1,2 +1,2 @@
 packages:
-  - local: ../
+ - local: ../
diff --git a/integration_tests/seeds/ad_group_history_data.csv b/integration_tests/seeds/ad_group_history_data.csv
@@ -1,5 +1,5 @@
-id,modification_time,_fivetran_synced,automated_keywords_opt_in,campaign_id,cpa_goal_amount,cpa_goal_currency,default_cpc_bid_amount,default_cpc_bid_currency,deleted,end_time,name,organization_id,serving_state_reasons,serving_status,start_time,status,storefronts
-976005125,2021-12-15T20:26:38.940Z,2021-12-29T20:21:04.037Z,true,973923054,,,,,false,2022-01-31T23:59:00Z,Kitty Cat Tap,3219110,,RUNNING,2021-12-09T00:00:00Z,ENABLED,
-983960442,2021-12-15T20:27:55.112Z,2021-12-29T20:21:04.451Z,true,973923054,,,,,false,2022-01-14T23:59:59.999Z,Bored Ad Group,3219110,,RUNNING,2021-12-14T00:00:00Z,ENABLED,
-976005125,2021-12-09T23:39:28.469Z,2021-12-15T20:22:57.207Z,true,973923054,,,,,false,2022-01-31T23:59:00Z,Kitty Cat Tap,3219110,,RUNNING,2021-12-09T00:00:00Z,ENABLED,
-983960442,2021-12-14T22:20:08.559Z,2021-12-15T20:22:57.779Z,true,973923054,,,,,false,2022-01-14T23:59:59.999Z,Bored Ad Group,3219110,,RUNNING,2021-12-14T00:00:00Z,ENABLED,
+id,modification_time,automated_keywords_opt_in,campaign_id,cpa_goal_amount,cpa_goal_currency,default_cpc_bid_amount,default_cpc_bid_currency,deleted,end_time,name,organization_id,serving_state_reasons,serving_status,start_time,status,storefronts
+976005125,2021-12-15T20:26:38.940Z,true,973923054,,,,,false,2022-01-31T23:59:00Z,Kitty Cat Tap,3219110,,RUNNING,2021-12-09T00:00:00Z,ENABLED,
+983960442,2021-12-15T20:27:55.112Z,true,973923054,,,,,false,2022-01-14T23:59:59.999Z,Bored Ad Group,3219110,,RUNNING,2021-12-14T00:00:00Z,ENABLED,
+976005125,2021-12-09T23:39:28.469Z,true,973923054,,,,,false,2022-01-31T23:59:00Z,Kitty Cat Tap,3219110,,RUNNING,2021-12-09T00:00:00Z,ENABLED,
+983960442,2021-12-14T22:20:08.559Z,true,973923054,,,,,false,2022-01-14T23:59:59.999Z,Bored Ad Group,3219110,,RUNNING,2021-12-14T00:00:00Z,ENABLED,
diff --git a/integration_tests/seeds/ad_group_report_data.csv b/integration_tests/seeds/ad_group_report_data.csv
@@ -1,14 +1,14 @@
-ad_group_id,date,_fivetran_synced,avg_cpa_amount,avg_cpa_currency,avg_cpt_amount,avg_cpt_currency,conversion_rate,conversions,impressions,lat_off_installs,lat_on_installs,local_spend_amount,local_spend_currency,new_downloads,redownloads,tap_through_rate,taps
-976005125,2021-12-20,2021-12-26T02:21:06.519Z,0,USD,0.1067,USD,0,0,63,0,0,0.32,USD,0,0,0.047600001096725464,3
-976005125,2021-12-17,2021-12-23T02:21:34.588Z,0,USD,1.018,USD,0,0,100,0,0,5.09,USD,0,0,0.05000000074505806,5
-983960442,2021-12-19,2021-12-25T02:21:29.472Z,0,USD,0,USD,0,0,2,0,0,0,USD,0,0,0.0,0
-976005125,2021-12-12,2021-12-18T02:22:34.240Z,2.78,USD,0.7943,USD,0.2857,2,249,0,0,5.56,USD,2,0,0.028100000694394112,7
-976005125,2021-12-09,2021-12-15T02:25:38.632Z,25.08,USD,0.809,USD,0.0323,1,1605,0,0,25.08,USD,1,0,0.019300000742077827,31
-976005125,2021-12-13,2021-12-19T02:24:17.782Z,5.26,USD,0.7514,USD,0.1429,1,492,0,0,5.26,USD,1,0,0.0142000000923872,7
-976005125,2021-12-11,2021-12-17T02:29:43.837Z,5.71,USD,1.142,USD,0.2,1,290,0,0,5.71,USD,1,0,0.017200000584125519,5
-976005125,2021-12-16,2021-12-22T02:21:47.476Z,5.07,USD,1.014,USD,0.2,1,318,0,0,5.07,USD,1,0,0.015699999406933784,5
-976005125,2021-12-14,2021-12-20T02:20:50.162Z,1.2575,USD,0.4573,USD,0.3636,4,350,0,0,5.03,USD,4,0,0.031399998813867569,11
-976005125,2021-12-19,2021-12-25T02:21:29.472Z,1.4875,USD,0.6611,USD,0.4444,4,424,0,0,5.95,USD,4,0,0.021199999377131462,9
-976005125,2021-12-18,2021-12-24T02:20:51.698Z,2.5,USD,0.5556,USD,0.2222,2,403,0,0,5,USD,2,0,0.022299999371170998,9
-976005125,2021-12-10,2021-12-16T02:22:32.969Z,4.3133,USD,0.9585,USD,0.2222,6,1494,1,0,25.88,USD,6,0,0.01810000091791153,27
-976005125,2021-12-15,2021-12-21T02:21:14.031Z,2.0167,USD,0.605,USD,0.3,3,546,0,1,6.05,USD,3,0,0.018300000578165054,10
+ad_group_id,date,avg_cpa_amount,avg_cpa_currency,avg_cpt_amount,avg_cpt_currency,conversion_rate,conversions,impressions,lat_off_installs,lat_on_installs,local_spend_amount,local_spend_currency,new_downloads,redownloads,tap_through_rate,taps
+976005125,2021-12-20,0,USD,0.1067,USD,0,0,63,0,0,0.32,USD,0,0,0.047600001096725464,3
+976005125,2021-12-17,0,USD,1.018,USD,0,0,100,0,0,5.09,USD,0,0,0.05000000074505806,5
+983960442,2021-12-19,0,USD,0,USD,0,0,2,0,0,0,USD,0,0,0.0,0
+976005125,2021-12-12,2.78,USD,0.7943,USD,0.2857,2,249,0,0,5.56,USD,2,0,0.028100000694394112,7
+976005125,2021-12-09,25.08,USD,0.809,USD,0.0323,1,1605,0,0,25.08,USD,1,0,0.019300000742077827,31
+976005125,2021-12-13,5.26,USD,0.7514,USD,0.1429,1,492,0,0,5.26,USD,1,0,0.0142000000923872,7
+976005125,2021-12-11,5.71,USD,1.142,USD,0.2,1,290,0,0,5.71,USD,1,0,0.017200000584125519,5
+976005125,2021-12-16,5.07,USD,1.014,USD,0.2,1,318,0,0,5.07,USD,1,0,0.015699999406933784,5
+976005125,2021-12-14,1.2575,USD,0.4573,USD,0.3636,4,350,0,0,5.03,USD,4,0,0.031399998813867569,11
+976005125,2021-12-19,1.4875,USD,0.6611,USD,0.4444,4,424,0,0,5.95,USD,4,0,0.021199999377131462,9
+976005125,2021-12-18,2.5,USD,0.5556,USD,0.2222,2,403,0,0,5,USD,2,0,0.022299999371170998,9
+976005125,2021-12-10,4.3133,USD,0.9585,USD,0.2222,6,1494,1,0,25.88,USD,6,0,0.01810000091791153,27
+976005125,2021-12-15,2.0167,USD,0.605,USD,0.3,3,546,0,1,6.05,USD,3,0,0.018300000578165054,10
diff --git a/integration_tests/seeds/ad_history_data.csv b/integration_tests/seeds/ad_history_data.csv
@@ -0,0 +1,2 @@
+creation_time,modification_time,org_id,campaign_id,ad_group_id,name,id,creative_id,creative_type,status
+2022-07-12T20:44:53.419Z,2022-07-12T20:44:53.419Z,3219110,1088388037,1088985837,Cat Ad Custom,1088718190,2139136,CUSTOM_PRODUCT_PAGE,ENABLED
diff --git a/integration_tests/seeds/ad_report_data.csv b/integration_tests/seeds/ad_report_data.csv
@@ -0,0 +1,3 @@
+date,_fivetran_synced,campaign_id,ad_group_id,ad_id,impressions,local_spend_amount,local_spend_currency,new_downloads,redownloads,taps
+2022-07-14,2022-07-20T05:17:57.811Z,1088388037,1088985837,1088718190,552,4.31,USD,2,0,6
+2022-07-13,2022-07-19T05:16:03.580Z,1088388037,1088985837,1088718190,300,1.85,USD,1,0,2
diff --git a/integration_tests/seeds/campaign_history_data.csv b/integration_tests/seeds/campaign_history_data.csv
@@ -1,4 +1,4 @@
-id,modification_time,_fivetran_synced,adam_id,budget_amount,budget_currency,budget_orders,daily_budget_amount,daily_budget_currency,deleted,end_time,loc_invoice_detail_buyer_email,loc_invoice_detail_buyer_name,loc_invoice_detail_client_name,loc_invoice_detail_order_number,name,organization_id,payment_model,serving_state_reasons,serving_status,start_time,status
-973923054,2021-12-10T19:17:21.119Z,2021-12-20T02:20:48.583Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,,RUNNING,2021-12-09T00:00:00Z,ENABLED
-973923054,2021-12-20T04:00:51.264Z,2021-12-26T20:21:01.489Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,"[""TOTAL_BUDGET_EXHAUSTED""]",NOT_RUNNING,2021-12-09T00:00:00Z,ENABLED
-973923054,2021-12-27T00:33:15.302Z,2021-12-29T20:21:03.853Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,"[""TOTAL_BUDGET_EXHAUSTED"",""PAUSED_BY_USER""]",NOT_RUNNING,2021-12-09T00:00:00Z,PAUSED
+id,modification_time,adam_id,budget_amount,budget_currency,budget_orders,daily_budget_amount,daily_budget_currency,deleted,end_time,loc_invoice_detail_buyer_email,loc_invoice_detail_buyer_name,loc_invoice_detail_client_name,loc_invoice_detail_order_number,name,organization_id,payment_model,serving_state_reasons,serving_status,start_time,status
+973923054,2021-12-10T19:17:21.119Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,,RUNNING,2021-12-09T00:00:00Z,ENABLED
+973923054,2021-12-20T04:00:51.264Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,"[""TOTAL_BUDGET_EXHAUSTED""]",NOT_RUNNING,2021-12-09T00:00:00Z,ENABLED
+973923054,2021-12-27T00:33:15.302Z,1589007550,100,USD,[],5,USD,false,2022-01-31T23:59:00Z,,,,,Tap Away,3219110,PAYG,"[""TOTAL_BUDGET_EXHAUSTED"",""PAUSED_BY_USER""]",NOT_RUNNING,2021-12-09T00:00:00Z,PAUSED
diff --git a/integration_tests/seeds/campaign_report_data.csv b/integration_tests/seeds/campaign_report_data.csv
@@ -1,13 +1,13 @@
-date,id,_fivetran_synced,avg_cpa_amount,avg_cpa_currency,avg_cpt_amount,avg_cpt_currency,conversion_rate,conversions,impressions,lat_off_installs,lat_on_installs,local_spend_amount,local_spend_currency,new_downloads,redownloads,tap_through_rate,taps
-2021-12-17,973923054,2021-12-23T02:21:34.555Z,0,USD,1.018,USD,0,0,100,0,0,5.09,USD,0,0,0.05000000074505806,5
-2021-12-20,973923054,2021-12-26T02:21:06.320Z,0,USD,0.1067,USD,0,0,63,0,0,0.32,USD,0,0,0.047600001096725464,3
-2021-12-12,973923054,2021-12-18T02:22:34.237Z,2.78,USD,0.7943,USD,0.2857,2,249,0,0,5.56,USD,2,0,0.028100000694394112,7
-2021-12-09,973923054,2021-12-15T02:25:38.759Z,25.08,USD,0.809,USD,0.0323,1,1605,0,0,25.08,USD,1,0,0.019300000742077827,31
-2021-12-13,973923054,2021-12-19T02:24:17.898Z,5.26,USD,0.7514,USD,0.1429,1,492,0,0,5.26,USD,1,0,0.0142000000923872,7
-2021-12-11,973923054,2021-12-17T02:29:43.841Z,5.71,USD,1.142,USD,0.2,1,290,0,0,5.71,USD,1,0,0.017200000584125519,5
-2021-12-16,973923054,2021-12-22T02:21:47.291Z,5.07,USD,1.014,USD,0.2,1,318,0,0,5.07,USD,1,0,0.015699999406933784,5
-2021-12-14,973923054,2021-12-20T02:20:50.169Z,1.2575,USD,0.4573,USD,0.3636,4,350,0,0,5.03,USD,4,0,0.031399998813867569,11
-2021-12-19,973923054,2021-12-25T02:21:28.955Z,1.4875,USD,0.6611,USD,0.4444,4,426,0,0,5.95,USD,4,0,0.0210999995470047,9
-2021-12-18,973923054,2021-12-24T02:20:51.681Z,2.5,USD,0.5556,USD,0.2222,2,403,0,0,5,USD,2,0,0.022299999371170998,9
-2021-12-10,973923054,2021-12-16T02:22:32.784Z,4.3133,USD,0.9585,USD,0.2222,6,1494,1,0,25.88,USD,6,0,0.01810000091791153,27
-2021-12-15,973923054,2021-12-21T02:21:14.049Z,2.0167,USD,0.605,USD,0.3,3,546,0,1,6.05,USD,3,0,0.018300000578165054,10
+date,id,avg_cpa_amount,avg_cpa_currency,avg_cpt_amount,avg_cpt_currency,conversion_rate,conversions,impressions,lat_off_installs,lat_on_installs,local_spend_amount,local_spend_currency,new_downloads,redownloads,tap_through_rate,taps
+2021-12-17,973923054,0,USD,1.018,USD,0,0,100,0,0,5.09,USD,0,0,0.05000000074505806,5
+2021-12-20,973923054,0,USD,0.1067,USD,0,0,63,0,0,0.32,USD,0,0,0.047600001096725464,3
+2021-12-12,973923054,2.78,USD,0.7943,USD,0.2857,2,249,0,0,5.56,USD,2,0,0.028100000694394112,7
+2021-12-09,973923054,25.08,USD,0.809,USD,0.0323,1,1605,0,0,25.08,USD,1,0,0.019300000742077827,31
+2021-12-13,973923054,5.26,USD,0.7514,USD,0.1429,1,492,0,0,5.26,USD,1,0,0.0142000000923872,7
+2021-12-11,973923054,5.71,USD,1.142,USD,0.2,1,290,0,0,5.71,USD,1,0,0.017200000584125519,5
+2021-12-16,973923054,5.07,USD,1.014,USD,0.2,1,318,0,0,5.07,USD,1,0,0.015699999406933784,5
+2021-12-14,973923054,1.2575,USD,0.4573,USD,0.3636,4,350,0,0,5.03,USD,4,0,0.031399998813867569,11
+2021-12-19,973923054,1.4875,USD,0.6611,USD,0.4444,4,426,0,0,5.95,USD,4,0,0.0210999995470047,9
+2021-12-18,973923054,2.5,USD,0.5556,USD,0.2222,2,403,0,0,5,USD,2,0,0.022299999371170998,9
+2021-12-10,973923054,4.3133,USD,0.9585,USD,0.2222,6,1494,1,0,25.88,USD,6,0,0.01810000091791153,27
+2021-12-15,973923054,2.0167,USD,0.605,USD,0.3,3,546,0,1,6.05,USD,3,0,0.018300000578165054,10