Merge pull request #36 from fivetran/bugfix/updating-vars-defaults

updating default vars

.buildkite/hooks/pre-command

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Export secrets for Docker containers.
+# Restrict exposing secrets only to the steps that need them
+export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
+export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -0,0 +1,73 @@
+steps:
+  - label: ":postgres: Run Tests - Postgres"
+    key: "run-dbt-postgres"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_POSTGRES_DBT_DBNAME"
+            - "CI_POSTGRES_DBT_HOST"
+            - "CI_POSTGRES_DBT_PASS"
+            - "CI_POSTGRES_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh postgres 
+
+  - label: ":snowflake-db: Run Tests - Snowflake"
+    key: "run_dbt_snowflake"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_SNOWFLAKE_DBT_ACCOUNT"
+            - "CI_SNOWFLAKE_DBT_DATABASE"
+            - "CI_SNOWFLAKE_DBT_PASS"
+            - "CI_SNOWFLAKE_DBT_ROLE"
+            - "CI_SNOWFLAKE_DBT_USER"
+            - "CI_SNOWFLAKE_DBT_WAREHOUSE"
+    commands: |
+      bash .buildkite/scripts/run_models.sh snowflake
+
+  - label: ":gcloud: Run Tests - BigQuery"
+    key: "run_dbt_bigquery"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "GCLOUD_SERVICE_KEY"
+    commands: |
+      bash .buildkite/scripts/run_models.sh bigquery
+
+  - label: ":amazon-redshift: Run Tests - Redshift"
+    key: "run_dbt_redshift"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_REDSHIFT_DBT_DBNAME"
+            - "CI_REDSHIFT_DBT_HOST"
+            - "CI_REDSHIFT_DBT_PASS"
+            - "CI_REDSHIFT_DBT_USER"
+    commands: |
+      bash .buildkite/scripts/run_models.sh redshift
+
+  - label: ":bricks: Run Tests - Databricks"
+    key: "run_dbt_databricks"
+    plugins:
+      - docker#v3.13.0:
+          image: "python:3.8"
+          shell: [ "/bin/bash", "-e", "-c" ]
+          environment:
+            - "BASH_ENV=/tmp/.bashrc"
+            - "CI_DATABRICKS_DBT_HOST"
+            - "CI_DATABRICKS_DBT_HTTP_PATH"
+            - "CI_DATABRICKS_DBT_TOKEN"
+    commands: |
+      bash .buildkite/scripts/run_models.sh databricks

.buildkite/scripts/run_models.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -euo pipefail
+
+apt-get update
+apt-get install libsasl2-dev
+
+python3 -m venv venv
+. venv/bin/activate
+pip install --upgrade pip setuptools
+pip install -r integration_tests/requirements.txt
+mkdir -p ~/.dbt
+cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml
+
+db=$1
+echo `pwd`
+cd integration_tests
+dbt deps
+dbt seed --target "$db" --full-refresh
+dbt run --target "$db" --full-refresh
+dbt test --target "$db"
+## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE
+dbt run --vars '{apple_search_ads__using_search_terms: False}' --target "$db" --full-refresh
+dbt test --target "$db"
+### END VARS CHUNK, REMOVE IF NOT USING

.github/pull_request_template.md

@@ -29,9 +29,9 @@ Pull Request
 
 **How did you test the PR changes?** 
 <!--- Proof of testing is required in order for the PR to be approved. -->
-<!--- To check a box, remove the space and insert an x in the box (eg. [x] CircleCi). --> 
+<!--- To check a box, remove the space and insert an x in the box (eg. [x] Buildkite). --> 
 <!--- To select a checkbox you simply need to add an "x" with no spaces between the brackets (eg. [x] Yes). -->
-- [ ] CircleCi <!--- CircleCi testing is only applicable to Fivetran employees. --> 
+- [ ] Buildkite <!--- Buildkite testing is only applicable to Fivetran employees. --> 
 - [ ] Local (please provide additional testing details below)
 
 **Select which warehouse(s) were used to test the PR**

CHANGELOG.md

@@ -1,3 +1,9 @@
+# dbt_apple_search_ads_source v0.1.3
+
+## 🕷️ Bugfixes 🕷️
+- Updated the default value for `apple_search_ads__using_search_terms` to `true` to be consistent across Ad Reporting and the rest of this package in the `src_apple_search_ads.yml` for the `search_term_report`. [#36](https://github.com/fivetran/dbt_apple_search_ads_source/pull/36)
+- Updated `README` to show package dependency matrix. [#36](https://github.com/fivetran/dbt_apple_search_ads_source/pull/36)
+
 # dbt_apple_search_ads_source v0.1.2
 
 This [PR #21](https://github.com/fivetran/dbt_apple_search_ads_source/pull/21) updates macros for history tables to only include columns that are referenced downstream.

README.md

@@ -114,7 +114,9 @@ vars:
 <details><summary>Expand for more details</summary>
 
 Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core™ setup guides](https://fivetran.com/docs/transformations/dbt#setupguide).
-
+
+</details>
+
 # 🔍 Does this package have dependencies?
 This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site.
 > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts.
@@ -127,8 +129,6 @@ packages:
       version: [">=0.8.0", "<0.9.0"]
 ```
 
-</details>
-
 # 🙌 How is this package maintained and can I contribute?
 ## Package Maintenance
 The Fivetran team maintaining this package _only_ maintains the latest version of the package. We highly recommend that you stay consistent with the [latest version](https://hub.getdbt.com/fivetran/apple_search_ads_source/latest/) of the package and refer to the [CHANGELOG](https://github.com/fivetran/dbt_apple_search_ads_source/blob/main/CHANGELOG.md) and release notes for more information on changes across versions.

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'apple_search_ads_source'
-version: '0.1.2'
+version: '0.1.3'
 config-version: 2
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 

integration_tests/ci/sample.profiles.yml

@@ -1,12 +1,13 @@
-# HEY! This file is used in the Apple Search Ads integrations tests with CircleCI.
+
+# HEY! This file is used in the dbt package integrations tests with Buildkite.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:
     send_anonymous_usage_stats: False
     use_colors: True
 
 integration_tests:
-  target: snowflake
+  target: redshift
   outputs:
     redshift:
       type: redshift
@@ -19,11 +20,11 @@ integration_tests:
       threads: 8
     bigquery:
       type: bigquery
-      method: service-account
-      keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}"
+      method: service-account-json
       project: 'dbt-package-testing'
       schema: apple_ads_source_integration_tests
       threads: 8
+      keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
       type: snowflake
       account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}"
@@ -34,26 +35,20 @@ integration_tests:
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
       schema: apple_ads_source_integration_tests
       threads: 8
-    spark:
-      type: spark
-      method: http
-      schema: apple_ads_source_integration_tests
-      host: "{{ env_var('CI_SPARK_DBT_HOST') }}"
-      organization: "{{ env_var('CI_SPARK_DBT_ORGANIZATION') }}"
-      token: "{{ env_var('CI_SPARK_DBT_TOKEN') }}"
-      cluster: "{{ env_var('CI_SPARK_DBT_CLUSTER') }}"
-      port: 443
-      connect_timeout: 60
-      connect_retries: 5
-      threads: 4
     postgres:
       type: postgres
       host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}"
       user: "{{ env_var('CI_POSTGRES_DBT_USER') }}"
-      password: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
+      dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      dbname: "{{ env_var('CI_POSTGRES_DBT_DATABASE') }}"
       schema: apple_ads_source_integration_tests
       threads: 8
-      keepalives_idle: 0
-      sslmode: prefer
+    databricks:
+      catalog: null
+      host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
+      http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
+      schema: apple_ads_source_integration_tests
+      threads: 2
+      token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
+      type: databricks

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'apple_search_ads_source_integration_tests'
-version: '0.1.2'
+version: '0.1.3'
 
 profile: 'integration_tests' 
 

integration_tests/requirements.txt

@@ -1,6 +1,7 @@
-dbt-snowflake>=1.0.0
-dbt-bigquery>=1.0.0
-dbt-redshift>=1.0.0
-dbt-postgres>=1.0.0
-dbt-spark>=1.0.0
-dbt-spark[PyHive]>=1.0.0
+dbt-snowflake>=1.0.0,<1.3.0
+dbt-bigquery>=1.0.0,<1.3.0
+dbt-redshift>=1.0.0,<1.3.0
+dbt-postgres>=1.0.0,<1.3.0
+dbt-spark>=1.0.0,<1.3.0
+dbt-spark[PyHive]>=1.0.0,<1.3.0
+dbt-databricks>=1.0.0,<1.3.0

models/src_apple_search_ads.yml

@@ -207,7 +207,7 @@ sources:
       - name: search_term_report
         description: Each record represents the daily performance of a search term in Apple Search Ads.
         config:
-          enabled: "{{ var('ad_reporting__apple_search_ads_enabled', true) }} and {{ var('apple_search_ads__using_search_terms', false) }}"
+          enabled: "{{ var('ad_reporting__apple_search_ads_enabled', true) }} and {{ var('apple_search_ads__using_search_terms', true) }}"
         identifier: "{{ var('apple_search_ads_search_term_report_identifier', 'search_term_report') }}"
         columns:
           - name: _fivetran_id