Dbt gatekeeper test/nathan lugg

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 executors:
   standard:
     docker:
-      - image: circleci/python:3.8
+      - image: circleci/python:3.10
     working_directory: "~/lib"
 
 # -----------------
@@ -33,7 +33,7 @@ commands:
             - virtualenv-{{ .Environment.CACHE_VERSION }}-
       - run:
           name: Install Python packages
-          command: pipenv install --dev --system
+          command: pipenv sync --dev --system
       - save_cache:
           key: virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
           paths:
@@ -64,7 +64,7 @@ jobs:
     executor: standard
     environment:
       SALT: himalayan
-      DBT_PROFILES_DIR: ~/lib/testing/
+      DBT_PROFILES_DIR: testing
       TESTING_PROFILE_NAME: jaffle_shop
     steps:
       - checkout

.gitignore

@@ -9,3 +9,5 @@ logs/
 # Python temp files
 *.pyc
 __pycache__
+
+*.duckdb

Makefile

@@ -1,6 +1,6 @@
 init:
-	pipenv update
-	pipenv run dbt deps
+	pipenv install
+	pipenv run dbt deps --project-dir jaffle_shop
 
 clean-env:
 	:> .env
@@ -11,21 +11,19 @@ env-development-salt:
 env-target:
 	echo DATABRICKS_TARGET=$$(git symbolic-ref --short HEAD | tr /- _) >> .env
 
-package-project:
-	for PACKAGE in dbt_project_evaluator ; do \
-		cp package_projects/$$PACKAGE.yml dbt_packages/$$PACKAGE/dbt_project.yml ; \
-	done
-
-build-env: clean-env env-development-salt env-target package-project
+build-env: clean-env env-development-salt env-target
 
 dbt-deps:
-	pipenv run dbt deps
+	pipenv run dbt deps --project-dir jaffle_shop
 
 dbt-build: build-env
-	pipenv run dbt build --selector jaffle_shop
+	pipenv run dbt build --selector jaffle_shop --project-dir jaffle_shop
 
 run-dbt-project-evaluator: dbt-deps build-env
-	pipenv run dbt --warn-error build --select package:dbt_project_evaluator dbt_project_evaluator_exceptions
+	pipenv run dbt --warn-error build \
+		--select package:dbt_project_evaluator dbt_project_evaluator_exceptions \
+		--select jaffle_shop \
+		--project-dir jaffle_shop
 
 lint: build-env
 	pipenv run sqlfluff lint
@@ -34,7 +32,7 @@ format: build-env
 	pipenv run sqlfluff fix
 
 run-python-tests:
-	pipenv run pytest --quiet --show-capture=no --tb=no
+	pipenv run pytest testing --quiet --show-capture=no --tb=no
 
 run-python-tests-detailed:
-	pipenv run pytest
+	pipenv run pytest testing

Pipfile

@@ -4,14 +4,19 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-dbt-core = "==1.3.0"
-dbt-databricks = {extras = ["odbc"]}
-pytest-mock = "*"
+dbt-core = "==1.7.17"
+dbt-duckdb = "*"
+duckdb = "*"
 glob2 = "*"
 
 [dev-packages]
+altair = "*"
+jupyterlab = "*"
+pandas = "*"
+pyarrow = "*"
 pytest = "*"
 pytest-mock = "*"
+vl-convert-python = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.10"

jaffle_shop/dbt_project.yml

@@ -0,0 +1,62 @@
+name: 'jaffle_shop'
+
+config-version: 2
+version: '0.1'
+
+profile: 'jaffle_shop'
+
+macro-paths: ["macros"]
+model-paths: ["models"]
+seed-paths: ["seeds"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['dbt_project_evaluator', 'spark_utils', 'dbt_utils']
+  # - macro_namespace: dbt
+  #   search_order: ['dbt_project_evaluator', 'dbt']
+
+models:
+  +file_format: delta
+
+  jaffle_shop:
+    +pk_required: true
+    +desc_required: true
+    +no_hardcoded_dbs: true
+
+    staging:
+      +materialized: view
+
+      src_seed:
+        +materialized: view
+
+    warehouse:
+      +materialized: table
+
+    final:
+      +materialized: table
+
+seeds:
+  dbt_project_evaluator:
+    dbt_project_evaluator_exceptions:
+      +enabled: false
+
+vars:
+  dbt_project_evaluator:
+    model_types: ['staging', 'intermediate', 'warehouse', 'final', 'metrics']
+    staging_folder_name: 'staging'
+    staging_prefixes: ['stg_']
+    intermediate_folder_name: 'intermediate'
+    intermediate_prefixes: ['int_', 'inter_']
+    warehouse_folder_name: 'warehouse'
+    warehouse_prefixes: ['wh_']
+    final_folder_name: 'final'
+    final_prefixes: ['fnl_']

jaffle_shop/macros/sensitive/hash_of_column.sql

@@ -1,13 +1,12 @@
 {% macro hash_of_column(column) %}
 
-    SHA2(
+    SHA256(
         TRIM(
             LOWER(
-                CAST({{ column|lower }} AS {{ dbt_utils.type_string() }})
+                CAST({{ column|lower }} AS {{ dbt.type_string() }})
                 || '{{ jaffle_shop.get_salt(column|lower) }}'
             )
-        ),
-        256
+        )
     ) AS {{column|lower}}_hash,
 
 {% endmacro %}

jaffle_shop/models/final/_models.yml

@@ -0,0 +1,29 @@
+version: 2
+
+models:
+  - name: fnl_sales_newcustomers
+    meta:
+      owner: '[email protected]'
+    description: A row per month with the number of customers that made their first order
+    columns:
+      - name: first_order_month
+        description: Primary key. The month we are counting the number of new customers in
+        tests:
+          - unique
+          - not_null
+      - name: num_new_customers
+        description: |
+          The number of new customers in the repsective month, where a new customer is
+          defined as a customer that made an order for the first time
+  - name: fnl_finance_returnsamout
+    meta:
+      owner: '[email protected]'
+    description: A row per customer (ID) and the amount (in AUD) of their returned orders
+    columns:
+      - name: customer_id
+        description: Primary key. The customer ID
+        tests:
+          - unique
+          - not_null
+      - name: total_returned_amount_aud
+        description: The total amount (in AUD) that the customer has returned

jaffle_shop/models/final/fnl_finance_returnsamout.sql

@@ -0,0 +1,16 @@
+WITH returned_ AS (
+  SELECT
+    customer_id
+    , SUM(total_amount_aud) AS total_returned_amount_aud
+  FROM {{ ref('wh_orders') }}
+  WHERE status = 'returned'
+  GROUP BY customer_id
+)
+
+
+SELECT
+  customers_.customer_id
+  , COALESCE(returned_.total_returned_amount_aud, 0) AS total_returned_amount_aud
+FROM {{ ref('wh_customers') }} AS customers_
+LEFT JOIN returned_
+  ON customers_.customer_id = returned_.customer_id

jaffle_shop/models/final/fnl_sales_newcustomers.sql

@@ -0,0 +1,14 @@
+WITH customers_ AS (
+  SELECT
+    customer_id
+    , DATE_TRUNC('MONTH', first_order) AS first_order_month
+  FROM {{ ref('wh_customers') }}
+  WHERE first_order IS NOT NULL
+)
+
+
+SELECT
+  first_order_month
+  , COUNT(*) AS num_new_customers
+FROM customers_
+GROUP BY first_order_month