Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
docs/profiles/*

# dbt_packages is a directory that gets created when you run dbt deps
dbt_packages/
dev/dags/dbt/jaffle_shop/dbt_packages/

# Byte-compiled / optimized / DLL files
__pycache__/
Expand Down
1 change: 1 addition & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ repos:
apache-airflow,
]
files: ^cosmos
exclude: "dev/dags/dbt/simple/dbt_packages/.*"

ci:
autofix_commit_msg: 🎨 [pre-commit.ci] Auto format from pre-commit.com hooks
Expand Down
181 changes: 181 additions & 0 deletions dev/dags/dbt/simple/dbt_packages/dbt_date/.circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
version: 2.1

jobs:
integration-tests-core:
docker:
- image: cimg/python:3.9.9
- image: cimg/postgres:14.0

resource_class: small

environment:
DBT_PROFILES_DIR: ./integration_tests/ci
DBT_PROJECT_DIR: ./integration_tests
BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
DBT_VERSION: 1.8.*

steps:
- checkout
- run: &pip-install-core
name: Install core Python packages & dbt-core
command: |
python3 -m venv venv
. venv/bin/activate
pip install -U pip setuptools wheel
pip install "dbt-core==$DBT_VERSION"

- run:
name: Install dbt adapter packages
command: |
python3 -m venv venv
. venv/bin/activate
pip install "dbt-postgres==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" "dbt-snowflake==$DBT_VERSION"
pip install "dbt-duckdb==$DBT_VERSION"

- run: &dbt-deps
name: Install dbt dependencies
command: |
. venv/bin/activate
dbt deps --project-dir $DBT_PROJECT_DIR
Comment thread
tatiana marked this conversation as resolved.

- run:
name: "Run Tests - Postgres"
environment:
POSTGRES_HOST: localhost
POSTGRES_TEST_USER: postgres
POSTGRES_TEST_PASSWORD: ""
POSTGRES_TEST_PORT: 5432
POSTGRES_TEST_DATABASE: circle_test
POSTGRES_TEST_SCHEMA: dbt_date_integration_tests
command: |
. venv/bin/activate
dbt build -t postgres --project-dir $DBT_PROJECT_DIR

- run:
name: "Set up GCP credentials"
command: |
echo "Writing to $BIGQUERY_SERVICE_KEY_PATH"
echo $BIGQUERY_SERVICE_KEY > $BIGQUERY_SERVICE_KEY_PATH
FILESIZE=$(stat -c%s "$BIGQUERY_SERVICE_KEY_PATH")
echo "Size of $BIGQUERY_SERVICE_KEY_PATH = $FILESIZE bytes."
echo "BIGQUERY_TEST_DATABASE = $BIGQUERY_TEST_DATABASE"

- run:
name: "Run Tests - BigQuery"
command: |
. venv/bin/activate
dbt build -t bigquery --project-dir $DBT_PROJECT_DIR

- run:
name: "Run Tests - Snowflake"
command: |
. venv/bin/activate
dbt build -t snowflake --project-dir $DBT_PROJECT_DIR

- run:
name: "Run Tests - DuckDB"
command: |
. venv/bin/activate
dbt build -t duckdb --project-dir $DBT_PROJECT_DIR

- store_artifacts:
path: ./logs

integration-tests-spark-thrift:
docker:
- image: cimg/python:3.9.9
- image: godatadriven/spark:3.1.1
environment:
WAIT_FOR: localhost:5432
command: >
--class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2
--name Thrift JDBC/ODBC Server
- image: postgres:9.6.17-alpine
environment:
POSTGRES_USER: dbt
POSTGRES_PASSWORD: dbt
POSTGRES_DB: metastore

resource_class: small

environment:
DBT_PROFILES_DIR: ./integration_tests/ci
DBT_PROJECT_DIR: ./integration_tests
DBT_VERSION: 1.8.*

steps:
- checkout
- run:
name: Install Ubuntu packages
command: |
sudo apt-get update
sudo apt-get install libsasl2-dev libsasl2-2
- run: *pip-install-core
- run:
name: Install dbt adapter packages
command: |
python3 -m venv venv
. venv/bin/activate
pip install "dbt-spark==$DBT_VERSION" "dbt-spark[PyHive]==$DBT_VERSION"
- run: *dbt-deps
- run:
name: Wait for Spark-Thrift
command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s
- run:
name: "Run Tests - Spark"
command: |
. venv/bin/activate
dbt build -t spark --project-dir $DBT_PROJECT_DIR

- store_artifacts:
path: ./logs

integration-tests-trino:
docker:
- image: cimg/python:3.11
- image: trinodb/trino:431

resource_class: small

environment:
DBT_PROFILES_DIR: ./integration_tests/ci
DBT_PROJECT_DIR: ./integration_tests
DBT_VERSION: 1.8.*

steps:
- checkout
- run: *pip-install-core
- run:
name: Install dbt adapter packages
command: |
python3 -m venv venv
. venv/bin/activate
pip install "dbt-trino==$DBT_VERSION"
- run: *dbt-deps
- setup_remote_docker
- run:
name: Run Trino server
command: |
docker run --name trino -p 8080:8080 -d -v `pwd`/integration_tests/docker/trino/catalog:/etc/trino/catalog trinodb/trino:431
timeout 5m bash -c -- 'while ! docker logs trino 2>&1 | tail -n 1 | grep "SERVER STARTED"; do sleep 2; done'
- run:
name: "Run Tests - Trino"
command: |
. venv/bin/activate
dbt build -t trino --project-dir $DBT_PROJECT_DIR

workflows:
version: 2
test-all:
jobs:
- hold:
type: approval
- integration-tests-core:
requires:
- hold
- integration-tests-spark-thrift:
requires:
- hold
- integration-tests-trino:
requires:
- hold
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
name: Bug report
about: I think I've found a bug
title: "[BUG]"
labels: ""
assignees: ""
---

### Is this a new bug in dbt-date?

- [ ] I believe this is a new bug in dbt-date
- [ ] I have searched the existing issues, and I could not find an existing issue for this bug

### Current Behavior

### Expected Behavior

### Steps To Reproduce

### Relevant log output

### Environment

```markdown
- OS:
- Python:
- dbt:
- dbt-expectations:
```

### Which database adapter are you using with dbt?

Note: dbt-date currently does not support database adapters other than the ones listed below.

- Postgres
- Snowflake
- BigQuery

### Additional Context
7 changes: 7 additions & 0 deletions dev/dags/dbt/simple/dbt_packages/dbt_date/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

target/
dbt_packages/
logs/
.python-version
integration_tests/.spark-warehouse
integration_tests/.hive-metastore
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
---
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: check-json
- id: check-ast
- id: check-merge-conflict
- id: check-toml
- id: check-yaml
args: [--unsafe]
- id: debug-statements
- id: detect-private-key
- id: end-of-file-fixer
exclude: macros/calendar_date/week_of_year.sql
- repo: https://github.com/pre-commit/mirrors-prettier
rev: "v3.1.0"
hooks:
- id: prettier
types_or: [json, markdown, yaml]
- repo: https://github.com/tconbeer/sqlfmt
rev: v0.26.0
hooks:
- id: sqlfmt
language_version: python
additional_dependencies: [".[jinjafmt]"]
Loading