diff --git a/.gitignore b/.gitignore index 5991c231ce..0af8bcb446 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ docs/profiles/* # dbt_packages is a directory that gets created when you run dbt deps -dbt_packages/ +dev/dags/dbt/jaffle_shop/dbt_packages/ # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ba224730e..a3d7822a9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,7 @@ repos: apache-airflow, ] files: ^cosmos +exclude: "dev/dags/dbt/simple/dbt_packages/.*" ci: autofix_commit_msg: 🎨 [pre-commit.ci] Auto format from pre-commit.com hooks diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/.circleci/config.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/.circleci/config.yml new file mode 100644 index 0000000000..8c4238f4d5 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/.circleci/config.yml @@ -0,0 +1,181 @@ +version: 2.1 + +jobs: + integration-tests-core: + docker: + - image: cimg/python:3.9.9 + - image: cimg/postgres:14.0 + + resource_class: small + + environment: + DBT_PROFILES_DIR: ./integration_tests/ci + DBT_PROJECT_DIR: ./integration_tests + BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" + DBT_VERSION: 1.8.* + + steps: + - checkout + - run: &pip-install-core + name: Install core Python packages & dbt-core + command: | + python3 -m venv venv + . venv/bin/activate + pip install -U pip setuptools wheel + pip install "dbt-core==$DBT_VERSION" + + - run: + name: Install dbt adapter packages + command: | + python3 -m venv venv + . venv/bin/activate + pip install "dbt-postgres==$DBT_VERSION" "dbt-bigquery==$DBT_VERSION" "dbt-snowflake==$DBT_VERSION" + pip install "dbt-duckdb==$DBT_VERSION" + + - run: &dbt-deps + name: Install dbt dependencies + command: | + . venv/bin/activate + dbt deps --project-dir $DBT_PROJECT_DIR + + - run: + name: "Run Tests - Postgres" + environment: + POSTGRES_HOST: localhost + POSTGRES_TEST_USER: postgres + POSTGRES_TEST_PASSWORD: "" + POSTGRES_TEST_PORT: 5432 + POSTGRES_TEST_DATABASE: circle_test + POSTGRES_TEST_SCHEMA: dbt_date_integration_tests + command: | + . venv/bin/activate + dbt build -t postgres --project-dir $DBT_PROJECT_DIR + + - run: + name: "Set up GCP credentials" + command: | + echo "Writing to $BIGQUERY_SERVICE_KEY_PATH" + echo $BIGQUERY_SERVICE_KEY > $BIGQUERY_SERVICE_KEY_PATH + FILESIZE=$(stat -c%s "$BIGQUERY_SERVICE_KEY_PATH") + echo "Size of $BIGQUERY_SERVICE_KEY_PATH = $FILESIZE bytes." + echo "BIGQUERY_TEST_DATABASE = $BIGQUERY_TEST_DATABASE" + + - run: + name: "Run Tests - BigQuery" + command: | + . venv/bin/activate + dbt build -t bigquery --project-dir $DBT_PROJECT_DIR + + - run: + name: "Run Tests - Snowflake" + command: | + . venv/bin/activate + dbt build -t snowflake --project-dir $DBT_PROJECT_DIR + + - run: + name: "Run Tests - DuckDB" + command: | + . venv/bin/activate + dbt build -t duckdb --project-dir $DBT_PROJECT_DIR + + - store_artifacts: + path: ./logs + + integration-tests-spark-thrift: + docker: + - image: cimg/python:3.9.9 + - image: godatadriven/spark:3.1.1 + environment: + WAIT_FOR: localhost:5432 + command: > + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name Thrift JDBC/ODBC Server + - image: postgres:9.6.17-alpine + environment: + POSTGRES_USER: dbt + POSTGRES_PASSWORD: dbt + POSTGRES_DB: metastore + + resource_class: small + + environment: + DBT_PROFILES_DIR: ./integration_tests/ci + DBT_PROJECT_DIR: ./integration_tests + DBT_VERSION: 1.8.* + + steps: + - checkout + - run: + name: Install Ubuntu packages + command: | + sudo apt-get update + sudo apt-get install libsasl2-dev libsasl2-2 + - run: *pip-install-core + - run: + name: Install dbt adapter packages + command: | + python3 -m venv venv + . venv/bin/activate + pip install "dbt-spark==$DBT_VERSION" "dbt-spark[PyHive]==$DBT_VERSION" + - run: *dbt-deps + - run: + name: Wait for Spark-Thrift + command: dockerize -wait tcp://localhost:10000 -timeout 15m -wait-retry-interval 5s + - run: + name: "Run Tests - Spark" + command: | + . venv/bin/activate + dbt build -t spark --project-dir $DBT_PROJECT_DIR + + - store_artifacts: + path: ./logs + + integration-tests-trino: + docker: + - image: cimg/python:3.11 + - image: trinodb/trino:431 + + resource_class: small + + environment: + DBT_PROFILES_DIR: ./integration_tests/ci + DBT_PROJECT_DIR: ./integration_tests + DBT_VERSION: 1.8.* + + steps: + - checkout + - run: *pip-install-core + - run: + name: Install dbt adapter packages + command: | + python3 -m venv venv + . venv/bin/activate + pip install "dbt-trino==$DBT_VERSION" + - run: *dbt-deps + - setup_remote_docker + - run: + name: Run Trino server + command: | + docker run --name trino -p 8080:8080 -d -v `pwd`/integration_tests/docker/trino/catalog:/etc/trino/catalog trinodb/trino:431 + timeout 5m bash -c -- 'while ! docker logs trino 2>&1 | tail -n 1 | grep "SERVER STARTED"; do sleep 2; done' + - run: + name: "Run Tests - Trino" + command: | + . venv/bin/activate + dbt build -t trino --project-dir $DBT_PROJECT_DIR + +workflows: + version: 2 + test-all: + jobs: + - hold: + type: approval + - integration-tests-core: + requires: + - hold + - integration-tests-spark-thrift: + requires: + - hold + - integration-tests-trino: + requires: + - hold diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/.github/ISSUE_TEMPLATE/bug_report.md b/dev/dags/dbt/simple/dbt_packages/dbt_date/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..6e29388018 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,39 @@ +--- +name: Bug report +about: I think I've found a bug +title: "[BUG]" +labels: "" +assignees: "" +--- + +### Is this a new bug in dbt-date? + +- [ ] I believe this is a new bug in dbt-date +- [ ] I have searched the existing issues, and I could not find an existing issue for this bug + +### Current Behavior + +### Expected Behavior + +### Steps To Reproduce + +### Relevant log output + +### Environment + +```markdown +- OS: +- Python: +- dbt: +- dbt-expectations: +``` + +### Which database adapter are you using with dbt? + +Note: dbt-date currently does not support database adapters other than the ones listed below. + +- Postgres +- Snowflake +- BigQuery + +### Additional Context diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/.gitignore b/dev/dags/dbt/simple/dbt_packages/dbt_date/.gitignore new file mode 100644 index 0000000000..404fb93a8d --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/.gitignore @@ -0,0 +1,7 @@ + +target/ +dbt_packages/ +logs/ +.python-version +integration_tests/.spark-warehouse +integration_tests/.hive-metastore diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/.pre-commit-config.yaml b/dev/dags/dbt/simple/dbt_packages/dbt_date/.pre-commit-config.yaml new file mode 100644 index 0000000000..adac6499fe --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/.pre-commit-config.yaml @@ -0,0 +1,27 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: check-json + - id: check-ast + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + args: [--unsafe] + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + exclude: macros/calendar_date/week_of_year.sql + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.1.0" + hooks: + - id: prettier + types_or: [json, markdown, yaml] + - repo: https://github.com/tconbeer/sqlfmt + rev: v0.26.0 + hooks: + - id: sqlfmt + language_version: python + additional_dependencies: [".[jinjafmt]"] diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/CHANGELOG.md b/dev/dags/dbt/simple/dbt_packages/dbt_date/CHANGELOG.md new file mode 100644 index 0000000000..2dfc590223 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/CHANGELOG.md @@ -0,0 +1,173 @@ +# dbt-date v0.10.1 + +## Fixes + +- Change marco month_name.sql by @flixflop in https://github.com/calogica/dbt-date/pull/123 + +# dbt-date v0.10.0 + +## New Features + +- Added Trino support by @damian3031 in https://github.com/calogica/dbt-date/pull/115 + +# dbt-date v0.9.2 + +## New Features + +- Add date() and datetime() short hand macros by @gregology in https://github.com/calogica/dbt-date/pull/112 + +# dbt-date v0.9.1 + +## Fixes + +- Fixed dataspine interval by @clausherther in https://github.com/calogica/dbt-date/pull/109 + +# dbt-date v0.9.0 + +## New Features + +- Add Spark support by @clausherther in https://github.com/calogica/dbt-date/pull/108 + +**Full Changelog**: https://github.com/calogica/dbt-date/compare/0.8.1...0.9.0 + +# dbt-date v0.8.0 + +## New Features + +- Add support for dbt-duckdb to the dbt_date package by @jwills in https://github.com/calogica/dbt-date/pull/105 + +## Docs + +- Update README.md by @JordanZimmitti in https://github.com/calogica/dbt-date/pull/99 +- Update README.md by @tgmof in https://github.com/calogica/dbt-date/pull/104 + +**Full Changelog**: https://github.com/calogica/dbt-date/compare/0.7.2...0.8.0 + +# dbt-date v0.7.2 + +## Fixes + +- Update refs to dbt-core macros by @clausherther in https://github.com/calogica/dbt-date/pull/97 +- Add dbt prefix to current_timestamp macro by @GtheSheep in https://github.com/calogica/dbt-date/pull/98 + +# dbt-date v0.7.1 + +- Fix calls to last_month and next_month by @clausherther in https://github.com/calogica/dbt-date/pull/95 + +# dbt-date v0.7.0 + +## Breaking Changes + +- Removed dependency on dbt-utils by @clausherther in https://github.com/calogica/dbt-date/pull/91 + +# dbt-date v0.6.3 + +- Switch to dbt-core's implementation of current_timestamp() by @clausherther in https://github.com/calogica/dbt-date/pull/88 + +# dbt-date v0.6.2 + +- Simplify convert from `source_tz` by @clausherther in https://github.com/calogica/dbt-date/pull/86 +- Fix issue of convert_timezone converting timestamps twice for timestamp_tz date types (issue #85, #77, #76 ) + +# dbt-date v0.6.1 + +- New: added `round_timestamp` macro by @jpmmcneill in https://github.com/calogica/dbt-date/pull/84 + +# dbt-date v0.6.0 + +- Move to dbt-utils 0.9.0 +- Remove references to deprecated dbt-utils cross-db macros by @clausherther in https://github.com/calogica/dbt-date/pull/79 + +# dbt-date v0.5.7 + +- Add github actions workflow by @clausherther in https://github.com/calogica/dbt-date/pull/69 +- Fix Redshift timezone conversion macro by @wellykachtel in https://github.com/calogica/dbt-date/pull/71 + +# dbt-date v0.5.6 + +- Fix missing bracket by @clausherther in https://github.com/calogica/dbt-date/pull/66 + +# dbt-date v0.5.5 + +- Fix README table of contents' links by @coisnepe [#61](https://github.com/calogica/dbt-date/pull/61) +- Fix timezone conversion macro on redshift by @msnidal [#63](https://github.com/calogica/dbt-date/pull/63) + +# dbt-date v0.5.4 + +- Updated Documentation [#60](https://github.com/calogica/dbt-date/pull/60) + +# dbt-date v0.5.3 + +- Allow negative shift year for fiscal periods ([#57](https://github.com/calogica/dbt-date/issues/57) [@boludo00](https://github.com/boludo00)) + +# dbt-date v0.5.2 + +- Fix [#55](https://github.com/calogica/dbt-date/issues/55) by removing dead macro. + +# dbt-date v0.5.1 + +- Fix `week_start` and `week_end` on Snowflake ([#53](https://github.com/calogica/dbt-date/issues/53), [#54](https://github.com/calogica/dbt-date/pull/54)) + +# dbt-date v0.5.0 + +- Deprecates support for dbt < 1.0.0 + +# dbt-date v0.4.2 + +## Under the hood + +- Patch: adds support for dbt 1.x + +# dbt-date v0.4.1 + +## Under the hood + +- Support for dbt 0.21.x + +# dbt-date v0.4.0 + +## Breaking Changes + +- Updates calls to adapter.dispatch to support `dbt >= 0.20` (see [Changes to dispatch in dbt v0.20 #34](https://github.com/calogica/dbt-date/issues/34)) + +- Requires `dbt >= 0.20` + +## Under the hood + +- Adds tests for timestamp and timezone macros (previously untested, new dbt version highlighted that) + +# dbt-date v0.3.1 + +_Patch release_ + +## Fixes + +- Fixed a bug in `snowflake__from_unixtimestamp` that prevented the core functionaility from being called ([#38](https://github.com/calogica/dbt-date/pull/38) by @swanderz) + +## Under the hood + +- Simplified `join` syntax ([#36](https://github.com/calogica/dbt-date/pull/36)) + +# dbt-date v0.3.0 + +## Breaking Changes + +- Switched `day_of_week` column in `get_date_dimension` from ISO to _not_ ISO to align with the rest of the package. [#33](https://github.com/calogica/dbt-date/pull/33) (@davesgonechina) + +## Features + +- Added `day_of_week_iso` column to `get_date_dimension` [#33](https://github.com/calogica/dbt-date/pull/33) (@davesgonechina) + +- Added `prior_year_iso_week_of_year` column to `get_date_dimension` + +## Fixes + +- Refactored Snowflake's `day_name` to not be ISO dependent [#33](https://github.com/calogica/dbt-date/pull/33) (@davesgonechina) + +- Fixed data types for `day_of_*` attributes in Redshift ([#28](https://github.com/calogica/dbt-date/pull/28) by @sparcs) + +- Fixed / added support for date parts other than `day` in `get_base_dates` ([#30](https://github.com/calogica/dbt-date/pull/30)) + +## Under the hood + +- Making it easier to shim macros for other platforms ([#27](https://github.com/calogica/dbt-date/pull/27) by @swanderz) diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/LICENSE b/dev/dags/dbt/simple/dbt_packages/dbt_date/LICENSE new file mode 100644 index 0000000000..956ea65b7a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Pádraic Slattery + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/README.md b/dev/dags/dbt/simple/dbt_packages/dbt_date/README.md new file mode 100644 index 0000000000..466662b5ee --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/README.md @@ -0,0 +1,843 @@ +
+ + + + + + + + + +
+ +# dbt-date + +`dbt-date` is an extension package for [**dbt**](https://github.com/dbt-labs/dbt) to handle common date logic and calendar functionality. + +Development of `dbt-date` is supported by [Xebia Data](https://xebia.com/digital-transformation/data-and-ai/) (formerly known as GoDataDriven): + +
+
+ Xebia logo +
+
+ +## Install + +Include in `packages.yml` + +```yaml +packages: + - package: godatadriven/dbt_date + version: 0.10.1 + # for the latest version tag +``` + +This package supports: + +- Postgres +- Snowflake +- BigQuery +- DuckDB +- Spark +- Trino + +## Variables + +The following variables need to be defined in your `dbt_project.yml` file: + +```yaml +vars: + "dbt_date:time_zone": "America/Los_Angeles" +``` + +You may specify [any valid timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) in place of `America/Los_Angeles`. +For example, use `America/New_York` for East Coast Time. + +## Available Macros + +### Date Dimension + +- [get_base_dates](#get_base_datesstart_datenone-end_datenone-n_datepartsnone-datepartday) +- [get_date_dimension](#get_date_dimensionstart_date-end_date) + +### Calendar Date + +- [convert_timezone](#convert_timezone-column-target_tznone-source_tznone) +- [date_part](#date_partdatepart-date) +- [day_name](#day_namedate-shorttrue) +- [day_of_month](#day_of_monthdate) +- [day_of_week](#day_of_weekdate-isoweektrue) +- [day_of_year](#day_of_yeardate) +- [from_unixtimestamp](#from_unixtimestampepochs-formatseconds) +- [iso_week_end](#iso_week_enddatenone-tznone) +- [iso_week_of_year](#iso_week_of_yeardatenone-tznone) +- [iso_week_start](#iso_week_startdatenone-tznone) +- [last_month_name](#last_month_nameshorttrue-tznone) +- [last_month_number](#last_month_numbertznone) +- [last_month](#last_monthtznone) +- [last_week](#last_weektznone) +- [month_name](#month_namedate-shorttrue-tznone) +- [n_days_ago](#n_days_agon-datenone-tznone) +- [n_days_away](#n_days_awayn-datenone-tznone) +- [n_months_ago](#n_months_agon-tznone) +- [n_months_away](#n_months_awayn-tznone) +- [n_weeks_ago](#n_weeks_agon-tznone) +- [n_weeks_away](#n_weeks_awayn-tznone) +- [next_month_name](#next_month_nameshorttrue-tznone) +- [next_month_number](#next_month_numbertznone) +- [next_month](#next_monthtznone) +- [next_week](#next_weektznone) +- [now](#nowtznone) +- [periods_since](#periods_sincedate_col-period_nameday-tznone) +- [round_timestamp](#round_timestamptimestamp) +- [to_unixtimestamp](#to_unixtimestamptimestamp) +- [today](#todaytznone) +- [tomorrow](#tomorrowdatenone-tznone) +- [week_end](#week_enddatenone-tznone) +- [week_of_year](#week_of_yeardatenone-tznone) +- [week_start](#week_startdatenone-tznone) +- [yesterday](#yesterdaydatenone-tznone) + +## Fiscal Date + +- [get_fiscal_periods](#get_fiscal_periodsdates-year_end_month-week_start_day-shift_year1) + +## Utils + +- [date](#dateyear-month-day) +- [datetime](#datetimeyear-month-day-hour0-minute0-second0-microsecond0-tznone) + +## Documentation + +### [get_base_dates](macros/get_base_dates.sql)(`start_date=None, end_date=None, n_dateparts=None, datepart="day"`) + +A wrapper around [`dbt_utils.date_spine`](https://github.com/dbt-labs/dbt-utils#date_spine-source) that allows you to specify either `start_date` and `end_date` for your date spine, or specify a number of periods (`n_dateparts`) in the past from today. + +Usage to build a daily date dimension for the years 2015 to 2022: + +```sql +{{ dbt_date.get_base_dates(start_date="2015-01-01", end_date="2023-01-01") }} +``` + +or to build a daily date dimension for the last 3 years: + +```sql +{{ dbt_date.get_base_dates(n_dateparts=365*3, datepart="day") }} +``` + +### [get_date_dimension](macros/get_date_dimension.sql)(`start_date, end_date`) + +Returns a query to build date dimension from/to specified dates, including a number of useful columns based on each date. +See the [example model](integration_tests/models/dim_date.sql) for details. + +Usage: + +```sql +{{ dbt_date.get_date_dimension("2015-01-01", "2022-12-31") }} +``` + +### Fiscal Periods + +### [get_fiscal_periods](macros/fiscal_date/get_fiscal_periods.sql)(`dates, year_end_month, week_start_day, shift_year=1`) + +Returns a query to build a fiscal period calendar based on the 4-5-4 week retail period concept. +See the [example model](integration_tests/models/dim_date_fiscal.sql) for details and this [blog post](https://calogica.com/sql/dbt/2018/11/15/retail-calendar-in-sql.html) for more context on custom business calendars. + +Usage: + +```sql +{{ dbt_date.get_fiscal_periods(ref("dates"), year_end_month, week_start_day) }} +``` + +Note: the first parameter expects a dbt `ref` variable, i.e. a reference to a model containing the necessary date dimension attributes, which can be generated via the `get_date_dimension` macro (see above). + +### Date + +### [convert_timezone](macros/calendar_date/convert_timezone.sql)( `column, target_tz=None, source_tz=None`) + +Cross-database implemention of convert_timezone function. + +Usage: + +```sql +{{ dbt_date.convert_timezone("my_column") }} +``` + +or, specify a target timezone: + +```sql +{{ dbt_date.convert_timezone("my_column", "America/New_York") }} +``` + +or, also specify a source timezone: + +```sql +{{ dbt_date.convert_timezone("my_column", "America/New_York", "UTC") }} +``` + +Using named parameters, we can also specify the source only and rely on the configuration parameter for the target: + +```sql +{{ dbt_date.convert_timezone("my_column", source_tz="UTC") }} +``` + +### [date_part](macros/calendar_date/date_part.sql)(`datepart, date`) + +Extracts date parts from date. + +Usage: + +```sql +{{ dbt_date.date_part("dayofweek", "date_col") }} as day_of_week +``` + +### [day_name](macros/calendar_date/day_name.sql)(`date, short=True`) + +Extracts name of weekday from date. + +Usage: + +```sql +{{ dbt_date.day_name("date_col") }} as day_of_week_short_name +``` + +```sql +{{ dbt_date.day_name("date_col", short=true) }} as day_of_week_short_name +``` + +```sql +{{ dbt_date.day_name("date_col", short=false) }} as day_of_week_long_name +``` + +### [day_of_month](macros/calendar_date/day_of_month.sql)(`date`) + +Extracts day of the month from a date (e.g. `2022-03-06` --> `6`). + +Usage: + +```sql +{{ dbt_date.day_of_month("date_col") }} as day_of_month +``` + +### [day_of_week](macros/calendar_date/day_of_week.sql)(`date, isoweek=true`) + +Extracts day of the week _number_ from a date, starting with **1**. +By default, uses `isoweek=True`, i.e. assumes week starts on _Monday_. + +Usage: + +```sql +{{ dbt_date.day_of_week("'2022-03-06'") }} as day_of_week_iso +``` + +returns: **7** (Sunday is the _last_ day of the ISO week) + +```sql +{{ dbt_date.day_of_week("'2022-03-06'", isoweek=False) }} as day_of_week +``` + +returns: **1** (Sunday is the _first_ day of the non-ISO week) + +### [day_of_year](macros/calendar_date/day_of_year.sql)(`date`) + +Extracts day of the year from a date (e.g. `2022-02-02` --> `33`). + +Usage: + +```sql +{{ dbt_date.day_of_year("date_col") }} as day_of_year +``` + +or + +```sql +{{ dbt_date.day_of_year("'2022-02-02'") }} as day_of_year +``` + +returns: **33** + +### [from_unixtimestamp](macros/calendar_date/from_unixtimestamp.sql)(`epochs, format="seconds"`) + +Converts an `epoch` into a timestamp. The default for `format` is `seconds`, which can overriden depending your data"s epoch format. + +Usage: + +```sql +{{ dbt_date.from_unixtimestamp("epoch_column") }} as timestamp_column +``` + +```sql +{{ dbt_date.from_unixtimestamp("epoch_column", format="milliseconds") }} as timestamp_column +``` + +See also: [to_unixtimestamp](#to_unixtimestamp) + +### [iso_week_end](macros/calendar_date/iso_week_end.sql)(`date=None, tz=None`) + +Computes the week ending date using ISO format, i.e. week starting **Monday** and ending **Sunday**. + +Usage: + +```sql +{{ dbt_date.iso_week_end("date_col") }} as iso_week_end_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.iso_week_end("date_col", tz="America/New_York") }} as iso_week_end_date +``` + +### [iso_week_of_year](macros/calendar_date/iso_week_of_year.sql)(`date=None, tz=None`) + +Computes the week of the year using ISO format, i.e. week starting **Monday**. + +Usage: + +```sql +{{ dbt_date.iso_week_of_year("date_col") }} as iso_week_of_year +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.iso_week_of_year("date_col", tz="America/New_York") }} as iso_week_of_year +``` + +### [iso_week_start](macros/calendar_date/iso_week_start.sql)(`date=None, tz=None`) + +Computes the week starting date using ISO format, i.e. week starting **Monday**. + +Usage: + +```sql +{{ dbt_date.iso_week_start("date_col") }} as iso_week_start_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.iso_week_start("date_col", tz="America/New_York") }} as iso_week_start_date +``` + +### [last_month_name](macros/calendar_date/last_month_name.sql)(`short=True, tz=None`) + +Extracts the name of the prior month from a date. + +```sql +{{ dbt_date.last_month_name() }} as last_month_short_name +``` + +```sql +{{ dbt_date.last_month_name(short=true) }} as last_month_short_name +``` + +```sql +{{ dbt_date.last_month_name(short=false) }} as last_month_long_name +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.last_month_name(tz="America/New_York") }} as last_month_short_name +``` + +### [last_month_number](macros/calendar_date/last_month_number.sql)(`tz=None`) + +Returns the number of the prior month. + +```sql +{{ dbt_date.last_month_number() }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.last_month_number(tz="America/New_York") }} +``` + +### [last_month](macros/calendar_date/last_month.sql)(`tz=None`) + +Returns the start date of the prior month. + +```sql +{{ dbt_date.last_month() }} as last_month_start_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.last_month(tz="America/New_York") }} as last_month_start_date +``` + +### [last_week](macros/calendar_date/last_week.sql)(`tz=None`) + +Convenience function to get the start date of last week (non-ISO) + +Wraps: + +```sql +{{ dbt_date.n_weeks_ago(1, tz) }} +``` + +Usage: + +```sql +{{ dbt_date.last_week()) }} as last_week_start_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.last_week(tz="America/New_York)) }} as last_week_start_date +``` + +### [month_name](macros/calendar_date/month_name.sql)(`date, short=True, tz=None`) + +Extracts the name of the month from a date. + +```sql +{{ dbt_date.month_name(date_col) }} as month_short_name +``` + +```sql +{{ dbt_date.month_name(date_col, short=true) }} as month_short_name +``` + +```sql +{{ dbt_date.month_name(date_col, short=false) }} as month_long_name +``` + +### [n_days_ago](macros/calendar_date/n_days_ago.sql)(`n, date=None, tz=None`) + +Gets date _n_ days ago, based on local date. + +Usage: + +```sql +{{ dbt_date.n_days_ago(7) }} +``` + +Alternatively, you can specify a date column instead of defaulting the local date: + +```sql +{{ dbt_date.n_days_ago(7, date="date_col") }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_days_ago(7, tz="America/New_York)) }} +``` + +### [n_days_away](macros/calendar_date/n_days_away.sql)(`n, date=None, tz=None`) + +Gets date _n_ days away, based on local date. + +Usage: + +```sql +{{ dbt_date.n_days_away(7) }} +``` + +Alternatively, you can specify a date column instead of defaulting the local date: + +```sql +{{ dbt_date.n_days_away(7, date="date_col") }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_days_away(7, tz="America/New_York)) }} +``` + +### [n_months_ago](macros/calendar_date/n_months_ago.sql)(`n, tz=None`) + +Gets date _n_ months ago, based on local date. + +Usage: + +```sql +{{ dbt_date.n_months_ago(12) }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_months_ago(12, tz="America/New_York)) }} +``` + +### [n_months_away](macros/calendar_date/n_months_away.sql)(`n, tz=None`) + +Gets date _n_ months away, based on local date. + +Usage: + +```sql +{{ dbt_date.n_months_ago(12) }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_months_away(12, tz="America/New_York)) }} +``` + +### [n_weeks_ago](macros/calendar_date/n_weeks_ago.sql)(`n, tz=None`) + +Gets date _n_ weeks ago, based on local date. + +Usage: + +```sql +{{ dbt_date.n_weeks_ago(12) }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_weeks_ago(12, tz="America/New_York)) }} +``` + +### [n_weeks_away](macros/calendar_date/n_weeks_away.sql)(`n, tz=None`) + +Gets date _n_ weeks away, based on local date. + +Usage: + +```sql +{{ dbt_date.n_weeks_away(12) }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.n_weeks_away(12, tz="America/New_York)) }} +``` + +### [next_month_name](macros/calendar_date/next_month_name.sql)(`short=True, tz=None`) + +Extracts the name of the next month from a date. + +```sql +{{ dbt_date.next_month_name() }} as next_month_short_name +``` + +```sql +{{ dbt_date.next_month_name(short=true) }} as next_month_short_name +``` + +```sql +{{ dbt_date.next_month_name(short=false) }} as next_month_long_name +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.next_month_name(tz="America/New_York") }} as next_month_short_name +``` + +### [next_month_number](macros/calendar_date/next_month_number.sql)(`tz=None`) + +Returns the number of the next month. + +```sql +{{ dbt_date.next_month_number() }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.next_month_number(tz="America/New_York") }} +``` + +### [next_month](macros/calendar_date/next_month.sql)(`tz=None`) + +Returns the start date of the next month. + +```sql +{{ dbt_date.next_month() }} as next_month_start_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.next_month(tz="America/New_York") }} as next_month_start_date +``` + +### [next_week](macros/calendar_date/next_week.sql)(`tz=None`) + +Convenience function to get the start date of next week (non-ISO) + +Wraps: + +```sql +{{ dbt_date.n_weeks_away(1, tz) }} +``` + +Usage: + +```sql +{{ dbt_date.next_week()) }} as next_week_start_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.next_week(tz="America/New_York") }} as next_week_start_date +``` + +### [now](macros/calendar_date/now.sql)(`tz=None`) + +Gets current timestamp based on local timezone (specified). Default is "America/Los_Angeles". + +Usage: + +```sql +{{ dbt_date.now() }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.now("America/New_York") }} +``` + +### [periods_since](macros/calendar_date/periods_since.sql)(`date_col, period_name='day', tz=None`) + +Returns the number of periods since a specified date or to `now`. + +Usage: + +```sql +{{ dbt_date.periods_since("my_date_column", period_name="day") }} +``` + +or, + +```sql +{{ dbt_date.periods_since("my_timestamp_column", period_name="minute") }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.periods_since("my_timestamp_column", period_name="minute", tz="UTC") }} +``` + +### [round_timestamp](macros/calendar_date/round_timestamp.sql)(`timestamp`) + +Rounds the given timestamp or date to the nearest date (return type is `timestamp`). + +```sql +select +{{ dbt_date.round_timestamp("timestamp_col") }} as nearest_date +... +``` + +A few examples: + +```sql +{{ dbt_date.round_timestamp("'2022-02-05 18:45:15'")}} +-- results in 2022-02-06 +``` + +```sql +{{ dbt_date.round_timestamp("'2022-02-05 11:45:15'")}} +-- results in 2022-02-05 +``` + +```sql +{{ dbt_date.round_timestamp("'2022-02-05 12:00:00'")}} +-- results in 2022-02-06 +``` + +```sql +{{ dbt_date.round_timestamp("'2022-02-05 00:00:00'")}} +-- results in 2022-02-05 +``` + +### [to_unixtimestamp](macros/calendar_date/to_unixtimestamp.sql)(`timestamp`) + +Gets Unix timestamp (epochs) based on provided timestamp. + +Usage: + +```sql +{{ dbt_date.to_unixtimestamp("my_timestamp_column") }} +``` + +```sql +{{ dbt_date.to_unixtimestamp(dbt_date.now()) }} +``` + +### [today](macros/calendar_date/today.sql)(`tz=None`) + +Gets date based on local timezone. + +Usage: + +```sql +{{ dbt_date.today() }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.today("America/New_York") }} +``` + +### [tomorrow](macros/calendar_date/tomorrow.sql)(`date=None, tz=None`) + +Gets tomorrow's date, based on local date. + +Usage: + +```sql +{{ dbt_date.tomorrow() }} +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.tomorrow(tz="America/New_York") }} as date_tomorrow +``` + +Alternatively, you can also override the anchor date from the default `today` to some other date: + +```sql +{{ dbt_date.tomorrow(date="date_col", tz="America/New_York") }} as date_tomorrow +``` + +### [week_end](macros/calendar_date/week_end.sql)(`date=None, tz=None`) + +Computes the week ending date using standard (US) format, i.e. week starting **Sunday**. + +Usage: + +If `date` is not specified, the date anchor defaults to `today`. + +```sql +{{ dbt_date.week_end() }} as week_end_date +``` + +or specify a date (column): + +```sql +{{ dbt_date.week_end("date_col") }} as week_end_date +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.week_end("date_col", tz="America/New_York") }} as week_end_date +``` + +### [week_of_year](macros/calendar_date/week_of_year.sql)(`date=None, tz=None`) + +Computes the week of the year using standard (US) format, i.e. week starting **Sunday** and ending **Saturday**. + +Usage: + +If `date` is not specified, the date anchor defaults to `today`. + +```sql +{{ dbt_date.week_of_year() }} as week_of_year +``` + +or specify a date (column): + +```sql +{{ dbt_date.week_of_year("date_col") }} as week_of_year +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.week_of_year("date_col", tz="America/New_York") }} as week_of_year +``` + +### [week_start](macros/calendar_date/week_start.sql)(`date=None, tz=None`) + +Computes the week starting date using standard (US) format, i.e. week starting **Sunday**. + +Usage: + +If `date` is not specified, the date anchor defaults to `today`. + +```sql +{{ dbt_date.week_start() }} as week_start +``` + +or specify a date (column): + +```sql +{{ dbt_date.week_start("date_col") }} as week_start +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.week_start("date_col", tz="America/New_York") }} as week_start +``` + +### [yesterday](macros/calendar_date/yesterday.sql)(`date=None, tz=None`) + +Gets yesterday's date, based on local date. + +Usage: + +If `date` is not specified, the date anchor defaults to `today`. + +```sql +{{ dbt_date.yesterday() }} as date_yesterday +``` + +or specify a date (column): + +```sql +{{ dbt_date.yesterday("date_col") }} as date_yesterday +``` + +or, optionally, you can override the default timezone: + +```sql +{{ dbt_date.yesterday(tz="America/New_York") }} as date_yesterday +``` + +### [date](macros/_utils/modules_datetime.sql)(`year`, `month`, `day`) + +Reduces the boilerplate syntax required to produce a `date` object. This is not converted to a string to allow pythonic manipulation. + +Usage: + +```sql +{% set date_object = dbt_date.date(1997, 9, 29) %} +``` + +### [datetime](macros/_utils/modules_datetime.sql)(`year`, `month`, `day`, `hour=0`, `minute=0`, `second=0`, `microsecond=0`, `tz=None`) + +Reduces the boilerplate syntax required to produce a `datetime` object. This is not converted to a string to allow pythonic manipulation. + +Usage: + +```sql +{% set datetime_object = dbt_date.datetime(1997, 9, 29, 6, 14) %} +``` + +or, optionally, you can override the default timezone: + +```sql +{% set datetime_object = dbt_date.datetime(1997, 9, 29, 6, 14, tz='America/New_York') %} +``` + +## Integration Tests (Developers Only) + +This project contains integration tests for all test macros in a separate `integration_tests` dbt project contained in this repo. + +To run the tests: + +1. You will need a profile called `integration_tests` in `~/.dbt/profiles.yml` pointing to a writable database. We only support postgres, BigQuery and Snowflake. +2. Then, from within the `integration_tests` folder, run `dbt build` to run the test models in `integration_tests/models/schema_tests/` and run the tests specified in `integration_tests/models/schema_tests/schema.yml` diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/dbt_project.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/dbt_project.yml new file mode 100644 index 0000000000..0f66aab9cb --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/dbt_project.yml @@ -0,0 +1,19 @@ +name: "dbt_date" +version: "0.5.0" + +config-version: 2 + +target-path: "target" +clean-targets: ["target", "dbt_packages"] +macro-paths: ["macros"] +log-path: "logs" + +require-dbt-version: [">=1.2.0", "<2.0.0"] +profile: integration_tests + +quoting: + identifier: false + schema: false + +vars: + "dbt_date:time_zone": "America/Los_Angeles" diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/images/xebia-logo-large-transparent.png b/dev/dags/dbt/simple/dbt_packages/dbt_date/images/xebia-logo-large-transparent.png new file mode 100644 index 0000000000..0eae7b461a Binary files /dev/null and b/dev/dags/dbt/simple/dbt_packages/dbt_date/images/xebia-logo-large-transparent.png differ diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/ci/profiles.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/ci/profiles.yml new file mode 100644 index 0000000000..767e5aca1a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/ci/profiles.yml @@ -0,0 +1,58 @@ +integration_tests: + outputs: + postgres: + type: postgres + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_TEST_USER') }}" + pass: "{{ env_var('POSTGRES_TEST_PASSWORD') }}" + port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_TEST_DATABASE') }}" + schema: "{{ env_var('POSTGRES_TEST_SCHEMA') }}" + threads: 5 + + bigquery: + type: bigquery + method: service-account + keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: "{{ env_var('BIGQUERY_TEST_SCHEMA') }}" + threads: 10 + + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}" + threads: 10 + + duckdb: + type: duckdb + path: ":memory:" + + spark: + type: spark + method: thrift + host: 127.0.0.1 + port: 10000 + user: dbt + schema: analytics + connect_retries: 5 + connect_timeout: 60 + retry_all: true + + trino: + type: trino + method: none + host: localhost + port: 8080 + user: admin + catalog: memory + schema: default + timezone: UTC + threads: 4 + + target: postgres diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/dbt_project.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/dbt_project.yml new file mode 100644 index 0000000000..7a0a7d2039 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/dbt_project.yml @@ -0,0 +1,39 @@ +name: "dbt_date_integration_tests" +version: "1.0" + +profile: "integration_tests" + +config-version: 2 + +model-paths: ["models"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] + +target-path: "target" +clean-targets: ["target", "dbt_modules", "dbt_packages"] + +dispatch: + - macro_namespace: dbt_date + search_order: ["dbt_date_integration_tests", "dbt_date"] # enable override + +flags: + partial_parse: True + send_anonymous_usage_stats: False + warn_error_options: + silence: # To silence or ignore dbt 1.8.x upgrade warnings + - TestsConfigDeprecation + +vars: + dbt_date_dispatch_list: ["dbt_date_integration_tests"] + "dbt_date:time_zone": "America/Los_Angeles" + +quoting: + database: false + identifier: false + schema: false + +models: + dbt_date_integration_tests: + schema: dbt_date_integration_tests + materialized: table diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-compose.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-compose.yml new file mode 100644 index 0000000000..0a00c39c85 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-compose.yml @@ -0,0 +1,27 @@ +version: "3.7" +services: + dbt-spark3-thrift: + image: godatadriven/spark:3.1.1 + ports: + - "10000:10000" + - "4040:4040" + depends_on: + - dbt-hive-metastore + command: > + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name Thrift JDBC/ODBC Server + volumes: + - ./.spark-warehouse/:/spark-warehouse/ + - ./docker/hive-site.xml:/usr/spark/conf/hive-site.xml + - ./docker/spark-defaults.conf:/usr/spark/conf/spark-defaults.conf + environment: + - WAIT_FOR=dbt-hive-metastore:5432 + + dbt-hive-metastore: + image: postgres:9.6.17-alpine + volumes: + - ./.hive-metastore/:/var/lib/postgresql/data + environment: + - POSTGRES_USER=dbt + - POSTGRES_PASSWORD=dbt + - POSTGRES_DB=metastore diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-start.sh b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-start.sh new file mode 100644 index 0000000000..a8341a634f --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-start.sh @@ -0,0 +1 @@ +docker-compose up -d diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-stop.sh b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-stop.sh new file mode 100644 index 0000000000..a20ef1ad14 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker-stop.sh @@ -0,0 +1 @@ +docker-compose down && rm -rf ./.hive-metastore/ && rm -rf ./.spark-warehouse/ diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/hive-site.xml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/hive-site.xml new file mode 100644 index 0000000000..457d04f316 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/hive-site.xml @@ -0,0 +1,46 @@ + + + + + + + + javax.jdo.option.ConnectionURL + jdbc:postgresql://dbt-hive-metastore/metastore + + + + javax.jdo.option.ConnectionDriverName + org.postgresql.Driver + + + + javax.jdo.option.ConnectionUserName + dbt + + + + javax.jdo.option.ConnectionPassword + dbt + + + + hive.metastore.schema.verification + false + + diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/spark-defaults.conf b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/spark-defaults.conf new file mode 100644 index 0000000000..30ec59591a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/spark-defaults.conf @@ -0,0 +1,9 @@ +spark.driver.memory 2g +spark.executor.memory 2g +spark.hadoop.datanucleus.autoCreateTables true +spark.hadoop.datanucleus.schema.autoCreateTables true +spark.hadoop.datanucleus.fixedDatastore false +spark.serializer org.apache.spark.serializer.KryoSerializer +spark.jars.packages org.apache.hudi:hudi-spark3-bundle_2.12:0.10.0 +spark.sql.extensions org.apache.spark.sql.hudi.HoodieSparkSessionExtension +spark.driver.userClassPathFirst true diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/trino/catalog/memory.properties b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/trino/catalog/memory.properties new file mode 100644 index 0000000000..56b3b5e6ae --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/docker/trino/catalog/memory.properties @@ -0,0 +1,2 @@ +connector.name=memory +memory.max-data-per-node=128MB diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/expression_is_true.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/expression_is_true.sql new file mode 100644 index 0000000000..18a97918ea --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/expression_is_true.sql @@ -0,0 +1,23 @@ +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + {{ + return( + adapter.dispatch("test_expression_is_true", "dbt_date_integration_tests")( + model, expression, column_name, condition + ) + ) + }} +{% endtest %} + +{% macro default__test_expression_is_true(model, expression, column_name, condition) %} + + with meet_condition as (select * from {{ model }} where {{ condition }}) + + select * + from meet_condition + {% if column_name is none %} where not ({{ expression }}) + {%- else %} where not ({{ column_name }} {{ expression }}) + {%- endif %} + +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_custom_schema.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_custom_schema.sql new file mode 100644 index 0000000000..d514eb6184 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_custom_schema.sql @@ -0,0 +1,10 @@ +{% macro generate_schema_name(custom_schema_name, node) -%} + + {%- set default_schema = target.schema -%} + {%- if custom_schema_name is none -%} {{ default_schema }} + + {%- else -%} {{ custom_schema_name | trim }} + + {%- endif -%} + +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_test_dates.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_test_dates.sql new file mode 100644 index 0000000000..388c06dc59 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/macros/get_test_dates.sql @@ -0,0 +1,206 @@ +{% macro get_test_dates() -%} + select + cast('2020-11-29' as date) as date_day, + cast('2020-11-28' as date) as prior_date_day, + cast('2020-11-30' as date) as next_date_day, + 'Sunday' as day_name, + 'Sun' as day_name_short, + 29 as day_of_month, + 1 as day_of_week, + 7 as iso_day_of_week, + 334 as day_of_year, + cast('{{ get_test_week_start_date()[0] }}' as date) as week_start_date, + cast('{{ get_test_week_end_date()[0] }}' as date) as week_end_date, + {{ get_test_week_of_year()[0] }} as week_of_year, + -- in ISO terms, this is the end of the prior week + cast('2020-11-23' as date) as iso_week_start_date, + cast('2020-11-29' as date) as iso_week_end_date, + 48 as iso_week_of_year, + 11 as month_number, + 'November' as month_name, + 'Nov' as month_name_short, + 1623076520 as unix_epoch, + cast( + '{{ get_test_timestamps()[0] }}' as {{ dbt.type_timestamp() }} + ) as time_stamp, + cast( + '{{ get_test_timestamps()[1] }}' as {{ dbt.type_timestamp() }} + ) as time_stamp_utc, + cast('2021-06-07' as {{ dbt.type_timestamp() }}) as rounded_timestamp, + cast('2021-06-08' as {{ dbt.type_timestamp() }}) as rounded_timestamp_utc, + -- These columns are here to make sure these macros get run during testing: + {{ dbt_date.last_month_number() }} as last_month_number, + {{ dbt_date.last_month_name(short=False) }} as last_month_name, + {{ dbt_date.last_month_name(short=True) }} as last_month_name_short, + {{ dbt_date.next_month_number() }} as next_month_number, + {{ dbt_date.next_month_name(short=False) }} as next_month_name, + {{ dbt_date.next_month_name(short=True) }} as next_month_name_short, + cast('{{ modules.datetime.date(1997, 9, 29) }}' as date) as datetime_date, + cast( + '{{ modules.datetime.datetime(1997, 9, 29, 6, 14, 0, tzinfo=modules.pytz.timezone(var("dbt_date:time_zone"))) }}' + as {{ dbt.type_timestamp() }} + ) as datetime_datetime + + union all + + select + cast('2020-12-01' as date) as date_day, + cast('2020-11-30' as date) as prior_date_day, + cast('2020-12-02' as date) as next_date_day, + 'Tuesday' as day_name, + 'Tue' as day_name_short, + 1 as day_of_month, + 3 as day_of_week, + 2 as iso_day_of_week, + 336 as day_of_year, + cast('{{ get_test_week_start_date()[1] }}' as date) as week_start_date, + cast('{{ get_test_week_end_date()[1] }}' as date) as week_end_date, + {{ get_test_week_of_year()[1] }} as week_of_year, + cast('2020-11-30' as date) as iso_week_start_date, + cast('2020-12-06' as date) as iso_week_end_date, + 49 as iso_week_of_year, + 12 as month_number, + 'December' as month_name, + 'Dec' as month_name_short, + 1623076520 as unix_epoch, + cast( + '{{ get_test_timestamps()[0] }}' as {{ dbt.type_timestamp() }} + ) as time_stamp, + cast( + '{{ get_test_timestamps()[1] }}' as {{ dbt.type_timestamp() }} + ) as time_stamp_utc, + cast('2021-06-07' as {{ dbt.type_timestamp() }}) as rounded_timestamp, + cast('2021-06-08' as {{ dbt.type_timestamp() }}) as rounded_timestamp_utc, + -- These columns are here to make sure these macros get run during testing: + {{ dbt_date.last_month_number() }} as last_month_number, + {{ dbt_date.last_month_name(short=False) }} as last_month_name, + {{ dbt_date.last_month_name(short=True) }} as last_month_name_short, + {{ dbt_date.next_month_number() }} as next_month_number, + {{ dbt_date.next_month_name(short=False) }} as next_month_name, + {{ dbt_date.next_month_name(short=True) }} as next_month_name_short, + cast('{{ modules.datetime.date(1997, 9, 29) }}' as date) as datetime_date, + cast( + '{{ modules.datetime.datetime(1997, 9, 29, 6, 14, 0, tzinfo=modules.pytz.timezone(var("dbt_date:time_zone"))) }}' + as {{ dbt.type_timestamp() }} + ) as datetime_datetime + +{%- endmacro %} + +{% macro get_test_week_of_year() -%} + {{ + return( + adapter.dispatch("get_test_week_of_year", "dbt_date_integration_tests")() + ) + }} +{%- endmacro %} + +{% macro default__get_test_week_of_year() -%} + {# weeks_of_year for '2020-11-29' and '2020-12-01', respectively #} + {{ return([48, 48]) }} +{%- endmacro %} + +{% macro snowflake__get_test_week_of_year() -%} + {# weeks_of_year for '2020-11-29' and '2020-12-01', respectively #} + {# Snowflake uses ISO year #} + {{ return([48, 49]) }} +{%- endmacro %} + +{% macro spark__get_test_week_of_year() -%} + {# weeks_of_year for '2020-11-29' and '2020-12-01', respectively #} + {# spark uses ISO year #} + {{ return([48, 49]) }} +{%- endmacro %} + +{% macro trino__get_test_week_of_year() -%} + {# weeks_of_year for '2020-11-29' and '2020-12-01', respectively #} + {# trino uses ISO year #} + {{ return([48, 49]) }} +{%- endmacro %} + + +{% macro get_test_week_start_date() -%} + {{ + return( + adapter.dispatch( + "get_test_week_start_date", "dbt_date_integration_tests" + )() + ) + }} +{%- endmacro %} + +{% macro default__get_test_week_start_date() -%} + {{ return(["2020-11-29", "2020-11-29"]) }} +{%- endmacro %} + +{% macro spark__get_test_week_start_date() -%} + {# spark does not support non-iso weeks #} + {{ return(["2020-11-23", "2020-11-30"]) }} +{%- endmacro %} + +{% macro trino__get_test_week_start_date() -%} + {# trino does not support non-iso weeks #} + {{ return(["2020-11-23", "2020-11-30"]) }} +{%- endmacro %} + + +{% macro get_test_week_end_date() -%} + {{ + return( + adapter.dispatch("get_test_week_end_date", "dbt_date_integration_tests")() + ) + }} +{%- endmacro %} + +{% macro default__get_test_week_end_date() -%} + {{ return(["2020-12-05", "2020-12-05"]) }} +{%- endmacro %} + +{% macro spark__get_test_week_end_date() -%} + {# spark does not support non-iso weeks #} + {{ return(["2020-11-29", "2020-12-06"]) }} +{%- endmacro %} + +{% macro trino__get_test_week_end_date() -%} + {# trino does not support non-iso weeks #} + {{ return(["2020-11-29", "2020-12-06"]) }} +{%- endmacro %} + + +{% macro get_test_timestamps() -%} + {{ + return( + adapter.dispatch("get_test_timestamps", "dbt_date_integration_tests")() + ) + }} +{%- endmacro %} + +{% macro default__get_test_timestamps() -%} + {{ + return( + [ + "2021-06-07 07:35:20.000000 America/Los_Angeles", + "2021-06-07 14:35:20.000000 UTC", + ] + ) + }} +{%- endmacro %} + +{% macro bigquery__get_test_timestamps() -%} + {{ return(["2021-06-07 07:35:20.000000", "2021-06-07 14:35:20.000000"]) }} +{%- endmacro %} + +{% macro snowflake__get_test_timestamps() -%} + {{ + return( + ["2021-06-07 07:35:20.000000 -0700", "2021-06-07 14:35:20.000000 -0000"] + ) + }} +{%- endmacro %} + +{% macro duckdb__get_test_timestamps() -%} + {{ return(["2021-06-07 07:35:20.000000", "2021-06-07 14:35:20.000000"]) }} +{%- endmacro %} + +{% macro spark__get_test_timestamps() -%} + {{ return(["2021-06-07 07:35:20.000000", "2021-06-07 14:35:20.000000"]) }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dates.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dates.sql new file mode 100644 index 0000000000..00b4e12af0 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dates.sql @@ -0,0 +1,2 @@ +{{ config(materialized="table") }} +{{ dbt_date.get_date_dimension("2015-01-01", "2022-12-31") }} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date.sql new file mode 100644 index 0000000000..a1f6fe4f56 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date.sql @@ -0,0 +1,19 @@ +{{ config(materialized="table") }} +with + date_dimension as (select * from {{ ref("dates") }}), + fiscal_periods as ( + {{ + dbt_date.get_fiscal_periods( + ref("dates"), year_end_month=1, week_start_day=1, shift_year=1 + ) + }} + ) +select + d.*, + f.fiscal_week_of_year, + f.fiscal_week_of_period, + f.fiscal_period_number, + f.fiscal_quarter_number, + f.fiscal_period_of_quarter +from date_dimension d +left join fiscal_periods f on d.date_day = f.date_day diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date_fiscal.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date_fiscal.sql new file mode 100644 index 0000000000..f7b5fe3463 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_date_fiscal.sql @@ -0,0 +1,11 @@ +{{ config(materialized="table") }} +with + fp as ( + {{ + dbt_date.get_fiscal_periods( + ref("dates"), year_end_month=1, week_start_day=1 + ) + }} + ) +select f.* +from fp f diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_hour.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_hour.sql new file mode 100644 index 0000000000..3be8f92110 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_hour.sql @@ -0,0 +1,7 @@ +{{ config(materialized="table") }} +with + periods_hours as ( + {{ dbt_date.get_base_dates(n_dateparts=24 * 28, datepart="hour") }} + ) +select d.* +from periods_hours d diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_week.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_week.sql new file mode 100644 index 0000000000..1cb8899f3b --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/dim_week.sql @@ -0,0 +1,4 @@ +{{ config(materialized="table") }} +with periods_weeks as ({{ dbt_date.get_base_dates(n_dateparts=52, datepart="week") }}) +select d.* +from periods_weeks d diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.sql new file mode 100644 index 0000000000..9f19a571c4 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.sql @@ -0,0 +1 @@ +{{ config(materialized="table") }} {{ dbt_date_integration_tests.get_test_dates() }} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.yml new file mode 100644 index 0000000000..3ab3edd04c --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/models/test_dates.yml @@ -0,0 +1,75 @@ +version: 2 +models: + - name: test_dates + tests: + - expression_is_true: + expression: "prior_date_day = {{ dbt_date.yesterday('date_day') }}" + - expression_is_true: + expression: "next_date_day = {{ dbt_date.tomorrow('date_day') }}" + - expression_is_true: + expression: "day_name = {{ dbt_date.day_name('date_day', short=False) }}" + - expression_is_true: + expression: "day_name_short = {{ dbt_date.day_name('date_day', short=True) }}" + - expression_is_true: + expression: "day_of_month = {{ dbt_date.day_of_month('date_day') }}" + - expression_is_true: + expression: "day_of_week = {{ dbt_date.day_of_week('date_day', isoweek=False) }}" + - expression_is_true: + expression: "iso_day_of_week = {{ dbt_date.day_of_week('date_day', isoweek=True) }}" + - expression_is_true: + expression: "day_of_year = {{ dbt_date.day_of_year('date_day') }}" + + - expression_is_true: + expression: "week_start_date = {{ dbt_date.week_start('date_day') }}" + - expression_is_true: + expression: "week_end_date = {{ dbt_date.week_end('date_day') }}" + - expression_is_true: + expression: "week_of_year = {{ dbt_date.week_of_year('date_day') }}" + - expression_is_true: + expression: "iso_week_start_date = {{ dbt_date.iso_week_start('date_day') }}" + - expression_is_true: + expression: "iso_week_end_date = {{ dbt_date.iso_week_end('date_day') }}" + - expression_is_true: + expression: "iso_week_of_year = {{ dbt_date.iso_week_of_year('date_day') }}" + - expression_is_true: + expression: "month_number = {{ dbt_date.date_part('month', 'date_day') }}" + - expression_is_true: + expression: "month_name = {{ dbt_date.month_name('date_day', short=False) }}" + - expression_is_true: + expression: "month_name_short = {{ dbt_date.month_name('date_day', short=True) }}" + - expression_is_true: + expression: "time_stamp_utc = {{ dbt_date.from_unixtimestamp('unix_epoch') }}" + - expression_is_true: + expression: "unix_epoch = {{ dbt_date.to_unixtimestamp('time_stamp_utc') }}" + - expression_is_true: + expression: "time_stamp = {{ dbt_date.convert_timezone('time_stamp_utc') }}" + - expression_is_true: + expression: "time_stamp = {{ dbt_date.convert_timezone('time_stamp_utc', source_tz='UTC') }}" + # - expression_is_true: + # expression: "time_stamp_utc = {{ dbt_date.convert_timezone('time_stamp', source_tz='America/Los_Angeles', target_tz='UTC') }}" + # - expression_is_true: + # expression: "time_stamp = {{ dbt_date.convert_timezone('time_stamp', source_tz='America/Los_Angeles', target_tz='America/Los_Angeles') }}" + - expression_is_true: + expression: "rounded_timestamp = {{ dbt_date.round_timestamp('time_stamp') }}" + - expression_is_true: + expression: "rounded_timestamp_utc = {{ dbt_date.round_timestamp('time_stamp_utc') }}" + - expression_is_true: + expression: "last_month_number = {{ dbt_date.last_month_number() }}" + - expression_is_true: + expression: "last_month_name = {{ dbt_date.last_month_name(short=False) }}" + - expression_is_true: + expression: "last_month_name_short = {{ dbt_date.last_month_name(short=True) }}" + - expression_is_true: + expression: "next_month_number = {{ dbt_date.next_month_number() }}" + - expression_is_true: + expression: "next_month_name = {{ dbt_date.next_month_name(short=False) }}" + - expression_is_true: + expression: "next_month_name_short = {{ dbt_date.next_month_name(short=True) }}" + - expression_is_true: + expression: "datetime_date = cast('{{ dbt_date.date(1997, 9, 29) }}' as date)" + - expression_is_true: + expression: "datetime_datetime = cast('{{ dbt_date.datetime(1997, 9, 29, 6, 14) }}' as {{ dbt.type_timestamp() }})" + + columns: + - name: date_day + - name: prior_date_day diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/packages.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/packages.yml new file mode 100644 index 0000000000..4a6b9c1948 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/packages.yml @@ -0,0 +1,2 @@ +packages: + - local: ../ diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/test.sh b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/test.sh new file mode 100644 index 0000000000..790660c120 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/integration_tests/test.sh @@ -0,0 +1,14 @@ +if [[ $# -gt 0 ]] +then + +i=1; +for t in "$@" +do + + dbt build -t $t + +done + +else + echo "Please specify one or more targets as command-line arguments, i.e. test.sh bq snowflake" +fi diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/date_spine.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/date_spine.sql new file mode 100644 index 0000000000..f5eb93971b --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/date_spine.sql @@ -0,0 +1,82 @@ +{% macro get_intervals_between(start_date, end_date, datepart) -%} + {{ + return( + adapter.dispatch("get_intervals_between", "dbt_date")( + start_date, end_date, datepart + ) + ) + }} +{%- endmacro %} + +{% macro default__get_intervals_between(start_date, end_date, datepart) -%} + {%- call statement("get_intervals_between", fetch_result=True) %} + + select {{ dbt.datediff(start_date, end_date, datepart) }} + + {%- endcall -%} + + {%- set value_list = load_result("get_intervals_between") -%} + + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} {{ return(1) }} + {%- endif -%} + +{%- endmacro %} + + +{% macro date_spine(datepart, start_date, end_date) %} + {{ + return( + adapter.dispatch("date_spine", "dbt_date")(datepart, start_date, end_date) + ) + }} +{%- endmacro %} + +{% macro default__date_spine(datepart, start_date, end_date) %} + + {# call as follows: + +date_spine( + "day", + "to_date('01/01/2016', 'mm/dd/yyyy')", + "dbt.dateadd(week, 1, current_date)" +) #} + with + rawdata as ( + + {{ + dbt_date.generate_series( + dbt_date.get_intervals_between(start_date, end_date, datepart) + ) + }} + + ), + + all_periods as ( + + select + ( + {{ + dbt.dateadd( + datepart, + "(row_number() over (order by 1) - 1)", + start_date, + ) + }} + ) as date_{{ datepart }} + from rawdata + + ), + + filtered as ( + + select * from all_periods where date_{{ datepart }} <= {{ end_date }} + + ) + + select * + from filtered + +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/generate_series.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/generate_series.sql new file mode 100644 index 0000000000..343d1c6c81 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/generate_series.sql @@ -0,0 +1,55 @@ +{% macro get_powers_of_two(upper_bound) %} + {{ return(adapter.dispatch("get_powers_of_two", "dbt_date")(upper_bound)) }} +{% endmacro %} + +{% macro default__get_powers_of_two(upper_bound) %} + + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} + + {% for _ in range(1, 100) %} + {% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} + {% endfor %} + +{% endmacro %} + + +{% macro generate_series(upper_bound) %} + {{ return(adapter.dispatch("generate_series", "dbt_date")(upper_bound)) }} +{% endmacro %} + +{% macro default__generate_series(upper_bound) %} + + {% set n = dbt_date.get_powers_of_two(upper_bound) %} + + with + p as ( + select 0 as generated_number + union all + select 1 + ), + unioned as ( + + select + + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 as generated_number + + from + + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} + + ) + + select * + from unioned + where generated_number <= {{ upper_bound }} + order by generated_number + +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/modules_datetime.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/modules_datetime.sql new file mode 100644 index 0000000000..7e5065ea15 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/_utils/modules_datetime.sql @@ -0,0 +1,23 @@ +{% macro date(year, month, day) %} + {{ return(modules.datetime.date(year, month, day)) }} +{% endmacro %} + +{% macro datetime( + year, month, day, hour=0, minute=0, second=0, microsecond=0, tz=None +) %} + {% set tz = tz if tz else var("dbt_date:time_zone") %} + {{ + return( + modules.datetime.datetime( + year=year, + month=month, + day=day, + hour=hour, + minute=minute, + second=second, + microsecond=microsecond, + tzinfo=modules.pytz.timezone(tz), + ) + ) + }} +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/convert_timezone.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/convert_timezone.sql new file mode 100644 index 0000000000..01a381be0b --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/convert_timezone.sql @@ -0,0 +1,52 @@ +{%- macro convert_timezone(column, target_tz=None, source_tz=None) -%} + {%- set source_tz = "UTC" if not source_tz else source_tz -%} + {%- set target_tz = var("dbt_date:time_zone") if not target_tz else target_tz -%} + {{ adapter.dispatch("convert_timezone", "dbt_date")(column, target_tz, source_tz) }} +{%- endmacro -%} + +{% macro default__convert_timezone(column, target_tz, source_tz) -%} + convert_timezone( + '{{ source_tz }}', + '{{ target_tz }}', + cast({{ column }} as {{ dbt.type_timestamp() }}) + ) +{%- endmacro -%} + +{%- macro bigquery__convert_timezone(column, target_tz, source_tz=None) -%} + timestamp(datetime({{ column }}, '{{ target_tz}}')) +{%- endmacro -%} + +{% macro postgres__convert_timezone(column, target_tz, source_tz) -%} + cast( + cast({{ column }} as {{ dbt.type_timestamp() }}) + at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' + as {{ dbt.type_timestamp() }} + ) +{%- endmacro -%} + +{%- macro redshift__convert_timezone(column, target_tz, source_tz) -%} + {{ return(dbt_date.default__convert_timezone(column, target_tz, source_tz)) }} +{%- endmacro -%} + +{% macro duckdb__convert_timezone(column, target_tz, source_tz) -%} + {{ return(dbt_date.postgres__convert_timezone(column, target_tz, source_tz)) }} +{%- endmacro -%} + +{%- macro spark__convert_timezone(column, target_tz, source_tz) -%} + from_utc_timestamp( + to_utc_timestamp({{ column }}, '{{ source_tz }}'), '{{ target_tz }}' + ) +{%- endmacro -%} + +{%- macro trino__convert_timezone(column, target_tz, source_tz) -%} + cast( + ( + at_timezone( + with_timezone( + cast({{ column }} as {{ dbt.type_timestamp() }}), '{{ source_tz }}' + ), + '{{ target_tz }}' + ) + ) as {{ dbt.type_timestamp() }} + ) +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/date_part.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/date_part.sql new file mode 100644 index 0000000000..9e6b201c7e --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/date_part.sql @@ -0,0 +1,15 @@ +{% macro date_part(datepart, date) -%} + {{ adapter.dispatch("date_part", "dbt_date")(datepart, date) }} +{%- endmacro %} + +{% macro default__date_part(datepart, date) -%} + date_part('{{ datepart }}', {{ date }}) +{%- endmacro %} + +{% macro bigquery__date_part(datepart, date) -%} + extract({{ datepart }} from {{ date }}) +{%- endmacro %} + +{% macro trino__date_part(datepart, date) -%} + extract({{ datepart }} from {{ date }}) +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_name.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_name.sql new file mode 100644 index 0000000000..6cb91b5ccb --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_name.sql @@ -0,0 +1,56 @@ +{%- macro day_name(date, short=True) -%} + {{ adapter.dispatch("day_name", "dbt_date")(date, short) }} +{%- endmacro %} + +{%- macro default__day_name(date, short) -%} + {%- set f = "Dy" if short else "Day" -%} to_char({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro snowflake__day_name(date, short) -%} + {%- if short -%} dayname({{ date }}) + {%- else -%} + -- long version not implemented on Snowflake so we're doing it manually :/ + case + dayname({{ date }}) + when 'Mon' + then 'Monday' + when 'Tue' + then 'Tuesday' + when 'Wed' + then 'Wednesday' + when 'Thu' + then 'Thursday' + when 'Fri' + then 'Friday' + when 'Sat' + then 'Saturday' + when 'Sun' + then 'Sunday' + end + {%- endif -%} + +{%- endmacro %} + +{%- macro bigquery__day_name(date, short) -%} + {%- set f = "%a" if short else "%A" -%} + format_date('{{ f }}', cast({{ date }} as date)) +{%- endmacro %} + +{%- macro postgres__day_name(date, short) -%} + {# FM = Fill mode, which suppresses padding blanks #} + {%- set f = "FMDy" if short else "FMDay" -%} to_char({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro duckdb__day_name(date, short) -%} + {%- if short -%} substr(dayname({{ date }}), 1, 3) + {%- else -%} dayname({{ date }}) + {%- endif -%} +{%- endmacro %} + +{%- macro spark__day_name(date, short) -%} + {%- set f = "E" if short else "EEEE" -%} date_format({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro trino__day_name(date, short) -%} + {%- set f = "a" if short else "W" -%} date_format({{ date }}, '%{{ f }}') +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_month.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_month.sql new file mode 100644 index 0000000000..3028c3b860 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_month.sql @@ -0,0 +1,5 @@ +{%- macro day_of_month(date) -%} {{ dbt_date.date_part("day", date) }} {%- endmacro %} + +{%- macro redshift__day_of_month(date) -%} + cast({{ dbt_date.date_part("day", date) }} as {{ dbt.type_bigint() }}) +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_week.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_week.sql new file mode 100644 index 0000000000..62460fb3b0 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_week.sql @@ -0,0 +1,100 @@ +{%- macro day_of_week(date, isoweek=true) -%} + {{ adapter.dispatch("day_of_week", "dbt_date")(date, isoweek) }} +{%- endmacro %} + +{%- macro default__day_of_week(date, isoweek) -%} + + {%- set dow = dbt_date.date_part("dayofweek", date) -%} + + {%- if isoweek -%} + case + -- Shift start of week from Sunday (0) to Monday (1) + when {{ dow }} = 0 then 7 else {{ dow }} + end + {%- else -%} {{ dow }} + 1 + {%- endif -%} + +{%- endmacro %} + +{%- macro snowflake__day_of_week(date, isoweek) -%} + + {%- if isoweek -%} + {%- set dow_part = "dayofweekiso" -%} {{ dbt_date.date_part(dow_part, date) }} + {%- else -%} + {%- set dow_part = "dayofweek" -%} + case + when {{ dbt_date.date_part(dow_part, date) }} = 7 + then 1 + else {{ dbt_date.date_part(dow_part, date) }} + 1 + end + {%- endif -%} + +{%- endmacro %} + +{%- macro bigquery__day_of_week(date, isoweek) -%} + + {%- set dow = dbt_date.date_part("dayofweek", date) -%} + + {%- if isoweek -%} + case + -- Shift start of week from Sunday (1) to Monday (2) + when {{ dow }} = 1 then 7 else {{ dow }} - 1 + end + {%- else -%} {{ dow }} + {%- endif -%} + +{%- endmacro %} + + +{%- macro postgres__day_of_week(date, isoweek) -%} + + {%- if isoweek -%} + {%- set dow_part = "isodow" -%} + -- Monday(1) to Sunday (7) + cast({{ dbt_date.date_part(dow_part, date) }} as {{ dbt.type_int() }}) + {%- else -%} + {%- set dow_part = "dow" -%} + -- Sunday(1) to Saturday (7) + cast({{ dbt_date.date_part(dow_part, date) }} + 1 as {{ dbt.type_int() }}) + {%- endif -%} + +{%- endmacro %} + + +{%- macro redshift__day_of_week(date, isoweek) -%} + + {%- set dow = dbt_date.date_part("dayofweek", date) -%} + + {%- if isoweek -%} + case + -- Shift start of week from Sunday (0) to Monday (1) + when {{ dow }} = 0 then 7 else cast({{ dow }} as {{ dbt.type_bigint() }}) + end + {%- else -%} cast({{ dow }} + 1 as {{ dbt.type_bigint() }}) + {%- endif -%} + +{%- endmacro %} + +{%- macro duckdb__day_of_week(date, isoweek) -%} + {{ return(dbt_date.postgres__day_of_week(date, isoweek)) }} +{%- endmacro %} + + +{%- macro spark__day_of_week(date, isoweek) -%} + + {%- set dow = "dayofweek_iso" if isoweek else "dayofweek" -%} + + {{ dbt_date.date_part(dow, date) }} + +{%- endmacro %} + + +{%- macro trino__day_of_week(date, isoweek) -%} + + {%- set dow = dbt_date.date_part("day_of_week", date) -%} + + {%- if isoweek -%} {{ dow }} + {%- else -%} case when {{ dow }} = 7 then 1 else {{ dow }} + 1 end + {%- endif -%} + +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_year.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_year.sql new file mode 100644 index 0000000000..5ddb45f002 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/day_of_year.sql @@ -0,0 +1,21 @@ +{%- macro day_of_year(date) -%} + {{ adapter.dispatch("day_of_year", "dbt_date")(date) }} +{%- endmacro %} + +{%- macro default__day_of_year(date) -%} + {{ dbt_date.date_part("dayofyear", date) }} +{%- endmacro %} + +{%- macro postgres__day_of_year(date) -%} + {{ dbt_date.date_part("doy", date) }} +{%- endmacro %} + +{%- macro redshift__day_of_year(date) -%} + cast({{ dbt_date.date_part("dayofyear", date) }} as {{ dbt.type_bigint() }}) +{%- endmacro %} + +{%- macro spark__day_of_year(date) -%} dayofyear({{ date }}) {%- endmacro %} + +{%- macro trino__day_of_year(date) -%} + {{ dbt_date.date_part("day_of_year", date) }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/from_unixtimestamp.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/from_unixtimestamp.sql new file mode 100644 index 0000000000..b08545c526 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/from_unixtimestamp.sql @@ -0,0 +1,110 @@ +{%- macro from_unixtimestamp(epochs, format="seconds") -%} + {{ adapter.dispatch("from_unixtimestamp", "dbt_date")(epochs, format) }} +{%- endmacro %} + +{%- macro default__from_unixtimestamp(epochs, format="seconds") -%} + {%- if format != "seconds" -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} + to_timestamp({{ epochs }}) +{%- endmacro %} + +{%- macro postgres__from_unixtimestamp(epochs, format="seconds") -%} + {%- if format != "seconds" -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} + cast(to_timestamp({{ epochs }}) at time zone 'UTC' as timestamp) +{%- endmacro %} + +{%- macro snowflake__from_unixtimestamp(epochs, format) -%} + {%- if format == "seconds" -%} {%- set scale = 0 -%} + {%- elif format == "milliseconds" -%} {%- set scale = 3 -%} + {%- elif format == "microseconds" -%} {%- set scale = 6 -%} + {%- else -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} + to_timestamp_ntz({{ epochs }}, {{ scale }}) + +{%- endmacro %} + +{%- macro bigquery__from_unixtimestamp(epochs, format) -%} + {%- if format == "seconds" -%} timestamp_seconds({{ epochs }}) + {%- elif format == "milliseconds" -%} timestamp_millis({{ epochs }}) + {%- elif format == "microseconds" -%} timestamp_micros({{ epochs }}) + {%- else -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} +{%- endmacro %} + +{%- macro trino__from_unixtimestamp(epochs, format) -%} + {%- if format == "seconds" -%} + cast( + from_unixtime({{ epochs }}) at time zone 'UTC' as {{ dbt.type_timestamp() }} + ) + {%- elif format == "milliseconds" -%} + cast( + from_unixtime_nanos( + {{ epochs }} * pow(10, 6) + ) at time zone 'UTC' as {{ dbt.type_timestamp() }} + ) + {%- elif format == "microseconds" -%} + cast( + from_unixtime_nanos( + {{ epochs }} * pow(10, 3) + ) at time zone 'UTC' as {{ dbt.type_timestamp() }} + ) + {%- elif format == "nanoseconds" -%} + cast( + from_unixtime_nanos( + {{ epochs }} + ) at time zone 'UTC' as {{ dbt.type_timestamp() }} + ) + {%- else -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} + +{%- endmacro %} + + +{%- macro duckdb__from_unixtimestamp(epochs, format="seconds") -%} + {%- if format != "seconds" -%} + {{ + exceptions.raise_compiler_error( + "value " + ~ format + ~ " for `format` for from_unixtimestamp is not supported." + ) + }} + {% endif -%} + cast(to_timestamp({{ epochs }}) at time zone 'UTC' as timestamp) +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_end.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_end.sql new file mode 100644 index 0000000000..5598369457 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_end.sql @@ -0,0 +1,16 @@ +{%- macro iso_week_end(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("iso_week_end", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro _iso_week_end(date, week_type) -%} + {%- set dt = dbt_date.iso_week_start(date) -%} {{ dbt_date.n_days_away(6, dt) }} +{%- endmacro %} + +{%- macro default__iso_week_end(date) -%} + {{ dbt_date._iso_week_end(date, "isoweek") }} +{%- endmacro %} + +{%- macro snowflake__iso_week_end(date) -%} + {{ dbt_date._iso_week_end(date, "weekiso") }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_of_year.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_of_year.sql new file mode 100644 index 0000000000..89f281c101 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_of_year.sql @@ -0,0 +1,34 @@ +{%- macro iso_week_of_year(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("iso_week_of_year", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro _iso_week_of_year(date, week_type) -%} + cast({{ dbt_date.date_part(week_type, date) }} as {{ dbt.type_int() }}) +{%- endmacro %} + +{%- macro default__iso_week_of_year(date) -%} + {{ dbt_date._iso_week_of_year(date, "isoweek") }} +{%- endmacro %} + +{%- macro snowflake__iso_week_of_year(date) -%} + {{ dbt_date._iso_week_of_year(date, "weekiso") }} +{%- endmacro %} + +{%- macro postgres__iso_week_of_year(date) -%} + -- postgresql week is isoweek, the first week of a year containing January 4 of + -- that year. + {{ dbt_date._iso_week_of_year(date, "week") }} +{%- endmacro %} + +{%- macro duckdb__iso_week_of_year(date) -%} + {{ return(dbt_date.postgres__iso_week_of_year(date)) }} +{%- endmacro %} + +{%- macro spark__iso_week_of_year(date) -%} + {{ dbt_date._iso_week_of_year(date, "week") }} +{%- endmacro %} + +{%- macro trino__iso_week_of_year(date) -%} + {{ dbt_date._iso_week_of_year(date, "week") }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_start.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_start.sql new file mode 100644 index 0000000000..871086fada --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/iso_week_start.sql @@ -0,0 +1,32 @@ +{%- macro iso_week_start(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("iso_week_start", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro _iso_week_start(date, week_type) -%} + cast({{ dbt.date_trunc(week_type, date) }} as date) +{%- endmacro %} + +{%- macro default__iso_week_start(date) -%} + {{ dbt_date._iso_week_start(date, "isoweek") }} +{%- endmacro %} + +{%- macro snowflake__iso_week_start(date) -%} + {{ dbt_date._iso_week_start(date, "week") }} +{%- endmacro %} + +{%- macro postgres__iso_week_start(date) -%} + {{ dbt_date._iso_week_start(date, "week") }} +{%- endmacro %} + +{%- macro duckdb__iso_week_start(date) -%} + {{ return(dbt_date.postgres__iso_week_start(date)) }} +{%- endmacro %} + +{%- macro spark__iso_week_start(date) -%} + {{ dbt_date._iso_week_start(date, "week") }} +{%- endmacro %} + +{%- macro trino__iso_week_start(date) -%} + {{ dbt_date._iso_week_start(date, "week") }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month.sql new file mode 100644 index 0000000000..f14370fa26 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month.sql @@ -0,0 +1 @@ +{%- macro last_month(tz=None) -%} {{ dbt_date.n_months_ago(1, tz) }} {%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_name.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_name.sql new file mode 100644 index 0000000000..6f7a10164a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_name.sql @@ -0,0 +1,3 @@ +{%- macro last_month_name(short=True, tz=None) -%} + {{ dbt_date.month_name(dbt_date.last_month(tz), short=short) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_number.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_number.sql new file mode 100644 index 0000000000..32fef4dd9e --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_month_number.sql @@ -0,0 +1,3 @@ +{%- macro last_month_number(tz=None) -%} + {{ dbt_date.date_part("month", dbt_date.last_month(tz)) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_week.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_week.sql new file mode 100644 index 0000000000..9cfa3675c3 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/last_week.sql @@ -0,0 +1 @@ +{%- macro last_week(tz=None) -%} {{ dbt_date.n_weeks_ago(1, tz) }} {%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/month_name.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/month_name.sql new file mode 100644 index 0000000000..4ae4452d68 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/month_name.sql @@ -0,0 +1,36 @@ +{%- macro month_name(date, short=True) -%} + {{ adapter.dispatch("month_name", "dbt_date")(date, short) }} +{%- endmacro %} + +{%- macro default__month_name(date, short) -%} + {%- set f = "MON" if short else "MONTH" -%} to_char({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro bigquery__month_name(date, short) -%} + {%- set f = "%b" if short else "%B" -%} + format_date('{{ f }}', cast({{ date }} as date)) +{%- endmacro %} + +{%- macro snowflake__month_name(date, short) -%} + {%- set f = "MON" if short else "MMMM" -%} to_char({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro postgres__month_name(date, short) -%} + {# FM = Fill mode, which suppresses padding blanks #} + {%- set f = "FMMon" if short else "FMMonth" -%} to_char({{ date }}, '{{ f }}') +{%- endmacro %} + + +{%- macro duckdb__month_name(date, short) -%} + {%- if short -%} substr(monthname({{ date }}), 1, 3) + {%- else -%} monthname({{ date }}) + {%- endif -%} +{%- endmacro %} + +{%- macro spark__month_name(date, short) -%} + {%- set f = "MMM" if short else "MMMM" -%} date_format({{ date }}, '{{ f }}') +{%- endmacro %} + +{%- macro trino__month_name(date, short) -%} + {%- set f = "b" if short else "M" -%} date_format({{ date }}, '%{{ f }}') +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_ago.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_ago.sql new file mode 100644 index 0000000000..807e7117cd --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_ago.sql @@ -0,0 +1,5 @@ +{%- macro n_days_ago(n, date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {%- set n = n | int -%} + cast({{ dbt.dateadd("day", -1 * n, dt) }} as date) +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_away.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_away.sql new file mode 100644 index 0000000000..5308fef130 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_days_away.sql @@ -0,0 +1,3 @@ +{%- macro n_days_away(n, date=None, tz=None) -%} + {{ dbt_date.n_days_ago(-1 * n, date, tz) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_ago.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_ago.sql new file mode 100644 index 0000000000..d95b86a591 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_ago.sql @@ -0,0 +1,4 @@ +{%- macro n_months_ago(n, tz=None) -%} + {%- set n = n | int -%} + {{ dbt.date_trunc("month", dbt.dateadd("month", -1 * n, dbt_date.today(tz))) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_away.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_away.sql new file mode 100644 index 0000000000..15aaf932f2 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_months_away.sql @@ -0,0 +1,4 @@ +{%- macro n_months_away(n, tz=None) -%} + {%- set n = n | int -%} + {{ dbt.date_trunc("month", dbt.dateadd("month", n, dbt_date.today(tz))) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_ago.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_ago.sql new file mode 100644 index 0000000000..d6c9675465 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_ago.sql @@ -0,0 +1,4 @@ +{%- macro n_weeks_ago(n, tz=None) -%} + {%- set n = n | int -%} + {{ dbt.date_trunc("week", dbt.dateadd("week", -1 * n, dbt_date.today(tz))) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_away.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_away.sql new file mode 100644 index 0000000000..3e573c907c --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/n_weeks_away.sql @@ -0,0 +1,4 @@ +{%- macro n_weeks_away(n, tz=None) -%} + {%- set n = n | int -%} + {{ dbt.date_trunc("week", dbt.dateadd("week", n, dbt_date.today(tz))) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month.sql new file mode 100644 index 0000000000..a7ce7af39f --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month.sql @@ -0,0 +1 @@ +{%- macro next_month(tz=None) -%} {{ dbt_date.n_months_away(1, tz) }} {%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_name.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_name.sql new file mode 100644 index 0000000000..fc1ecfdcf6 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_name.sql @@ -0,0 +1,3 @@ +{%- macro next_month_name(short=True, tz=None) -%} + {{ dbt_date.month_name(dbt_date.next_month(tz), short=short) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_number.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_number.sql new file mode 100644 index 0000000000..87bffb048d --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_month_number.sql @@ -0,0 +1,3 @@ +{%- macro next_month_number(tz=None) -%} + {{ dbt_date.date_part("month", dbt_date.next_month(tz)) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_week.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_week.sql new file mode 100644 index 0000000000..a4d58b9f63 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/next_week.sql @@ -0,0 +1 @@ +{%- macro next_week(tz=None) -%} {{ dbt_date.n_weeks_away(1, tz) }} {%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/now.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/now.sql new file mode 100644 index 0000000000..432b7d6d47 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/now.sql @@ -0,0 +1,3 @@ +{%- macro now(tz=None) -%} + {{ dbt_date.convert_timezone(dbt.current_timestamp(), tz) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/periods_since.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/periods_since.sql new file mode 100644 index 0000000000..f98fec069a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/periods_since.sql @@ -0,0 +1,3 @@ +{%- macro periods_since(date_col, period_name="day", tz=None) -%} + {{ dbt.datediff(date_col, dbt_date.now(tz), period_name) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/round_timestamp.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/round_timestamp.sql new file mode 100644 index 0000000000..7fd17fdd48 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/round_timestamp.sql @@ -0,0 +1,3 @@ +{% macro round_timestamp(timestamp) %} + {{ dbt.date_trunc("day", dbt.dateadd("hour", 12, timestamp)) }} +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/to_unixtimestamp.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/to_unixtimestamp.sql new file mode 100644 index 0000000000..1f61cd1963 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/to_unixtimestamp.sql @@ -0,0 +1,23 @@ +{%- macro to_unixtimestamp(timestamp) -%} + {{ adapter.dispatch("to_unixtimestamp", "dbt_date")(timestamp) }} +{%- endmacro %} + +{%- macro default__to_unixtimestamp(timestamp) -%} + {{ dbt_date.date_part("epoch", timestamp) }} +{%- endmacro %} + +{%- macro snowflake__to_unixtimestamp(timestamp) -%} + {{ dbt_date.date_part("epoch_seconds", timestamp) }} +{%- endmacro %} + +{%- macro bigquery__to_unixtimestamp(timestamp) -%} + unix_seconds({{ timestamp }}) +{%- endmacro %} + +{%- macro spark__to_unixtimestamp(timestamp) -%} + unix_timestamp({{ timestamp }}) +{%- endmacro %} + +{%- macro trino__to_unixtimestamp(timestamp) -%} + to_unixtime({{ timestamp }} at time zone 'UTC') +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/today.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/today.sql new file mode 100644 index 0000000000..db38bc5af5 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/today.sql @@ -0,0 +1 @@ +{%- macro today(tz=None) -%} cast({{ dbt_date.now(tz) }} as date) {%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/tomorrow.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/tomorrow.sql new file mode 100644 index 0000000000..a81d273e3a --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/tomorrow.sql @@ -0,0 +1,3 @@ +{%- macro tomorrow(date=None, tz=None) -%} + {{ dbt_date.n_days_away(1, date, tz) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_end.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_end.sql new file mode 100644 index 0000000000..abf728b2ed --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_end.sql @@ -0,0 +1,18 @@ +{%- macro week_end(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("week_end", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro default__week_end(date) -%} {{ last_day(date, "week") }} {%- endmacro %} + +{%- macro snowflake__week_end(date) -%} + {%- set dt = dbt_date.week_start(date) -%} {{ dbt_date.n_days_away(6, dt) }} +{%- endmacro %} + +{%- macro postgres__week_end(date) -%} + {%- set dt = dbt_date.week_start(date) -%} {{ dbt_date.n_days_away(6, dt) }} +{%- endmacro %} + +{%- macro duckdb__week_end(date) -%} + {{ return(dbt_date.postgres__week_end(date)) }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_of_year.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_of_year.sql new file mode 100644 index 0000000000..e9c50551f6 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_of_year.sql @@ -0,0 +1,23 @@ +{%- macro week_of_year(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("week_of_year", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro default__week_of_year(date) -%} + cast({{ dbt_date.date_part("week", date) }} as {{ dbt.type_int() }}) +{%- endmacro %} + +{%- macro postgres__week_of_year(date) -%} + {# postgresql 'week' returns isoweek. Use to_char instead. + WW = the first week starts on the first day of the year #} + cast(to_char({{ date }}, 'WW') as {{ dbt.type_int() }}) +{%- endmacro %} + +{%- macro duckdb__week_of_year(date) -%} + cast(ceil(dayofyear({{ date }}) / 7) as int) +{%- endmacro %} + +{# {%- macro spark__week_of_year(date) -%} +weekofyear({{ date }}) +{%- endmacro %} #} + diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_start.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_start.sql new file mode 100644 index 0000000000..4f45d6bf07 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/week_start.sql @@ -0,0 +1,32 @@ +{%- macro week_start(date=None, tz=None) -%} + {%- set dt = date if date else dbt_date.today(tz) -%} + {{ adapter.dispatch("week_start", "dbt_date")(dt) }} +{%- endmacro -%} + +{%- macro default__week_start(date) -%} + cast({{ dbt.date_trunc("week", date) }} as date) +{%- endmacro %} + +{%- macro snowflake__week_start(date) -%} + {# + Get the day of week offset: e.g. if the date is a Sunday, + dbt_date.day_of_week returns 1, so we subtract 1 to get a 0 offset + #} + {% set off_set = dbt_date.day_of_week(date, isoweek=False) ~ " - 1" %} + cast({{ dbt.dateadd("day", "-1 * (" ~ off_set ~ ")", date) }} as date) +{%- endmacro %} + +{%- macro postgres__week_start(date) -%} + -- Sunday as week start date + cast( + {{ + dbt.dateadd( + "day", -1, dbt.date_trunc("week", dbt.dateadd("day", 1, date)) + ) + }} as date + ) +{%- endmacro %} + +{%- macro duckdb__week_start(date) -%} + {{ return(dbt_date.postgres__week_start(date)) }} +{%- endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/yesterday.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/yesterday.sql new file mode 100644 index 0000000000..3c55968cdb --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/calendar_date/yesterday.sql @@ -0,0 +1,3 @@ +{%- macro yesterday(date=None, tz=None) -%} + {{ dbt_date.n_days_ago(1, date, tz) }} +{%- endmacro -%} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_periods.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_periods.sql new file mode 100644 index 0000000000..c0f6cf1385 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_periods.sql @@ -0,0 +1,81 @@ +{% macro get_fiscal_periods(dates, year_end_month, week_start_day, shift_year=1) %} + {# +This macro requires you to pass in a ref to a date dimension, created via +dbt_date.get_date_dimension()s +#} + with + fscl_year_dates_for_periods as ( + {{ + dbt_date.get_fiscal_year_dates( + dates, year_end_month, week_start_day, shift_year + ) + }} + ), + fscl_year_w13 as ( + + select + f.*, + -- We count the weeks in a 13 week period + -- and separate the 4-5-4 week sequences + mod( + cast((f.fiscal_week_of_year - 1) as {{ dbt.type_int() }}), 13 + ) as w13_number, + -- Chop weeks into 13 week merch quarters + cast( + least( + floor((f.fiscal_week_of_year - 1) / 13.0), 3 + ) as {{ dbt.type_int() }} + ) as quarter_number + from fscl_year_dates_for_periods f + + ), + fscl_periods as ( + + select + f.date_day, + f.fiscal_year_number, + f.week_start_date, + f.week_end_date, + f.fiscal_week_of_year, + case + -- we move week 53 into the 3rd period of the quarter + when f.fiscal_week_of_year = 53 + then 3 + when f.w13_number between 0 and 3 + then 1 + when f.w13_number between 4 and 8 + then 2 + when f.w13_number between 9 and 12 + then 3 + end as period_of_quarter, + f.quarter_number + from fscl_year_w13 f + + ), + fscl_periods_quarters as ( + + select + f.*, + cast( + ( + (f.quarter_number * 3) + f.period_of_quarter + ) as {{ dbt.type_int() }} + ) as fiscal_period_number + from fscl_periods f + + ) + select + date_day, + fiscal_year_number, + week_start_date, + week_end_date, + fiscal_week_of_year, + dense_rank() over ( + partition by fiscal_period_number order by fiscal_week_of_year + ) as fiscal_week_of_period, + fiscal_period_number, + quarter_number + 1 as fiscal_quarter_number, + period_of_quarter as fiscal_period_of_quarter + from fscl_periods_quarters + order by 1, 2 +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_year_dates.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_year_dates.sql new file mode 100644 index 0000000000..3fe83c9295 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/fiscal_date/get_fiscal_year_dates.sql @@ -0,0 +1,114 @@ +{% macro get_fiscal_year_dates( + dates, year_end_month=12, week_start_day=1, shift_year=1 +) %} + {{ + adapter.dispatch("get_fiscal_year_dates", "dbt_date")( + dates, year_end_month, week_start_day, shift_year + ) + }} +{% endmacro %} + +{% macro default__get_fiscal_year_dates( + dates, year_end_month, week_start_day, shift_year +) %} + -- this gets all the dates within a fiscal year + -- determined by the given year-end-month + -- ending on the saturday closest to that month's end date + with + fsc_date_dimension as (select * from {{ dates }}), + year_month_end as ( + + select + d.year_number - {{ shift_year }} as fiscal_year_number, d.month_end_date + from fsc_date_dimension d + where d.month_of_year = {{ year_end_month }} + group by 1, 2 + + ), + weeks as ( + + select + d.year_number, + d.month_of_year, + d.date_day as week_start_date, + cast({{ dbt.dateadd("day", 6, "d.date_day") }} as date) as week_end_date + from fsc_date_dimension d + where d.day_of_week = {{ week_start_day }} + + ), + -- get all the weeks that start in the month the year ends + year_week_ends as ( + + select + d.year_number - {{ shift_year }} as fiscal_year_number, d.week_end_date + from weeks d + where d.month_of_year = {{ year_end_month }} + group by 1, 2 + + ), + -- then calculate which Saturday is closest to month end + weeks_at_month_end as ( + + select + d.fiscal_year_number, + d.week_end_date, + m.month_end_date, + rank() over ( + partition by d.fiscal_year_number + order by + abs( + {{ + dbt.datediff( + "d.week_end_date", "m.month_end_date", "day" + ) + }} + ) + + ) as closest_to_month_end + from year_week_ends d + join year_month_end m on d.fiscal_year_number = m.fiscal_year_number + ), + fiscal_year_range as ( + + select + w.fiscal_year_number, + cast( + {{ + dbt.dateadd( + "day", + 1, + "lag(w.week_end_date) over(order by w.week_end_date)", + ) + }} as date + ) as fiscal_year_start_date, + w.week_end_date as fiscal_year_end_date + from weeks_at_month_end w + where w.closest_to_month_end = 1 + + ), + fiscal_year_dates as ( + + select + d.date_day, + m.fiscal_year_number, + m.fiscal_year_start_date, + m.fiscal_year_end_date, + w.week_start_date, + w.week_end_date, + -- we reset the weeks of the year starting with the merch year start + -- date + dense_rank() over ( + partition by m.fiscal_year_number order by w.week_start_date + ) as fiscal_week_of_year + from fsc_date_dimension d + join + fiscal_year_range m + on d.date_day + between m.fiscal_year_start_date and m.fiscal_year_end_date + join weeks w on d.date_day between w.week_start_date and w.week_end_date + + ) + select * + from fiscal_year_dates + order by 1 +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_base_dates.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_base_dates.sql new file mode 100644 index 0000000000..cba16be4c5 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_base_dates.sql @@ -0,0 +1,109 @@ +{% macro get_base_dates( + start_date=None, end_date=None, n_dateparts=None, datepart="day" +) %} + {{ + adapter.dispatch("get_base_dates", "dbt_date")( + start_date, end_date, n_dateparts, datepart + ) + }} +{% endmacro %} + +{% macro default__get_base_dates(start_date, end_date, n_dateparts, datepart) %} + + {%- if start_date and end_date -%} + {%- set start_date = ( + "cast('" ~ start_date ~ "' as " ~ dbt.type_timestamp() ~ ")" + ) -%} + {%- set end_date = ( + "cast('" ~ end_date ~ "' as " ~ dbt.type_timestamp() ~ ")" + ) -%} + + {%- elif n_dateparts and datepart -%} + + {%- set start_date = dbt.dateadd( + datepart, -1 * n_dateparts, dbt_date.today() + ) -%} + {%- set end_date = dbt_date.tomorrow() -%} + {%- endif -%} + + with + date_spine as ( + + {{ + dbt_date.date_spine( + datepart=datepart, + start_date=start_date, + end_date=end_date, + ) + }} + + ) + select + cast(d.date_{{ datepart }} as {{ dbt.type_timestamp() }}) as date_{{ datepart }} + from date_spine d +{% endmacro %} + +{% macro bigquery__get_base_dates(start_date, end_date, n_dateparts, datepart) %} + + {%- if start_date and end_date -%} + {%- set start_date = "cast('" ~ start_date ~ "' as datetime )" -%} + {%- set end_date = "cast('" ~ end_date ~ "' as datetime )" -%} + + {%- elif n_dateparts and datepart -%} + + {%- set start_date = dbt.dateadd( + datepart, -1 * n_dateparts, dbt_date.today() + ) -%} + {%- set end_date = dbt_date.tomorrow() -%} + {%- endif -%} + + with + date_spine as ( + + {{ + dbt_date.date_spine( + datepart=datepart, + start_date=start_date, + end_date=end_date, + ) + }} + + ) + select + cast(d.date_{{ datepart }} as {{ dbt.type_timestamp() }}) as date_{{ datepart }} + from date_spine d +{% endmacro %} + + +{% macro trino__get_base_dates(start_date, end_date, n_dateparts, datepart) %} + + {%- if start_date and end_date -%} + {%- set start_date = ( + "cast('" ~ start_date ~ "' as " ~ dbt.type_timestamp() ~ ")" + ) -%} + {%- set end_date = ( + "cast('" ~ end_date ~ "' as " ~ dbt.type_timestamp() ~ ")" + ) -%} + + {%- elif n_dateparts and datepart -%} + + {%- set start_date = dbt.dateadd(datepart, -1 * n_dateparts, dbt_date.now()) -%} + {%- set end_date = dbt_date.tomorrow() -%} + {%- endif -%} + + with + date_spine as ( + + {{ + dbt_date.date_spine( + datepart=datepart, + start_date=start_date, + end_date=end_date, + ) + }} + + ) + select + cast(d.date_{{ datepart }} as {{ dbt.type_timestamp() }}) as date_{{ datepart }} + from date_spine d +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_date_dimension.sql b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_date_dimension.sql new file mode 100644 index 0000000000..159c338dc7 --- /dev/null +++ b/dev/dags/dbt/simple/dbt_packages/dbt_date/macros/get_date_dimension.sql @@ -0,0 +1,180 @@ +{% macro get_date_dimension(start_date, end_date) %} + {{ adapter.dispatch("get_date_dimension", "dbt_date")(start_date, end_date) }} +{% endmacro %} + +{% macro default__get_date_dimension(start_date, end_date) %} + with + base_dates as ({{ dbt_date.get_base_dates(start_date, end_date) }}), + dates_with_prior_year_dates as ( + + select + cast(d.date_day as date) as date_day, + cast( + {{ dbt.dateadd("year", -1, "d.date_day") }} as date + ) as prior_year_date_day, + cast( + {{ dbt.dateadd("day", -364, "d.date_day") }} as date + ) as prior_year_over_year_date_day + from base_dates d + + ) + select + d.date_day, + {{ dbt_date.yesterday("d.date_day") }} as prior_date_day, + {{ dbt_date.tomorrow("d.date_day") }} as next_date_day, + d.prior_year_date_day as prior_year_date_day, + d.prior_year_over_year_date_day, + {{ dbt_date.day_of_week("d.date_day", isoweek=false) }} as day_of_week, + {{ dbt_date.day_of_week("d.date_day", isoweek=true) }} as day_of_week_iso, + {{ dbt_date.day_name("d.date_day", short=false) }} as day_of_week_name, + {{ dbt_date.day_name("d.date_day", short=true) }} as day_of_week_name_short, + {{ dbt_date.day_of_month("d.date_day") }} as day_of_month, + {{ dbt_date.day_of_year("d.date_day") }} as day_of_year, + + {{ dbt_date.week_start("d.date_day") }} as week_start_date, + {{ dbt_date.week_end("d.date_day") }} as week_end_date, + {{ dbt_date.week_start("d.prior_year_over_year_date_day") }} + as prior_year_week_start_date, + {{ dbt_date.week_end("d.prior_year_over_year_date_day") }} + as prior_year_week_end_date, + {{ dbt_date.week_of_year("d.date_day") }} as week_of_year, + + {{ dbt_date.iso_week_start("d.date_day") }} as iso_week_start_date, + {{ dbt_date.iso_week_end("d.date_day") }} as iso_week_end_date, + {{ dbt_date.iso_week_start("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_start_date, + {{ dbt_date.iso_week_end("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_end_date, + {{ dbt_date.iso_week_of_year("d.date_day") }} as iso_week_of_year, + + {{ dbt_date.week_of_year("d.prior_year_over_year_date_day") }} + as prior_year_week_of_year, + {{ dbt_date.iso_week_of_year("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_of_year, + + cast( + {{ dbt_date.date_part("month", "d.date_day") }} as {{ dbt.type_int() }} + ) as month_of_year, + {{ dbt_date.month_name("d.date_day", short=false) }} as month_name, + {{ dbt_date.month_name("d.date_day", short=true) }} as month_name_short, + + cast({{ dbt.date_trunc("month", "d.date_day") }} as date) as month_start_date, + cast({{ last_day("d.date_day", "month") }} as date) as month_end_date, + + cast( + {{ dbt.date_trunc("month", "d.prior_year_date_day") }} as date + ) as prior_year_month_start_date, + cast( + {{ last_day("d.prior_year_date_day", "month") }} as date + ) as prior_year_month_end_date, + + cast( + {{ dbt_date.date_part("quarter", "d.date_day") }} as {{ dbt.type_int() }} + ) as quarter_of_year, + cast( + {{ dbt.date_trunc("quarter", "d.date_day") }} as date + ) as quarter_start_date, + cast({{ last_day("d.date_day", "quarter") }} as date) as quarter_end_date, + + cast( + {{ dbt_date.date_part("year", "d.date_day") }} as {{ dbt.type_int() }} + ) as year_number, + cast({{ dbt.date_trunc("year", "d.date_day") }} as date) as year_start_date, + cast({{ last_day("d.date_day", "year") }} as date) as year_end_date + from dates_with_prior_year_dates d + order by 1 +{% endmacro %} + +{% macro postgres__get_date_dimension(start_date, end_date) %} + with + base_dates as ({{ dbt_date.get_base_dates(start_date, end_date) }}), + dates_with_prior_year_dates as ( + + select + cast(d.date_day as date) as date_day, + cast( + {{ dbt.dateadd("year", -1, "d.date_day") }} as date + ) as prior_year_date_day, + cast( + {{ dbt.dateadd("day", -364, "d.date_day") }} as date + ) as prior_year_over_year_date_day + from base_dates d + + ) + select + d.date_day, + {{ dbt_date.yesterday("d.date_day") }} as prior_date_day, + {{ dbt_date.tomorrow("d.date_day") }} as next_date_day, + d.prior_year_date_day as prior_year_date_day, + d.prior_year_over_year_date_day, + {{ dbt_date.day_of_week("d.date_day", isoweek=true) }} as day_of_week, + + {{ dbt_date.day_name("d.date_day", short=false) }} as day_of_week_name, + {{ dbt_date.day_name("d.date_day", short=true) }} as day_of_week_name_short, + {{ dbt_date.day_of_month("d.date_day") }} as day_of_month, + {{ dbt_date.day_of_year("d.date_day") }} as day_of_year, + + {{ dbt_date.week_start("d.date_day") }} as week_start_date, + {{ dbt_date.week_end("d.date_day") }} as week_end_date, + {{ dbt_date.week_start("d.prior_year_over_year_date_day") }} + as prior_year_week_start_date, + {{ dbt_date.week_end("d.prior_year_over_year_date_day") }} + as prior_year_week_end_date, + {{ dbt_date.week_of_year("d.date_day") }} as week_of_year, + + {{ dbt_date.iso_week_start("d.date_day") }} as iso_week_start_date, + {{ dbt_date.iso_week_end("d.date_day") }} as iso_week_end_date, + {{ dbt_date.iso_week_start("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_start_date, + {{ dbt_date.iso_week_end("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_end_date, + {{ dbt_date.iso_week_of_year("d.date_day") }} as iso_week_of_year, + + {{ dbt_date.week_of_year("d.prior_year_over_year_date_day") }} + as prior_year_week_of_year, + {{ dbt_date.iso_week_of_year("d.prior_year_over_year_date_day") }} + as prior_year_iso_week_of_year, + + cast( + {{ dbt_date.date_part("month", "d.date_day") }} as {{ dbt.type_int() }} + ) as month_of_year, + {{ dbt_date.month_name("d.date_day", short=false) }} as month_name, + {{ dbt_date.month_name("d.date_day", short=true) }} as month_name_short, + + cast({{ dbt.date_trunc("month", "d.date_day") }} as date) as month_start_date, + cast({{ last_day("d.date_day", "month") }} as date) as month_end_date, + + cast( + {{ dbt.date_trunc("month", "d.prior_year_date_day") }} as date + ) as prior_year_month_start_date, + cast( + {{ last_day("d.prior_year_date_day", "month") }} as date + ) as prior_year_month_end_date, + + cast( + {{ dbt_date.date_part("quarter", "d.date_day") }} as {{ dbt.type_int() }} + ) as quarter_of_year, + cast( + {{ dbt.date_trunc("quarter", "d.date_day") }} as date + ) as quarter_start_date, + {# last_day does not support quarter because postgresql does not support quarter interval. #} + cast( + {{ + dbt.dateadd( + "day", + "-1", + dbt.dateadd( + "month", "3", dbt.date_trunc("quarter", "d.date_day") + ), + ) + }} as date + ) as quarter_end_date, + + cast( + {{ dbt_date.date_part("year", "d.date_day") }} as {{ dbt.type_int() }} + ) as year_number, + cast({{ dbt.date_trunc("year", "d.date_day") }} as date) as year_start_date, + cast({{ last_day("d.date_day", "year") }} as date) as year_end_date + from dates_with_prior_year_dates d + order by 1 +{% endmacro %} diff --git a/dev/dags/dbt/simple/dbt_packages/dbt_date/packages.yml b/dev/dags/dbt/simple/dbt_packages/dbt_date/packages.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev/dags/dbt/simple/package-lock.yml b/dev/dags/dbt/simple/package-lock.yml new file mode 100644 index 0000000000..9cf994ac02 --- /dev/null +++ b/dev/dags/dbt/simple/package-lock.yml @@ -0,0 +1,4 @@ +packages: + - package: godatadriven/dbt_date + version: 0.11.0 +sha1_hash: 1166423b7913acc5dbfae59492255eddd23a7db2 diff --git a/dev/dags/dbt/simple/packages.yml b/dev/dags/dbt/simple/packages.yml new file mode 100644 index 0000000000..9a7510e9c7 --- /dev/null +++ b/dev/dags/dbt/simple/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: godatadriven/dbt_date + version: 0.11.0