-
Notifications
You must be signed in to change notification settings - Fork 1.6k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
ADAP-387: Stub materialized view as a materialization #7211
Changes from all commits
9c40e87
19aec7b
cda0eb4
40379b3
37eb64f
65672a9
f794560
845d7a3
2b9faaa
158b9f9
3287562
eca26b8
f3faee2
e4c7005
53b4adc
88f6cdd
056a356
66754dc
0252ea3
fcc480d
345c2b2
caad12c
d6eee2d
60a5392
e9c423e
eeb0d3f
33a025b
703cff4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
{% macro db_api__materialized_view__create(relation, sql) %} | ||
{{ adapter.dispatch('db_api__materialized_view__create', 'dbt')(relation, sql) }} | ||
{% endmacro %} | ||
|
||
{% macro default__db_api__materialized_view__create(relation, sql) -%} | ||
|
||
{{ exceptions.raise_not_implemented( | ||
'db_api__materialized_view__create macro not implemented for adapter '+adapter.type()) }} | ||
|
||
{% endmacro %} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
{% macro db_api__materialized_view__refresh(relation) %} | ||
{{ adapter.dispatch('db_api__materialized_view__refresh', 'dbt')(relation) }} | ||
{% endmacro %} | ||
|
||
{% macro default__db_api__materialized_view__refresh(relation) -%} | ||
|
||
{{ exceptions.raise_not_implemented( | ||
'db_api__materialized_view__refresh not implemented for adapter '+adapter.type()) }} | ||
|
||
{% endmacro %} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
{%- materialization materialized_view, default -%} | ||
{% set full_refresh_mode = should_full_refresh() %} | ||
{%- set existing_relation = load_cached_relation(this) -%} | ||
{%- set target_relation = this.incorporate(type='view') -%} | ||
{%- set intermediate_relation = make_intermediate_relation(target_relation) -%} | ||
|
||
-- the intermediate_relation should not already exist in the database; get_relation | ||
-- will return None in that case. Otherwise, we get a relation that we can drop | ||
-- later, before we try to use this name for the current operation | ||
{%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} | ||
/* | ||
This relation (probably) doesn't exist yet. If it does exist, it's a leftover from | ||
a previous run, and we're going to try to drop it immediately. At the end of this | ||
materialization, we're going to rename the "existing_relation" to this identifier, | ||
and then we're going to drop it. In order to make sure we run the correct one of: | ||
- drop view ... | ||
- drop table ... | ||
We need to set the type of this relation to be the type of the existing_relation, if it exists, | ||
or else "view" as a sane default if it does not. Note that if the existing_relation does not | ||
exist, then there is nothing to move out of the way and subsequentally drop. In that case, | ||
this relation will be effectively unused. | ||
*/ | ||
{%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} | ||
{%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} | ||
-- as above, the backup_relation should not already exist | ||
{%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} | ||
-- grab current tables grants config for comparision later on | ||
{% set grant_config = config.get('grants') %} | ||
|
||
{{ run_hooks(pre_hooks, inside_transaction=False) }} | ||
|
||
-- drop the temp relations if they exist already in the database | ||
{{ drop_relation_if_exists(preexisting_intermediate_relation) }} | ||
{{ drop_relation_if_exists(preexisting_backup_relation) }} | ||
|
||
-- `BEGIN` happens here: | ||
{{ run_hooks(pre_hooks, inside_transaction=True) }} | ||
|
||
-- cleanup | ||
-- move the existing view out of the way | ||
{% if existing_relation is none %} | ||
{% set build_sql = strategy__materialized_view__create(target_relation, sql) %} | ||
{% elif full_refresh_mode or existing_relation.type != 'view' %} | ||
McKnight-42 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{% set build_sql = strategy__materialized_view__full_refresh(target_relation, sql, backup_relation, intermediate_relation) %} | ||
{% else %} | ||
{% set build_sql = strategy__materialized_view__full_refresh(target_relation) %} | ||
{% endif %} | ||
mikealfare marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
-- possible conditional wrapper for build_sql statement based on configs as a option? | ||
-- build model | ||
{% call statement("main") %} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We need to make sure this can actually exist outside of the if block above. For example, does |
||
{{ build_sql }} | ||
{% endcall %} | ||
|
||
{% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} | ||
{% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} | ||
|
||
{% do persist_docs(target_relation, model) %} | ||
|
||
{{ run_hooks(post_hooks, inside_transaction=True) }} | ||
|
||
{{ adapter.commit() }} | ||
|
||
{{ drop_relation_if_exists(backup_relation) }} | ||
McKnight-42 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{{ drop_relation_if_exists(intermediate_relation) }} | ||
|
||
{{ run_hooks(post_hooks, inside_transaction=False) }} | ||
|
||
{{ return({'relations': [target_relation]}) }} | ||
|
||
{%- endmaterialization -%} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
{% macro strategy__materialized_view__create(relation, sql) %} | ||
McKnight-42 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{{ adapter.dispatch('strategy__materialized_view__create', 'dbt')(relation, sql) }} | ||
{% endmacro %} | ||
|
||
{% macro default__strategy__materialized_view__create(relation, sql) %} | ||
{{ db_api__materialized_view__create(relation, sql) }} | ||
{% endmacro %} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
{% macro strategy__materialized_view__full_refresh(relation, sql, backup_relation=None, intermediate_relation=None) %} | ||
{{ adapter.dispatch('strategy__materialized_view__full_refresh', 'dbt')(relation, sql, backup_relation, intermediate_relation) }} | ||
{% endmacro %} | ||
|
||
|
||
{% macro default__strategy__materialized_view__full_refresh(relation, sql, backup_relation=None, intermediate_relation=None) %} | ||
{% if backup_relation %} | ||
{{ db_api__materialized_view__create(intermediate_relation, sql) }} | ||
{{ adapter.rename_relation(target_relation, backup_relation) }} | ||
{{ adapter.rename_relation(intermediate_relation, target_relation) }} | ||
{{ drop_relation_if_exists(backup_relation) }} | ||
{% else %} | ||
{{ drop_relation_if_exists(target_relation) }} | ||
{{ db_api__materialized_view__create(target_relation, sql) }} | ||
{% endif %} | ||
|
||
{% endmacro %} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
import pytest | ||
|
||
from dbt.tests.util import run_dbt | ||
|
||
|
||
models__model_sql = """ | ||
{{ config(materialized='materialized_view') }} | ||
select 1 as id | ||
|
||
""" | ||
|
||
|
||
@pytest.fixture(scope="class") | ||
def models(): | ||
return {"model.sql": models__model_sql} | ||
|
||
|
||
def test_basic(project): | ||
run_dbt(["run"], expect_pass=False) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Referencing @Fleid's direction here for convenience:
This is going to vary by adapter; but I'd like to avoid creating versions of this materialization in adapters if possible. Hence I think we should adopt something like the strategies approach that we did with the incremental materialization. In other words, we have "replace" strategy, an "alter" strategy, a "refresh" strategy, etc. In some cases, those strategies may point to the same macros (e.g. create_materialized_view_as()); but that would allow maintainers to update just the piece that is specific to that platform without needing to maintain the rest of the materialization.