From e085e98c2ebe81c71f7a7bdaa587f1cd5a763c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BC=8E=E6=98=95?= Date: Tue, 3 Dec 2024 20:00:27 +0800 Subject: [PATCH] =?UTF-8?q?feat(maxcompute):=20=E6=B7=BB=E5=8A=A0=E5=85=8B?= =?UTF-8?q?=E9=9A=86=E8=A1=A8=E5=8A=9F=E8=83=BD=E5=B9=B6=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E8=A1=A8=E5=88=9B=E5=BB=BA=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增克隆表功能，支持在 MaxCompute 中创建表的副本 - 优化表创建逻辑，移除不必要的 if not exists 判断 - 添加更新表注释和列注释的功能 - 新增测试用例以验证克隆表和注释功能 --- .idea/dbt-maxcompute.iml | 15 - dbt/adapters/maxcompute/__version__.py | 2 +- dbt/adapters/maxcompute/connections.py | 13 +- dbt/adapters/maxcompute/impl.py | 1 + dbt/adapters/maxcompute/wrapper.py | 7 +- .../macros/adapters/persist_docs.sql | 16 + .../macros/materializations/clone.sql | 8 + .../macros/materializations/hooks.sql | 10 + .../materializations/snapshots/snapshot.sql | 1 - .../macros/relations/table/create.sql | 9 +- .../macros/relations/view/create.sql | 2 +- tests/functional/adapter/data/seed_model.sql | 16 + tests/functional/adapter/data/seed_run.sql | 16 + tests/functional/adapter/test_catalog.py | 25 + tests/functional/adapter/test_dbt_clone.py | 27 + tests/functional/adapter/test_dbt_show.py | 17 + tests/functional/adapter/test_empty.py | 9 + tests/functional/adapter/test_ephemeral.py | 74 ++ tests/functional/adapter/test_grants.py | 9 + tests/functional/adapter/test_hooks.py | 656 ++++++++++++++++++ .../adapter/test_materialized_view.py | 9 + tests/functional/adapter/test_persist_docs.py | 47 ++ tests/functional/adapter/test_python_model.py | 11 + .../functional/adapter/test_query_comment.py | 32 + tests/functional/adapter/test_simple_copy.py | 30 + .../adapter/test_simple_snapshot.py | 145 ++++ .../adapter/test_store_test_failures.py | 7 + 27 files changed, 1186 insertions(+), 28 deletions(-) delete mode 100644 .idea/dbt-maxcompute.iml create mode 100644 dbt/include/maxcompute/macros/adapters/persist_docs.sql create mode 100644 dbt/include/maxcompute/macros/materializations/clone.sql create mode 100644 dbt/include/maxcompute/macros/materializations/hooks.sql create mode 100644 tests/functional/adapter/data/seed_model.sql create mode 100644 tests/functional/adapter/data/seed_run.sql create mode 100644 tests/functional/adapter/test_catalog.py create mode 100644 tests/functional/adapter/test_dbt_clone.py create mode 100644 tests/functional/adapter/test_dbt_show.py create mode 100644 tests/functional/adapter/test_empty.py create mode 100644 tests/functional/adapter/test_ephemeral.py create mode 100644 tests/functional/adapter/test_grants.py create mode 100644 tests/functional/adapter/test_hooks.py create mode 100644 tests/functional/adapter/test_materialized_view.py create mode 100644 tests/functional/adapter/test_persist_docs.py create mode 100644 tests/functional/adapter/test_python_model.py create mode 100644 tests/functional/adapter/test_query_comment.py create mode 100644 tests/functional/adapter/test_simple_copy.py create mode 100644 tests/functional/adapter/test_simple_snapshot.py create mode 100644 tests/functional/adapter/test_store_test_failures.py diff --git a/.idea/dbt-maxcompute.iml b/.idea/dbt-maxcompute.iml deleted file mode 100644 index fa73b3a..0000000 --- a/.idea/dbt-maxcompute.iml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - diff --git a/dbt/adapters/maxcompute/__version__.py b/dbt/adapters/maxcompute/__version__.py index db0f2db..72ebff9 100644 --- a/dbt/adapters/maxcompute/__version__.py +++ b/dbt/adapters/maxcompute/__version__.py @@ -1 +1 @@ -version = "1.8.0-alpha.9" +version = "1.8.0-dev" diff --git a/dbt/adapters/maxcompute/connections.py b/dbt/adapters/maxcompute/connections.py index 67fcc52..e22dd1a 100644 --- a/dbt/adapters/maxcompute/connections.py +++ b/dbt/adapters/maxcompute/connections.py @@ -1,6 +1,5 @@ from contextlib import contextmanager from dataclasses import dataclass -from typing import Dict, Any from dbt_common.exceptions import DbtConfigError, DbtRuntimeError from dbt.adapters.contracts.connection import Credentials, AdapterResponse @@ -77,10 +76,6 @@ def get_response(cls, cursor): logger.debug("Current instance id is " + cursor._instance.id) return AdapterResponse(_message="OK") - def set_query_header(self, query_header_context: Dict[str, Any]) -> None: - # no query header will better - pass - @contextmanager def exception_handler(self, sql: str): try: @@ -100,6 +95,14 @@ def exception_handler(self, sql: str): def cancel(self, connection): connection.handle.cancel() + def begin(self): + logger.debug("Trigger beginning transaction, actually do nothing...") + + # FIXME: Sometimes the number of commits is greater than the number of begins. + # It should be a problem with the micro, which can be reproduced through the test of dbt_show. + def commit(self): + logger.debug("Committing transaction, actually do nothing...") + def add_begin_query(self): pass diff --git a/dbt/adapters/maxcompute/impl.py b/dbt/adapters/maxcompute/impl.py index 5a7b657..9f9a48c 100644 --- a/dbt/adapters/maxcompute/impl.py +++ b/dbt/adapters/maxcompute/impl.py @@ -443,6 +443,7 @@ def load_dataframe( pd_dataframe = pd.read_csv( file_path, delimiter=field_delimiter, parse_dates=timestamp_columns ) + logger.debug(f"Load csv to table {database}.{schema}.{table_name}") # make sure target table exist for i in range(10): try: diff --git a/dbt/adapters/maxcompute/wrapper.py b/dbt/adapters/maxcompute/wrapper.py index 88d7632..886e903 100644 --- a/dbt/adapters/maxcompute/wrapper.py +++ b/dbt/adapters/maxcompute/wrapper.py @@ -59,12 +59,12 @@ def remove_comments(input_string): result = re.sub(r"/\*[^+].*?\*/", "", input_string, flags=re.DOTALL) return result - operation = remove_comments(operation) + # operation = remove_comments(operation) parameters = param_normalization(parameters) operation = replace_sql_placeholders(operation, parameters) - # retry three times, each time wait for 10 seconds - retry_times = 3 + # retry ten times, each time wait for 10 seconds + retry_times = 10 for i in range(retry_times): try: super().execute(operation) @@ -78,6 +78,7 @@ def remove_comments(input_string): or e.code == "ODPS-0110061" or e.code == "ODPS-0130131" or e.code == "ODPS-0420111" + or e.code == "ODPS-0130071" ): if i == retry_times - 1: raise e diff --git a/dbt/include/maxcompute/macros/adapters/persist_docs.sql b/dbt/include/maxcompute/macros/adapters/persist_docs.sql new file mode 100644 index 0000000..1670137 --- /dev/null +++ b/dbt/include/maxcompute/macros/adapters/persist_docs.sql @@ -0,0 +1,16 @@ +{% macro maxcompute__alter_column_comment(relation, column_dict) %} + {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %} + {% for column_name in column_dict if (column_name in existing_columns) %} + {% set comment = column_dict[column_name]['description'] %} + {% set escaped_comment = quote_and_escape(comment) %} + alter table {{ relation.render() }} change column {{ column_name }} comment {{ escaped_comment }}; + {% endfor %} +{% endmacro %} + +{% macro maxcompute__alter_relation_comment(relation, relation_comment) -%} +{% if relation.is_table -%} + alter table {{relation.render()}} set comment {{quote_and_escape(relation_comment)}}; +{% else -%} + {{ exceptions.raise_compiler_error("MaxCompute not support set comment to view") }} +{% endif -%} +{% endmacro %} diff --git a/dbt/include/maxcompute/macros/materializations/clone.sql b/dbt/include/maxcompute/macros/materializations/clone.sql new file mode 100644 index 0000000..97ad86f --- /dev/null +++ b/dbt/include/maxcompute/macros/materializations/clone.sql @@ -0,0 +1,8 @@ +{% macro maxcompute__can_clone_table() %} + {{ return(True) }} +{% endmacro %} + + +{% macro maxcompute__create_or_replace_clone(this_relation, defer_relation) %} + clone table {{ this_relation.render() }} to {{ defer_relation.render() }} if exists overwrite; +{% endmacro %} diff --git a/dbt/include/maxcompute/macros/materializations/hooks.sql b/dbt/include/maxcompute/macros/materializations/hooks.sql new file mode 100644 index 0000000..9539d22 --- /dev/null +++ b/dbt/include/maxcompute/macros/materializations/hooks.sql @@ -0,0 +1,10 @@ +{% macro run_hooks(hooks, inside_transaction=True) %} + {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %} + {% set rendered = render(hook.get('sql')) | trim %} + {% if (rendered | length) > 0 %} + {% call statement(auto_begin=inside_transaction) %} + {{ rendered }} + {% endcall %} + {% endif %} + {% endfor %} +{% endmacro %} diff --git a/dbt/include/maxcompute/macros/materializations/snapshots/snapshot.sql b/dbt/include/maxcompute/macros/materializations/snapshots/snapshot.sql index 16b6ffc..62ede4d 100644 --- a/dbt/include/maxcompute/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/maxcompute/macros/materializations/snapshots/snapshot.sql @@ -14,7 +14,6 @@ {%- endmacro %} - {% macro build_snapshot_staging_table(strategy, sql, target_relation) %} {% set temp_relation = make_temp_relation(target_relation) %} diff --git a/dbt/include/maxcompute/macros/relations/table/create.sql b/dbt/include/maxcompute/macros/relations/table/create.sql index 6b16611..547bf0c 100644 --- a/dbt/include/maxcompute/macros/relations/table/create.sql +++ b/dbt/include/maxcompute/macros/relations/table/create.sql @@ -12,7 +12,7 @@ {%- set is_delta = is_transactional and primary_keys is not none and primary_keys|length > 0 -%} {% call statement('create_table', auto_begin=False) -%} - create table if not exists {{ relation.render() }} ( + create table {{ relation.render() }} ( {% set contract_config = config.get('contract') %} {% if contract_config.enforced and (not temporary) %} {{ get_assert_columns_equivalent(sql) }} @@ -57,12 +57,17 @@ {{ c.name }} {{ c.dtype }} {% if primary_keys and c.name in primary_keys -%}not null{%- endif %} {% if model_columns and c.name in model_columns -%} - {{ "COMMENT" }} '{{ model_columns[c.name].description }}' + {{ "COMMENT" }} {{ quote_and_escape(model_columns[c.name].description) }} {%- endif %} {{ "," if not loop.last }} {% endfor %} {%- endmacro %} +{% macro quote_and_escape(input_string) %} + {% set escaped_string = input_string | replace("'", "\\'") %} + '{{ escaped_string }}' +{% endmacro %} + -- Compared to get_table_columns_and_constraints, only the surrounding brackets are deleted {% macro get_table_columns_and_constraints_without_brackets() -%} {# loop through user_provided_columns to create DDL with data types and constraints #} diff --git a/dbt/include/maxcompute/macros/relations/view/create.sql b/dbt/include/maxcompute/macros/relations/view/create.sql index a4ef081..68d765c 100644 --- a/dbt/include/maxcompute/macros/relations/view/create.sql +++ b/dbt/include/maxcompute/macros/relations/view/create.sql @@ -2,7 +2,7 @@ {%- set sql_header = config.get('sql_header', none) -%} {{ sql_header if sql_header is not none }} - create or replace view {{ relation.render() }} + create view {{ relation.render() }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} {{ get_assert_columns_equivalent(sql) }} diff --git a/tests/functional/adapter/data/seed_model.sql b/tests/functional/adapter/data/seed_model.sql new file mode 100644 index 0000000..13a63b2 --- /dev/null +++ b/tests/functional/adapter/data/seed_model.sql @@ -0,0 +1,16 @@ +drop table if exists {schema}.on_model_hook; + +create table {schema}.on_model_hook ( + test_state string, -- start|end + target_dbname string, + target_host string, + target_name string, + target_schema string, + target_type string, + target_user string, + target_pass string, + target_threads int, + run_started_at string, + invocation_id string, + thread_id string +); diff --git a/tests/functional/adapter/data/seed_run.sql b/tests/functional/adapter/data/seed_run.sql new file mode 100644 index 0000000..5c875ac --- /dev/null +++ b/tests/functional/adapter/data/seed_run.sql @@ -0,0 +1,16 @@ +drop table if exists {schema}.on_run_hook; + +create table {schema}.on_run_hook ( + test_state string, -- start|end + target_dbname string, + target_host string, + target_name string, + target_schema string, + target_type string, + target_user string, + target_pass string, + target_threads int, + run_started_at string, + invocation_id string, + thread_id string +); diff --git a/tests/functional/adapter/test_catalog.py b/tests/functional/adapter/test_catalog.py new file mode 100644 index 0000000..01c2b90 --- /dev/null +++ b/tests/functional/adapter/test_catalog.py @@ -0,0 +1,25 @@ +import pytest +from dbt.artifacts.schemas.catalog import CatalogArtifact +from dbt.tests.adapter.catalog.relation_types import CatalogRelationTypes + + +class TestCatalogAdapter(CatalogRelationTypes): + + @pytest.fixture(scope="class", autouse=True) + def models(self): + from dbt.tests.adapter.catalog import files + + yield {"my_table.sql": files.MY_TABLE, "my_view.sql": files.MY_VIEW} + + @pytest.mark.parametrize( + "node_name,relation_type", + [ + ("seed.test.my_seed", "TABLE"), + ("model.test.my_table", "TABLE"), + ("model.test.my_view", "VIEW"), + ], + ) + def test_relation_types_populate_correctly( + self, docs: CatalogArtifact, node_name: str, relation_type: str + ): + super().test_relation_types_populate_correctly(docs, node_name, relation_type) diff --git a/tests/functional/adapter/test_dbt_clone.py b/tests/functional/adapter/test_dbt_clone.py new file mode 100644 index 0000000..c0702c4 --- /dev/null +++ b/tests/functional/adapter/test_dbt_clone.py @@ -0,0 +1,27 @@ +import pytest +from dbt.tests.adapter.dbt_clone.test_dbt_clone import ( + BaseClonePossible, + BaseCloneSameTargetAndState, +) + + +class TestMaxComputeClonePossible(BaseClonePossible): + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=f"{project.test_schema}_seeds" + ) + project.adapter.drop_schema(relation) + + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + pass + + +class TestCloneSameTargetAndState(BaseCloneSameTargetAndState): + pass diff --git a/tests/functional/adapter/test_dbt_show.py b/tests/functional/adapter/test_dbt_show.py new file mode 100644 index 0000000..1dfa956 --- /dev/null +++ b/tests/functional/adapter/test_dbt_show.py @@ -0,0 +1,17 @@ +from dbt.tests.adapter.dbt_show.test_dbt_show import ( + BaseShowSqlHeader, + BaseShowLimit, + BaseShowDoesNotHandleDoubleLimit, +) + + +class TestPostgresShowSqlHeader(BaseShowSqlHeader): + pass + + +class TestPostgresShowLimit(BaseShowLimit): + pass + + +class TestShowDoesNotHandleDoubleLimit(BaseShowDoesNotHandleDoubleLimit): + pass diff --git a/tests/functional/adapter/test_empty.py b/tests/functional/adapter/test_empty.py new file mode 100644 index 0000000..1da9571 --- /dev/null +++ b/tests/functional/adapter/test_empty.py @@ -0,0 +1,9 @@ +from dbt.tests.adapter.empty.test_empty import BaseTestEmpty, BaseTestEmptyInlineSourceRef + + +class TestEmpty(BaseTestEmpty): + pass + + +class TestEmptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + pass diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py new file mode 100644 index 0000000..be57eb4 --- /dev/null +++ b/tests/functional/adapter/test_ephemeral.py @@ -0,0 +1,74 @@ +from dbt.tests.adapter.ephemeral.test_ephemeral import ( + BaseEphemeral, + BaseEphemeralMulti, + BaseEphemeralNested, + BaseEphemeralErrorHandling, +) +import os +import re +from dbt.tests.util import check_relations_equal, run_dbt + + +# change test project to actually project, here is 'dingxin' +class TestEphemeralMulti(BaseEphemeralMulti): + def test_ephemeral_multi(self, project): + run_dbt(["seed"]) + results = run_dbt(["run"]) + assert len(results) == 3 + + check_relations_equal(project.adapter, ["seed", "dependent"]) + check_relations_equal(project.adapter, ["seed", "double_dependent"]) + check_relations_equal(project.adapter, ["seed", "super_dependent"]) + assert os.path.exists("./target/run/test/models/double_dependent.sql") + with open("./target/run/test/models/double_dependent.sql", "r") as fp: + sql_file = fp.read() + + sql_file = re.sub(r"\d+", "", sql_file) + expected_sql = ( + "create view `dingxin`.`test_test_ephemeral`.`double_dependent__dbt_tmp` as (" + "with __dbt__cte__base as (" + "select * from test_test_ephemeral.seed" + "), __dbt__cte__base_copy as (" + "select * from __dbt__cte__base" + ")-- base_copy just pulls from base. Make sure the listed" + "-- graph of CTEs all share the same dbt_cte__base cte" + "select * from __dbt__cte__base where gender = 'Male'" + "union all" + "select * from __dbt__cte__base_copy where gender = 'Female'" + ");" + ) + sql_file = "".join(sql_file.split()) + expected_sql = "".join(expected_sql.split()) + assert sql_file == expected_sql + + pass + + +class TestEphemeralNested(BaseEphemeralNested): + def test_ephemeral_nested(self, project): + results = run_dbt(["run"]) + assert len(results) == 2 + assert os.path.exists("./target/run/test/models/root_view.sql") + with open("./target/run/test/models/root_view.sql", "r") as fp: + sql_file = fp.read() + + sql_file = re.sub(r"\d+", "", sql_file) + expected_sql = ( + "create view `dingxin`.`test_test_ephemeral`.`root_view__dbt_tmp` as (" + "with __dbt__cte__ephemeral_level_two as (" + "select * from `dingxin`.`test_test_ephemeral`.`source_table`" + "), __dbt__cte__ephemeral as (" + "select * from __dbt__cte__ephemeral_level_two" + ")select * from __dbt__cte__ephemeral" + ");" + ) + + sql_file = "".join(sql_file.split()) + expected_sql = "".join(expected_sql.split()) + assert sql_file == expected_sql + + pass + + +class TestEphemeralErrorHandling(BaseEphemeralErrorHandling): + pass diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py new file mode 100644 index 0000000..0d96516 --- /dev/null +++ b/tests/functional/adapter/test_grants.py @@ -0,0 +1,9 @@ +import pytest +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants + + +@pytest.mark.skip( + reason="Please use webconsole/pyodps for permission operations. Dbt will not perform any modification operations." +) +class TestSeedGrants(BaseSeedGrants): + pass diff --git a/tests/functional/adapter/test_hooks.py b/tests/functional/adapter/test_hooks.py new file mode 100644 index 0000000..971b904 --- /dev/null +++ b/tests/functional/adapter/test_hooks.py @@ -0,0 +1,656 @@ +from pathlib import Path + +from dbt_common.exceptions import CompilationError + +# TODO: does this belong in dbt-tests-adapter? +from dbt.exceptions import ParsingError +import pytest + +from dbt.tests.adapter.hooks import fixtures +from dbt.tests.util import run_dbt, write_file + +MODEL_PRE_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id, + thread_id + ) VALUES ( + 'start', + '{{ target.dbname }}', + '{{ target.host }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}', + '{{ thread_id }}' + ) +""" + +MODEL_POST_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id, + thread_id + ) VALUES ( + 'end', + '{{ target.dbname }}', + '{{ target.host }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}', + '{{ thread_id }}' + ) +""" + + +class BaseTestPrePost: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_model.sql")) + + def get_ctx_vars(self, state, count, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + "thread_id", + ] + field_list = ", ".join(["`{}`".format(f) for f in fields]) + query = f"select {field_list} from {project.test_schema}.on_model_hook where test_state = '{state}'" + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into hooks table" + assert len(vals) >= count, "too few rows in hooks table" + assert len(vals) <= count, "too many rows in hooks table" + return [{k: v for k, v in zip(fields, val)} for val in vals] + + def check_hooks(self, state, project, host, count=1): + ctxs = self.get_ctx_vars(state, count=count, project=project) + for ctx in ctxs: + assert ctx["test_state"] == state + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "maxcompute" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + assert ctx["thread_id"].startswith("Thread-") + + +class BasePrePostModelHooks(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [ + MODEL_PRE_HOOK, + ], + "post-hook": [ + MODEL_POST_HOOK, + ], + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks} + + def test_pre_and_post_run_hooks(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooks(BasePrePostModelHooks): + pass + + +class TestPrePostModelHooksUnderscores(BasePrePostModelHooks): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre_hook": [ + MODEL_PRE_HOOK, + ], + "post_hook": [ + MODEL_POST_HOOK, + ], + } + } + } + + +class BaseHookRefs(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "hooked": { + "post-hook": [ + """ + insert into {{this.schema}}.on_model_hook select + test_state, + '{{ target.dbname }}' as target_dbname, + '{{ target.host }}' as target_host, + '{{ target.name }}' as target_name, + '{{ target.schema }}' as target_schema, + '{{ target.type }}' as target_type, + '{{ target.user }}' as target_user, + '{{ target.get(pass, "") }}' as target_pass, + {{ target.threads }} as target_threads, + '{{ run_started_at }}' as run_started_at, + '{{ invocation_id }}' as invocation_id, + '{{ thread_id }}' as thread_id + from {{ ref('post') }}""".strip() + ], + } + }, + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "hooked.sql": fixtures.models__hooked, + "post.sql": fixtures.models__post, + "pre.sql": fixtures.models__pre, + } + + def test_pre_post_model_hooks_refed(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestHookRefs(BaseHookRefs): + pass + + +properties__seed_models = """ +version: 2 +seeds: +- name: example_seed + config: + transactional: true + columns: + - name: new_col + data_tests: + - not_null +""" + + +class BasePrePostModelHooksOnSeeds: + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": fixtures.seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def models(self): + + return {"schema.yml": properties__seed_models} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "post-hook": [ + "alter table {{ this }} add column new_col int", + "update {{ this }} set new_col = 1", + # call any macro to track dependency: https://github.com/dbt-labs/dbt-core/issues/6806 + "select cast(null as int) as id", + ], + "quote_columns": False, + }, + } + + def test_hooks_on_seeds(self, project): + res = run_dbt(["seed"]) + assert len(res) == 1, "Expected exactly one item" + res = run_dbt(["test"]) + assert len(res) == 1, "Expected exactly one item" + + +class TestPrePostModelHooksOnSeeds(BasePrePostModelHooksOnSeeds): + pass + + +class BaseHooksRefsOnSeeds: + """ + This should not succeed, and raise an explicit error + https://github.com/dbt-labs/dbt-core/issues/6806 + """ + + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": fixtures.seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": fixtures.properties__seed_models, "post.sql": fixtures.models__post} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seeds": { + "post-hook": [ + "select * from {{ ref('post') }}", + ], + }, + } + + def test_hook_with_ref_on_seeds(self, project): + with pytest.raises(ParsingError) as excinfo: + run_dbt(["parse"]) + assert "Seeds cannot depend on other nodes" in str(excinfo.value) + + +class TestHooksRefsOnSeeds(BaseHooksRefsOnSeeds): + pass + + +class BasePrePostModelHooksOnSeedsPlusPrefixed(BasePrePostModelHooksOnSeeds): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add column new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSeedsPlusPrefixed(BasePrePostModelHooksOnSeedsPlusPrefixed): + pass + + +class BasePrePostModelHooksOnSeedsPlusPrefixedWhitespace(BasePrePostModelHooksOnSeeds): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "models": {}, + "seeds": { + "+post-hook": [ + "alter table {{ this }} add column new_col int", + "update {{ this }} set new_col = 1", + ], + "quote_columns": False, + }, + } + + +class TestPrePostModelHooksOnSeedsPlusPrefixedWhitespace( + BasePrePostModelHooksOnSeedsPlusPrefixedWhitespace +): + pass + + +class BasePrePostModelHooksOnSnapshots: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + path = Path(project.project_root) / "test-snapshots" + Path.mkdir(path) + write_file(fixtures.snapshots__test_snapshot, path, "snapshot.sql") + + @pytest.fixture(scope="class") + def models(self): + return {"schema.yml": fixtures.properties__test_snapshot_models} + + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": fixtures.seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add column new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + def test_hooks_on_snapshots(self, project): + res = run_dbt(["seed"]) + assert len(res) == 1, "Expected exactly one item" + res = run_dbt(["snapshot"]) + assert len(res) == 1, "Expected exactly one item" + res = run_dbt(["test"]) + assert len(res) == 1, "Expected exactly one item" + + +class TestPrePostModelHooksOnSnapshots(BasePrePostModelHooksOnSnapshots): + pass + + +class PrePostModelHooksInConfigSetup(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "macro-paths": ["macros"], + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks_configured} + + +class BasePrePostModelHooksInConfig(PrePostModelHooksInConfigSetup): + def test_pre_and_post_model_hooks_model(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooksInConfig(BasePrePostModelHooksInConfig): + pass + + +class BasePrePostModelHooksInConfigWithCount(PrePostModelHooksInConfigSetup): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + # outside transaction (runs first) + {"sql": "set a=b;", "transaction": False}, + ], + "post-hook": [ + # outside transaction (runs second) + {"sql": "set b=a;", "transaction": False}, + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + } + } + + def test_pre_and_post_model_hooks_model_and_project(self, project, dbt_profile_target): + run_dbt() + + self.check_hooks("start", project, dbt_profile_target.get("host", None), count=2) + self.check_hooks("end", project, dbt_profile_target.get("host", None), count=2) + + +class TestPrePostModelHooksInConfigWithCount(BasePrePostModelHooksInConfigWithCount): + pass + + +class BasePrePostModelHooksInConfigKwargs(BasePrePostModelHooksInConfig): + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks_kwargs} + + +class TestPrePostModelHooksInConfigKwargs(BasePrePostModelHooksInConfigKwargs): + pass + + +class BasePrePostSnapshotHooksInConfigKwargs(BasePrePostModelHooksOnSnapshots): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + path = Path(project.project_root) / "test-kwargs-snapshots" + Path.mkdir(path) + write_file(fixtures.snapshots__test_snapshot, path, "snapshot.sql") + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "seed-paths": ["seeds"], + "snapshot-paths": ["test-kwargs-snapshots"], + "models": {}, + "snapshots": { + "post-hook": [ + "alter table {{ this }} add column new_col int", + "update {{ this }} set new_col = 1", + ] + }, + "seeds": { + "quote_columns": False, + }, + } + + +class TestPrePostSnapshotHooksInConfigKwargs(BasePrePostSnapshotHooksInConfigKwargs): + pass + + +class BaseDuplicateHooksInConfigs: + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks_error} + + def test_run_duplicate_hook_defs(self, project): + with pytest.raises(CompilationError) as exc: + run_dbt() + assert "pre_hook" in str(exc.value) + assert "pre-hook" in str(exc.value) + + +class TestDuplicateHooksInConfigs(BaseDuplicateHooksInConfigs): + pass + + +import os +from pathlib import Path + +import pytest + +from dbt_common.exceptions import DbtDatabaseError +from dbt.tests.adapter.hooks import fixtures +from dbt.tests.util import check_table_does_not_exist, run_dbt + + +class BasePrePostRunHooks: + @pytest.fixture(scope="function") + def setUp(self, project): + project.run_sql_file(project.test_data_dir / Path("seed_run.sql")) + project.run_sql(f"drop table if exists {project.test_schema}.schemas") + project.run_sql(f"drop table if exists {project.test_schema}.db_schemas") + os.environ["TERM_TEST"] = "TESTING" + + @pytest.fixture(scope="class") + def macros(self): + return { + "hook.sql": fixtures.macros__hook, + "before-and-after.sql": fixtures.macros__before_and_after, + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks} + + @pytest.fixture(scope="class") + def seeds(self): + return {"example_seed.csv": fixtures.seeds__example_seed_csv} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": [ + "{{ custom_run_hook('start', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.start_hook_order_test ( id int )", + "drop table {{ target.schema }}.start_hook_order_test", + "{{ log(env_var('TERM_TEST'), info=True) }}", + ], + "on-run-end": [ + "{{ custom_run_hook('end', target, run_started_at, invocation_id) }}", + "create table {{ target.schema }}.end_hook_order_test ( id int )", + "drop table {{ target.schema }}.end_hook_order_test", + "create table {{ target.schema }}.schemas ( schema string )", + "insert into {{ target.schema }}.schemas (schema) values {% for schema in schemas %}( '{{ schema }}' ){% if not loop.last %},{% endif %}{% endfor %}", + "create table {{ target.schema }}.db_schemas ( db string, schema string )", + "insert into {{ target.schema }}.db_schemas (db, schema) values {% for db, schema in database_schemas %}('{{ db }}', '{{ schema }}' ){% if not loop.last %},{% endif %}{% endfor %}", + ], + "seeds": { + "quote_columns": False, + }, + } + + def get_ctx_vars(self, state, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + "thread_id", + ] + field_list = ", ".join(["`{}`".format(f) for f in fields]) + query = f"select {field_list} from {project.test_schema}.on_run_hook where test_state = '{state}'" + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into on_run_hook table" + assert len(vals) == 1, "too many rows in hooks table" + ctx = dict([(k, v) for (k, v) in zip(fields, vals[0])]) + + return ctx + + def assert_used_schemas(self, project): + schemas_query = "select * from {}.schemas".format(project.test_schema) + results = project.run_sql(schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.test_schema + + db_schemas_query = "select * from {}.db_schemas".format(project.test_schema) + results = project.run_sql(db_schemas_query, fetch="all") + assert len(results) == 1 + assert results[0][0] == project.database + assert results[0][1] == project.test_schema + + def check_hooks(self, state, project, host): + ctx = self.get_ctx_vars(state, project) + + assert ctx["test_state"] == state + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == "maxcompute" + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + assert ctx["thread_id"].startswith("Thread-") or ctx["thread_id"] == "MainThread" + + def test_pre_and_post_run_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["run"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + def test_pre_and_post_seed_hooks(self, setUp, project, dbt_profile_target): + run_dbt(["seed"]) + + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + check_table_does_not_exist(project.adapter, "start_hook_order_test") + check_table_does_not_exist(project.adapter, "end_hook_order_test") + self.assert_used_schemas(project) + + +class TestPrePostRunHooks(BasePrePostRunHooks): + pass + + +class BaseAfterRunHooks: + @pytest.fixture(scope="class") + def macros(self): + return {"temp_macro.sql": fixtures.macros_missing_column} + + @pytest.fixture(scope="class") + def models(self): + return {"test_column.sql": fixtures.models__missing_column} + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + # The create and drop table statements here validate that these hooks run + # in the same order that they are defined. Drop before create is an error. + # Also check that the table does not exist below. + "on-run-start": "- {{ export_table_check() }}" + } + + def test_missing_column_pre_hook(self, project): + with pytest.raises(DbtDatabaseError): + run_dbt(["run"], expect_pass=False) + + +class TestAfterRunHooks(BaseAfterRunHooks): + pass diff --git a/tests/functional/adapter/test_materialized_view.py b/tests/functional/adapter/test_materialized_view.py new file mode 100644 index 0000000..9f71174 --- /dev/null +++ b/tests/functional/adapter/test_materialized_view.py @@ -0,0 +1,9 @@ +import pytest +from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic + + +@pytest.mark.skip( + reason="`get_create_materialized_view_as_sql` has not been implemented for this adapter." +) +class TestMaterializedViewMaxCompute(MaterializedViewBasic): + pass diff --git a/tests/functional/adapter/test_persist_docs.py b/tests/functional/adapter/test_persist_docs.py new file mode 100644 index 0000000..c57a8b1 --- /dev/null +++ b/tests/functional/adapter/test_persist_docs.py @@ -0,0 +1,47 @@ +import json + +import pytest +from dbt.tests.adapter.persist_docs.test_persist_docs import ( + BasePersistDocs, + BasePersistDocsColumnMissing, + BasePersistDocsCommentOnQuotedColumn, +) +from dbt.tests.adapter.persist_docs import fixtures +from dbt.tests.util import run_dbt + +_MODELS__VIEW = """ +{{ config(materialized='table') }} +select 2 as id, 'Bob' as name +""" + + +# Note: Not support persist docs to view. +class TestPersistDocsRedshift(BasePersistDocs): + @pytest.fixture(scope="class") + def models(self): + return { + "no_docs_model.sql": fixtures._MODELS__NO_DOCS_MODEL, + "table_model.sql": fixtures._MODELS__TABLE, + } + + def test_has_comments_pglike(self, project): + run_dbt(["docs", "generate"]) + with open("target/catalog.json") as fp: + catalog_data = json.load(fp) + assert "nodes" in catalog_data + assert len(catalog_data["nodes"]) == 3 + table_node = catalog_data["nodes"]["model.test.table_model"] + view_node = self._assert_has_table_comments(table_node) + + no_docs_node = catalog_data["nodes"]["model.test.no_docs_model"] + self._assert_has_view_comments(no_docs_node, False, False) + + pass + + +class TestPersistDocsRedshiftColumn(BasePersistDocsColumnMissing): + pass + + +class TestPersistDocsCommentOnQuotedColumn(BasePersistDocsCommentOnQuotedColumn): + pass diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py new file mode 100644 index 0000000..aea46f1 --- /dev/null +++ b/tests/functional/adapter/test_python_model.py @@ -0,0 +1,11 @@ +import pytest +from dbt.tests.adapter.python_model.test_python_model import ( + BasePythonModelTests, + BasePythonIncrementalTests, +) +from dbt.tests.adapter.python_model.test_spark import BasePySparkTests + + +@pytest.mark.skip(reason="Materialization only supports languages ['sql']; got 'python'") +class TestBasePythonModelTests(BasePythonModelTests): + pass diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py new file mode 100644 index 0000000..30399b2 --- /dev/null +++ b/tests/functional/adapter/test_query_comment.py @@ -0,0 +1,32 @@ +from dbt.tests.adapter.query_comment.test_query_comment import ( + BaseQueryComments, + BaseMacroQueryComments, + BaseMacroArgsQueryComments, + BaseMacroInvalidQueryComments, + BaseNullQueryComments, + BaseEmptyQueryComments, +) + + +class TestQueryComments(BaseQueryComments): + pass + + +class TestMacroQueryComments(BaseMacroQueryComments): + pass + + +class TestMacroArgsQueryComments(BaseMacroArgsQueryComments): + pass + + +class TestMacroInvalidQueryComments(BaseMacroInvalidQueryComments): + pass + + +class TestNullQueryComments(BaseNullQueryComments): + pass + + +class TestEmptyQueryComments(BaseEmptyQueryComments): + pass diff --git a/tests/functional/adapter/test_simple_copy.py b/tests/functional/adapter/test_simple_copy.py new file mode 100644 index 0000000..9a0643b --- /dev/null +++ b/tests/functional/adapter/test_simple_copy.py @@ -0,0 +1,30 @@ +import pytest +from dbt.tests.adapter.simple_copy.test_simple_copy import SimpleCopyBase, EmptyModelsArentRunBase +from dbt.tests.adapter.simple_copy.test_copy_uppercase import BaseSimpleCopyUppercase +from dbt.tests.util import run_dbt, rm_file, write_file, check_relations_equal + + +class TestSimpleCopyBase(SimpleCopyBase): + pass + + +@pytest.mark.skip(reason="This test is ok, but we need re-implement get_tables_in_schema method") +class TestEmptyModelsArentRun(EmptyModelsArentRunBase): + + def test_dbt_doesnt_run_empty_models(self, project): + results = run_dbt(["seed"]) + assert len(results) == 1 + results = run_dbt() + assert len(results) == 7 + + tables = self.get_tables_in_schema(project) + + assert "empty" not in tables.keys() + assert "disabled" not in tables.keys() + + def get_tables_in_schema(self, project): + odps = project.adapter.get_odps_client() + tables = odps.list_tables(schema=self.test_schema) + return {table.name: table.type for table in tables} + + pass diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py new file mode 100644 index 0000000..930aa2d --- /dev/null +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -0,0 +1,145 @@ +import pytest +from dbt.tests.adapter.simple_snapshot import common, seeds, snapshots +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSimpleSnapshot, BaseSnapshotCheck +from dbt.tests.util import run_dbt, relation_from_name + +schema_seed__yml = """ +seeds: + - name: seed + config: + transactional: true + column_types: { + updated_at: 'timestamp' + } +model: + - name: fact + config: + transactional: true +""" + + +class TestSnapshot(BaseSimpleSnapshot): + @pytest.fixture(scope="class") + def seeds(self): + """ + This seed file contains all records needed for tests, including records which will be inserted after the + initial snapshot. This table will only need to be loaded once at the class level. It will never be altered, hence requires no further + setup or teardown. + """ + return {"seed.csv": seeds.SEED_CSV, "schema.yaml": schema_seed__yml} + + def test_updates_are_captured_by_snapshot(self, project): + """ + Update the last 5 records. Show that all ids are current, but the last 5 reflect updates. + """ + self.update_fact_records( + {"updated_at": "updated_at + interval 1 day"}, "id between 16 and 20" + ) + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(16, 21), + ) + + def create_fact_from_seed(self, where: str = None): # type: ignore + to_table_name = relation_from_name(self.project.adapter, "fact") + from_table_name = relation_from_name(self.project.adapter, "seed") + where_clause = where + sql = f"drop table if exists {to_table_name}" + self.project.run_sql(sql) + sql = f""" + clone table {from_table_name} to {to_table_name} + """ + self.project.run_sql(sql) + sql = f""" + delete from {to_table_name} where not({where_clause}) + """ + self.project.run_sql(sql) + + def test_new_column_captured_by_snapshot(self, project): + """ + Add a column to `fact` and populate the last 10 records with a non-null value. + Show that all ids are current, but the last 10 reflect updates and the first 10 don't + i.e. if the column is added, but not updated, the record doesn't reflect that it's updated + """ + self.add_fact_column("full_name", "varchar(200) default null") + self.update_fact_records( + { + "full_name": "first_name || ' ' || last_name", + "updated_at": "updated_at + interval 1 day", + }, + "id between 11 and 20", + ) + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(11, 21), + ) + + pass + + def test_new_column_captured_by_snapshot(self, project): + """ + Add a column to `fact` and populate the last 10 records with a non-null value. + Show that all ids are current, but the last 10 reflect updates and the first 10 don't + i.e. if the column is added, but not updated, the record doesn't reflect that it's updated + """ + self.add_fact_column("full_name", "varchar(200)") + self.update_fact_records( + { + "full_name": "first_name || ' ' || last_name", + "updated_at": "updated_at + interval 1 day", + }, + "id between 11 and 20", + ) + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(11, 21), + ) + + +class TestSnapshotCheck(BaseSnapshotCheck): + @pytest.fixture(scope="class") + def seeds(self): + """ + This seed file contains all records needed for tests, including records which will be inserted after the + initial snapshot. This table will only need to be loaded once at the class level. It will never be altered, hence requires no further + setup or teardown. + """ + return {"seed.csv": seeds.SEED_CSV, "schema.yaml": schema_seed__yml} + + def create_fact_from_seed(self, where: str = None): # type: ignore + to_table_name = relation_from_name(self.project.adapter, "fact") + from_table_name = relation_from_name(self.project.adapter, "seed") + where_clause = where + sql = f"drop table if exists {to_table_name}" + self.project.run_sql(sql) + sql = f""" + clone table {from_table_name} to {to_table_name} + """ + self.project.run_sql(sql) + sql = f""" + delete from {to_table_name} where not({where_clause}) + """ + self.project.run_sql(sql) + + def test_column_selection_is_reflected_in_snapshot(self, project): + """ + Update the first 10 records on a non-tracked column. + Update the middle 10 records on a tracked column. (hence records 6-10 are updated on both) + Show that all ids are current, and only the tracked column updates are reflected in `snapshot`. + """ + self.update_fact_records( + {"last_name": "substr(last_name, 1, 3)"}, "id between 1 and 10" + ) # not tracked + self.update_fact_records( + {"email": "substr(email, 1, 3)"}, "id between 6 and 15" + ) # tracked + run_dbt(["snapshot"]) + self._assert_results( + ids_with_current_snapshot_records=range(1, 21), + ids_with_closed_out_snapshot_records=range(6, 16), + ) + + pass diff --git a/tests/functional/adapter/test_store_test_failures.py b/tests/functional/adapter/test_store_test_failures.py new file mode 100644 index 0000000..9d7ae39 --- /dev/null +++ b/tests/functional/adapter/test_store_test_failures.py @@ -0,0 +1,7 @@ +from dbt.tests.adapter.store_test_failures_tests.test_store_test_failures import ( + BaseStoreTestFailures, +) + + +class TestStoreTestFailures(BaseStoreTestFailures): + pass