From 6a29323add80cbb1beb0e20beb14d69c742ec576 Mon Sep 17 00:00:00 2001 From: Dan M <47741043+dmarts@users.noreply.github.com> Date: Wed, 13 May 2020 15:09:25 +0100 Subject: [PATCH 1/2] Macro and test case --- README.md | 13 +++++++ .../data_test_fewer_rows_than_table_1.csv | 4 +++ .../data_test_fewer_rows_than_table_2.csv | 5 +++ .../schema_tests/test_fewer_rows_than.sql | 9 +++++ macros/schema_tests/fewer_rows_than.sql | 35 +++++++++++++++++++ 5 files changed, 66 insertions(+) create mode 100644 integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv create mode 100644 integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv create mode 100644 integration_tests/models/schema_tests/test_fewer_rows_than.sql create mode 100644 macros/schema_tests/fewer_rows_than.sql diff --git a/README.md b/README.md index 9f5afcfa..d86dbf8d 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,19 @@ models: ``` +#### fewer_rows_than ([source](macros/schema_tests/fewer_rows_than.sql)) +This schema test asserts that this model has fewer rows than the referenced model. + +Usage: +```yaml +version: 2 + +models: + - name: model_name + tests: + - dbt_utils.fewer_rows_than: + compare_model: ref('other_table_name') + #### equality ([source](macros/schema_tests/equality.sql)) This schema test asserts the equality of two relations. Optionally specify a subset of columns to compare. diff --git a/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv b/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv new file mode 100644 index 00000000..ba0e5ec6 --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv @@ -0,0 +1,4 @@ +field +1 +2 +3 diff --git a/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv b/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv new file mode 100644 index 00000000..eb0f035a --- /dev/null +++ b/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv @@ -0,0 +1,5 @@ +field +1 +2 +3 +4 diff --git a/integration_tests/models/schema_tests/test_fewer_rows_than.sql b/integration_tests/models/schema_tests/test_fewer_rows_than.sql new file mode 100644 index 00000000..c2ad4cbd --- /dev/null +++ b/integration_tests/models/schema_tests/test_fewer_rows_than.sql @@ -0,0 +1,9 @@ +with data as ( + + select * from {{ ref('data_test_fewer_rows_than_table_1') }} + +) + +select + field +from data \ No newline at end of file diff --git a/macros/schema_tests/fewer_rows_than.sql b/macros/schema_tests/fewer_rows_than.sql new file mode 100644 index 00000000..93254ca9 --- /dev/null +++ b/macros/schema_tests/fewer_rows_than.sql @@ -0,0 +1,35 @@ +{% macro test_fewer_rows_than(model) %} + +{% set compare_model = kwargs.get('compare_model', kwargs.get('arg')) %} + +with a as ( + + select count(*) as count_model from {{ model }} + +), +b as ( + + select count(*) as count_comparison from {{ compare_model }} + +), +counts as ( + + select + (select count_model from a) as count_model, + (select count_comparison from b) as count_comparison + +), +final as ( + + select + case + when count_model > count_comparison then count_model - count_comparison + 1 + when count_model = count_comparison then count_model - count_comparison + else 0 end as excess_rows + from counts + +) + +select excess_rows from final + +{% endmacro %} From 033dce9d7bb4b5dcd41c39ce4427ca7956d1346c Mon Sep 17 00:00:00 2001 From: Dan M <47741043+dmarts@users.noreply.github.com> Date: Tue, 5 Jan 2021 16:28:15 +0000 Subject: [PATCH 2/2] Update fewer_rows_than.sql --- macros/schema_tests/fewer_rows_than.sql | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/macros/schema_tests/fewer_rows_than.sql b/macros/schema_tests/fewer_rows_than.sql index 93254ca9..175c855a 100644 --- a/macros/schema_tests/fewer_rows_than.sql +++ b/macros/schema_tests/fewer_rows_than.sql @@ -4,32 +4,36 @@ with a as ( - select count(*) as count_model from {{ model }} + select count(*) as count_ourmodel from {{ model }} ), b as ( - select count(*) as count_comparison from {{ compare_model }} + select count(*) as count_comparisonmodel from {{ compare_model }} ), counts as ( select - (select count_model from a) as count_model, - (select count_comparison from b) as count_comparison + (select count_ourmodel from a) as count_model_with_fewer_rows, + (select count_comparisonmodel from b) as count_model_with_more_rows ), final as ( select case - when count_model > count_comparison then count_model - count_comparison + 1 - when count_model = count_comparison then count_model - count_comparison - else 0 end as excess_rows + -- fail the test if we have more rows than the reference model and return the row count delta + when count_model_with_fewer_rows > count_model_with_more_rows then (count_model_with_fewer_rows - count_model_with_more_rows) + -- fail the test if they are the same number + when count_model = count_comparison then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta from counts ) -select excess_rows from final +select row_count_delta from final {% endmacro %}