add some test coverage

core/dbt/include/global_project/macros/materializations/test.sql

@@ -13,7 +13,7 @@
       {{ fail_calc }} {{ error_if }} as should_error
     from (
       {{ sql }}
-      {{ "limit " ~ limit if limit }}
+      {{ "limit " ~ limit if limit != none }}
     ) _dbt_internal_test
 
   {% endcall %}

core/dbt/task/test.py

@@ -77,7 +77,6 @@ def execute_test(
             )
 
         # generate materialization macro
-        # simple `select(*)` of the compiled test node
         macro_func = MacroGenerator(materialization_macro, context)
         # execute materialization macro
         macro_func()
@@ -117,16 +116,19 @@ def execute(self, test: CompiledTestNode, manifest: Manifest):
         num_errors = utils.pluralize(result.failures, 'result')
         status = None
         message = None
+        failures = 0
         if severity == "ERROR" and result.should_error:
             status = TestStatus.Fail
             message = f'Got {num_errors}, configured to fail if {test.config.error_if}'
+            failures = result.failures
         elif result.should_warn:
             if flags.WARN_ERROR:
                 status = TestStatus.Fail
                 message = f'Got {num_errors}, configured to fail if {test.config.warn_if}'
             else:
                 status = TestStatus.Warn
                 message = f'Got {num_errors}, configured to warn if {test.config.warn_if}'
+            failures = result.failures
         else:
             status = TestStatus.Pass
 
@@ -138,7 +140,7 @@ def execute(self, test: CompiledTestNode, manifest: Manifest):
             execution_time=0,
             message=message,
             adapter_response={},
-            failures=int(result.failures),
+            failures=failures,
         )
 
     def after_execute(self, result):

test/integration/008_schema_tests_test/local_dependency/dbt_project.yml

@@ -0,0 +1,7 @@
+name: 'local_dep'
+version: '1.0'
+config-version: 2
+
+profile: 'default'
+
+macro-paths: ["macros"]

test/integration/008_schema_tests_test/local_dependency/macros/equality.sql

@@ -0,0 +1,63 @@
+{#-- taken from dbt-utils --#}
+{% test equality(model, compare_model, compare_columns=None) %}
+  {{ return(adapter.dispatch('test_equality')(model, compare_model, compare_columns)) }}
+{% endtest %}
+
+{% macro default__test_equality(model, compare_model, compare_columns=None) %}
+
+{% set set_diff %}
+    count(*) + abs(
+        sum(case when which_diff = 'a_minus_b' then 1 else 0 end) -
+        sum(case when which_diff = 'b_minus_a' then 1 else 0 end)
+    )
+{% endset %}
+
+{#-- Needs to be set at parse time, before we return '' below --#}
+{{ config(fail_calc = set_diff) }}
+
+{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #}
+{%- if not execute -%}
+    {{ return('') }}
+{% endif %}
+-- setup
+{%- do dbt_utils._is_relation(model, 'test_equality') -%}
+{#-
+If the compare_cols arg is provided, we can run this test without querying the
+information schema — this allows the model to be an ephemeral model
+-#}
+
+{%- if not compare_columns -%}
+    {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%}
+    {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%}
+{%- endif -%}
+
+{% set compare_cols_csv = compare_columns | join(', ') %}
+
+with a as (
+    select * from {{ model }}
+),
+b as (
+    select * from {{ compare_model }}
+),
+a_minus_b as (
+    select {{compare_cols_csv}} from a
+    {{ dbt_utils.except() }}
+    select {{compare_cols_csv}} from b
+),
+b_minus_a as (
+    select {{compare_cols_csv}} from b
+    {{ dbt_utils.except() }}
+    select {{compare_cols_csv}} from a
+),
+
+unioned as (
+
+    select 'a_minus_b' as which_diff, * from a_minus_b
+    union all
+    select 'b_minus_a' as which_diff, * from b_minus_a
+
+)
+
+select * from unioned
+
+{% endmacro %}

test/integration/008_schema_tests_test/macros-v2/custom-configs/test.sql

@@ -0,0 +1,25 @@
+{% test where(model, column_name) %}
+  {{ config(where = "1 = 0") }}
+  select * from {{ model }}
+{% endtest %}
+
+{% test error_if(model, column_name) %}
+  {{ config(error_if = "<= 0", warn_if = "<= 0") }}
+  select * from {{ model }}
+{% endtest %}
+
+
+{% test warn_if(model, column_name) %}
+  {{ config(warn_if = "<= 0", severity = "WARN") }}
+  select * from {{ model }}
+{% endtest %}
+
+{% test limit(model, column_name) %}
+  {{ config(limit = 0) }}
+  select * from {{ model }}
+{% endtest %}
+
+{% test fail_calc(model, column_name) %}
+  {{ config(fail_calc = "count(*) - count(*)") }}
+  select * from {{ model }}
+{% endtest %}

test/integration/008_schema_tests_test/macros-v2/macros/tests.sql

@@ -33,3 +33,4 @@
     select {{ validation_message }} as validation_error
     where {{ eq }} = 0
 {% endtest %}
+

test/integration/008_schema_tests_test/models-v2/custom-configs/schema.yml

@@ -0,0 +1,12 @@
+version: 2
+
+models:
+  - name: table_copy
+    description: "A copy of the table"
+    # passes
+    tests:
+      - where
+      - error_if
+      - warn_if
+      - limit
+      - fail_calc

test/integration/008_schema_tests_test/models-v2/custom-configs/table_copy.sql

@@ -0,0 +1,8 @@
+
+{{
+    config(
+        materialized='table'
+    )
+}}
+
+select * from {{ this.schema }}.seed

test/integration/008_schema_tests_test/models-v2/custom/schema.yml

@@ -15,3 +15,6 @@ models:
         tests:
           - every_value_is_blue
           - rejected_values: { values: ['orange', 'purple'] }
+    # passes
+    tests:
+      - local_dep.equality: { compare_model: ref('table_copy') }

test/integration/008_schema_tests_test/test_schema_v2_tests.py

@@ -95,7 +95,6 @@ def test_postgres_schema_test_exclude_failures(self):
             self.assertTestFailed(result)
 
 
-
 class TestMalformedSchemaTests(DBTIntegrationTest):
 
     def setUp(self):
@@ -125,6 +124,42 @@ def test_postgres_malformed_schema_strict_will_break_run(self):
             self.run_dbt(strict=False)
 
 
+class TestCustomConfigSchemaTests(DBTIntegrationTest):
+    def setUp(self):
+        DBTIntegrationTest.setUp(self)
+        self.run_sql_file("seed.sql")
+
+    @property
+    def schema(self):
+        return 'schema_tests_008'
+
+    @property
+    def models(self):
+        return "models-v2/custom-configs"
+
+    @property
+    def project_config(self):
+        return {
+            'config-version': 2,
+            "macro-paths": ["macros-v2/custom-configs"],
+        }
+
+    @use_profile('postgres')
+    def test_postgres_config(self):
+        """ Test that tests use configs properly. All tests for
+        this project will fail, configs are set to make test pass. """
+        results = self.run_dbt()
+        results = self.run_dbt(['test'], strict=False)
+
+        self.assertEqual(len(results), 5)
+        for result in results:
+            self.assertFalse(result.skipped)
+            self.assertEqual(
+                result.failures, 0,
+                'test {} failed'.format(result.node.name)
+            )
+
+
 class TestHooksInTests(DBTIntegrationTest):
 
     @property
@@ -172,6 +207,9 @@ def schema(self):
     def packages_config(self):
         return {
             'packages': [
+                {
+                    "local": "./local_dependency",
+                },
                 {
                     'git': 'https://github.com/fishtown-analytics/dbt-integration-project',
                     'revision': 'dbt/0.17.0',
@@ -206,7 +244,7 @@ def test_postgres_schema_tests(self):
         self.assertEqual(len(results), 4)
 
         test_results = self.run_schema_validations()
-        self.assertEqual(len(test_results), 5)
+        self.assertEqual(len(test_results), 6)
 
         expected_failures = [
             'not_null_table_copy_email',

test/integration/base.py

@@ -570,7 +570,6 @@ def profile_config(self):
 
     def run_dbt(self, args=None, expect_pass=True, strict=True, parser=True, profiles_dir=True):
         res, success = self.run_dbt_and_check(args=args, strict=strict, parser=parser, profiles_dir=profiles_dir)
-
         self.assertEqual(
             success, expect_pass,
             "dbt exit state did not match expected")