Skip to content

Commit

Permalink
fixup! Update test FQNs, compiled paths
Browse files Browse the repository at this point in the history
  • Loading branch information
jtcohen6 committed Sep 26, 2021
1 parent 3d654c4 commit 4aac13f
Show file tree
Hide file tree
Showing 6 changed files with 22 additions and 35 deletions.
27 changes: 7 additions & 20 deletions core/dbt/parser/schemas.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import itertools
import os
import pathlib

from abc import ABCMeta, abstractmethod
from hashlib import md5
Expand Down Expand Up @@ -288,7 +289,6 @@ def _parse_generic_test(
test: Dict[str, Any],
tags: List[str],
column_name: Optional[str],
path: Optional[str] = None,
) -> ParsedGenericTestNode:
try:
builder = TestBuilder(
Expand All @@ -309,24 +309,12 @@ def _parse_generic_test(
original_name = os.path.basename(target.original_file_path)
compiled_path = get_pseudo_test_path(builder.compiled_name, original_name)

fqn_path = []
# generic tests defined on sources: piggypack off the source table FQN,
# which is yaml file relative path + source name + source table name
if hasattr(target, "fqn"):
fqn_path.extend(target.fqn) # type: ignore
# otherwise: build FQN out of yaml file relative path + target name
elif path:
fqn_path = self.get_fqn_prefix(path.relative_path) # type: ignore
fqn_path.append(target.name)
# we shouldn't ever get here
else:
fqn_path = get_pseudo_test_path(builder.fqn_name, original_name)
# if defined on a column, include column name in the FQN
if column_name:
fqn_path.append(column_name)

# finally, add the test name itself
fqn = fqn_path + [builder.fqn_name]
# fqn is the relative path of the yaml file where this generic test is defined,
# minus the project-level directory and the file name itself
# TODO pass a consistent path object from both UnparsedNode and UnpatchedSourceDefinition
path = pathlib.Path(target.original_file_path)
relative_path = path.relative_to(*path.parts[:1])
fqn = self.get_fqn(relative_path, builder.fqn_name)

# this is the ContextConfig that is used in render_update
config: ContextConfig = self.initial_config(fqn)
Expand Down Expand Up @@ -407,7 +395,6 @@ def parse_node(self, block: GenericTestBlock) -> ParsedGenericTestNode:
test=block.test,
tags=block.tags,
column_name=block.column_name,
path=block.path,
)
self.add_test_node(block, node)
return node
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,8 +408,8 @@ def test__postgres__exposure_parents(self):

results = self.run_dbt(['ls', '--select', '1+exposure:user_exposure'])
assert len(results) == 5
assert sorted(results) == ['exposure:test.user_exposure', 'test.test.unique_users_id',
'test.test.unique_users_rollup_gender', 'test.users', 'test.users_rollup']
assert sorted(results) == ['exposure:test.user_exposure', 'test.unique_users_id',
'test.unique_users_rollup_gender', 'test.users', 'test.users_rollup']

results = self.run_dbt(['run', '-m', '+exposure:user_exposure'])
# users, users_rollup
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1348,7 +1348,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
},
'deferred': False,
'description': '',
'fqn': ['test', 'model', 'id', 'not_null_model_id'],
'fqn': ['test', 'not_null_model_id'],
'name': 'not_null_model_id',
'original_file_path': model_schema_yml_path,
'package_name': 'test',
Expand Down Expand Up @@ -1439,7 +1439,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
},
'deferred': False,
'description': '',
'fqn': ['test', 'model', 'test_nothing_model_'],
'fqn': ['test', 'test_nothing_model_'],
'name': 'test_nothing_model_',
'original_file_path': model_schema_yml_path,
'package_name': 'test',
Expand Down Expand Up @@ -1485,7 +1485,7 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False):
},
'deferred': False,
'description': '',
'fqn': ['test', 'model', 'id', 'unique_model_id'],
'fqn': ['test', 'unique_model_id'],
'name': 'unique_model_id',
'original_file_path': model_schema_yml_path,
'package_name': 'test',
Expand Down
10 changes: 5 additions & 5 deletions test/integration/047_dbt_ls_test/test_ls.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ def expect_seed_output(self):
def expect_test_output(self):
expectations = {
'name': ('not_null_outer_id', 't', 'unique_outer_id'),
'selector': ('test.outer.id.not_null_outer_id', 'test.t', 'test.outer.id.unique_outer_id'),
'selector': ('test.not_null_outer_id', 'test.t', 'test.unique_outer_id'),
'json': (
{
'name': 'not_null_outer_id',
Expand Down Expand Up @@ -447,8 +447,8 @@ def expect_all_output(self):
'test.outer',
'test.seed',
'source:test.my_source.my_table',
'test.outer.id.not_null_outer_id',
'test.outer.id.unique_outer_id',
'test.not_null_outer_id',
'test.unique_outer_id',
'test.t',
}
# analyses have their type inserted into their fqn like tests
Expand All @@ -467,12 +467,12 @@ def expect_all_output(self):

def expect_select(self):
results = self.run_dbt_ls(['--resource-type', 'test', '--select', 'outer'])
self.assertEqual(set(results), {'test.outer.id.not_null_outer_id', 'test.outer.id.unique_outer_id'})
self.assertEqual(set(results), {'test.not_null_outer_id', 'test.unique_outer_id'})

self.run_dbt_ls(['--resource-type', 'test', '--select', 'inner'], expect_pass=True)

results = self.run_dbt_ls(['--resource-type', 'test', '--select', '+inner'])
self.assertEqual(set(results), {'test.outer.id.not_null_outer_id', 'test.outer.id.unique_outer_id'})
self.assertEqual(set(results), {'test.not_null_outer_id', 'test.unique_outer_id'})

results = self.run_dbt_ls(['--resource-type', 'model', '--select', 'outer+'])
self.assertEqual(set(results), {'test.outer', 'test.sub.inner'})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def test_postgres_changed_seed_contents_state(self):

results = self.run_dbt(['ls', '--select', 'state:modified+', '--state', './state'])
assert len(results) == 7
assert set(results) == {'test.seed', 'test.table_model', 'test.view_model', 'test.ephemeral_model', 'test.view_model.id.not_null_view_model_id', 'test.view_model.id.unique_view_model_id', 'exposure:test.my_exposure'}
assert set(results) == {'test.seed', 'test.table_model', 'test.view_model', 'test.ephemeral_model', 'test.not_null_view_model_id', 'test.unique_view_model_id', 'exposure:test.my_exposure'}

shutil.rmtree('./state')
self.copy_state()
Expand Down
8 changes: 4 additions & 4 deletions test/unit/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,12 +330,12 @@ def test__parse_basic_source_tests(self):
self.assertEqual(tests[0].tags, [])
self.assertEqual(tests[0].sources, [['my_source', 'my_table']])
self.assertEqual(tests[0].column_name, 'color')
self.assertEqual(tests[0].fqn, ['snowplow', 'my_source', 'my_table', 'color', tests[0].name])
self.assertEqual(tests[0].fqn, ['snowplow', tests[0].name])
self.assertEqual(tests[1].config.severity, 'WARN')
self.assertEqual(tests[1].tags, [])
self.assertEqual(tests[1].sources, [['my_source', 'my_table']])
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].fqn, ['snowplow', 'my_source', 'my_table', 'color', tests[1].name])
self.assertEqual(tests[1].fqn, ['snowplow', tests[1].name])

file_id = 'snowplow://' + normalize('models/test_one.yml')
self.assertIn(file_id, self.parser.manifest.files)
Expand Down Expand Up @@ -414,7 +414,7 @@ def test__parse_basic_model_tests(self):
self.assertEqual(tests[0].column_name, 'color')
self.assertEqual(tests[0].package_name, 'snowplow')
self.assertTrue(tests[0].name.startswith('accepted_values_'))
self.assertEqual(tests[0].fqn, ['snowplow', 'my_model', 'color', tests[0].name])
self.assertEqual(tests[0].fqn, ['snowplow', tests[0].name])
self.assertEqual(tests[0].unique_id.split('.'), ['test', 'snowplow', tests[0].name, '9d4814efde'])
self.assertEqual(tests[0].test_metadata.name, 'accepted_values')
self.assertIsNone(tests[0].test_metadata.namespace)
Expand All @@ -434,7 +434,7 @@ def test__parse_basic_model_tests(self):
self.assertEqual(tests[1].refs, [['my_model']])
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].column_name, 'color')
self.assertEqual(tests[1].fqn, ['snowplow', 'my_model', 'color', tests[1].name])
self.assertEqual(tests[1].fqn, ['snowplow', tests[1].name])
self.assertTrue(tests[1].name.startswith('foreign_package_test_case_'))
self.assertEqual(tests[1].package_name, 'snowplow')
self.assertEqual(tests[1].unique_id.split('.'), ['test', 'snowplow', tests[1].name, '13958f62f7'])
Expand Down

0 comments on commit 4aac13f

Please sign in to comment.