diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 0a1ce68e4b..4d5f4b797b 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -64,11 +64,14 @@ logger = get_logger(__name__) -def _normalize_path(path: str) -> str: +def _normalize_path(path: str | None) -> str: """ Converts a potentially Windows path string into a Posix-friendly path. """ - return Path(path.replace("\\", "/")).as_posix() + if path is None: + return "" + else: + return Path(path.replace("\\", "/")).as_posix() class CosmosLoadDbtException(Exception): @@ -88,7 +91,8 @@ class DbtNode: unique_id: str resource_type: DbtResourceType depends_on: list[str] - file_path: Path + path_base: Path + original_file_path: Path package_name: str | None = None tags: list[str] = field(default_factory=lambda: []) config: dict[str, Any] = field(default_factory=lambda: {}) @@ -98,6 +102,11 @@ class DbtNode: downstream: list[str] = field(default_factory=lambda: []) fqn: list[str] | None = None + @property + def file_path(self) -> Path: + """Combined path to the node's file (path_base / original_file_path).""" + return self.path_base / self.original_file_path + @property def meta(self) -> dict[str, Any]: """ @@ -171,14 +180,15 @@ def owner(self) -> str: @property def context_dict(self) -> dict[str, Any]: """ - Returns a dictionary containing all the attributes of the DbtNode object, - ensuring that the output is JSON serializable so it can be stored in Airflow's db + Returns a JSON-serializable dictionary containing a curated subset of + DbtNode attributes, suitable for storing in Airflow's database. """ return { "unique_id": self.unique_id, "resource_type": self.resource_type.value, # convert enum to value "depends_on": self.depends_on, "file_path": str(self.file_path), # convert path to string + "original_file_path": str(self.original_file_path), # convert original path to string "tags": self.tags, "config": self.config, "has_test": self.has_test, @@ -306,6 +316,8 @@ def run_command( def parse_dbt_ls_output(project_path: Path | None, ls_stdout: str) -> dict[str, DbtNode]: """Parses the output of `dbt ls` into a dictionary of `DbtNode` instances.""" + if project_path is None: + raise CosmosLoadDbtException("project_path is required to parse dbt ls output") nodes = {} for line in ls_stdout.split("\n"): try: @@ -313,8 +325,8 @@ def parse_dbt_ls_output(project_path: Path | None, ls_stdout: str) -> dict[str, except json.decoder.JSONDecodeError: logger.debug("Skipped dbt ls line: %s", line) else: - base_path = ( - project_path.parent / node_dict["package_name"] if node_dict.get("package_name") else project_path # type: ignore + base_path: Path = ( + project_path.parent / node_dict["package_name"] if node_dict.get("package_name") else project_path ) # dbt-core defined the node path via "original_file_path", dbt fusion identifies it via "path" @@ -336,7 +348,8 @@ def parse_dbt_ls_output(project_path: Path | None, ls_stdout: str) -> dict[str, package_name=node_dict.get("package_name"), resource_type=DbtResourceType(node_dict["resource_type"]), depends_on=node_dict.get("depends_on", {}).get("nodes", []), - file_path=base_path / node_file_path, # type: ignore[arg-type] + path_base=base_path, + original_file_path=Path(_normalize_path(node_file_path)), tags=node_dict.get("tags") or [], config=node_dict.get("config") or {}, has_freshness=( @@ -376,9 +389,9 @@ def _build_dbt_node_from_manifest_resource( package_name = node_dict.get("package_name") is_root_project_node = manifest_project_name is None or (package_name == manifest_project_name) if package_name and not is_root_project_node: - resolved_path = project_path / packages_subpath / package_name / _normalize_path(original_file_path) + path_base = project_path / packages_subpath / package_name else: - resolved_path = project_path / _normalize_path(original_file_path) + path_base = project_path resource_type = DbtResourceType(node_dict["resource_type"]) return DbtNode( @@ -386,7 +399,8 @@ def _build_dbt_node_from_manifest_resource( package_name=package_name, resource_type=resource_type, depends_on=node_dict.get("depends_on", {}).get("nodes", []), - file_path=resolved_path, + path_base=path_base, + original_file_path=Path(_normalize_path(original_file_path)), tags=node_dict.get("tags") or [], config=node_dict.get("config") or {}, has_freshness=( @@ -978,11 +992,8 @@ def load_via_custom_parser(self) -> None: unique_id=f"{model.type.value}.{self.project.project_name}.{model_name}", resource_type=DbtResourceType(model.type.value), depends_on=list(model.config.upstream_models), - file_path=Path( - model.path.as_posix().replace( - self.render_config.project_path.as_posix(), self.execution_config.project_path.as_posix() - ) - ), + path_base=self.execution_config.project_path, + original_file_path=model.path.relative_to(self.render_config.project_path), tags=tags or [], config=config, ) diff --git a/tests/airflow/test_graph.py b/tests/airflow/test_graph.py index 28e8dd9711..171062ef60 100644 --- a/tests/airflow/test_graph.py +++ b/tests/airflow/test_graph.py @@ -57,7 +57,8 @@ unique_id=f"{DbtResourceType.SEED.value}.{SAMPLE_PROJ_PATH.stem}.seed_parent", resource_type=DbtResourceType.SEED, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={ "meta": { "cosmos": { @@ -73,7 +74,8 @@ unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent", resource_type=DbtResourceType.MODEL, depends_on=[parent_seed.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", "meta": {"owner": "parent_node"}}, has_test=True, @@ -83,13 +85,15 @@ unique_id=f"{DbtResourceType.TEST.value}.{SAMPLE_PROJ_PATH.stem}.test_parent", resource_type=DbtResourceType.TEST, depends_on=[parent_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) child_node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/child.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/child.sql"), tags=["nightly"], config={"materialized": "table", "meta": {"cosmos": {"operator_kwargs": {"queue": "custom_queue"}}}}, ) @@ -98,7 +102,8 @@ unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child2.v2", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/child2_v2.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/child2_v2.sql"), tags=["nightly"], config={"materialized": "table", "meta": {"cosmos": {"operator_kwargs": {"pool": "custom_pool"}}}}, ) @@ -112,7 +117,8 @@ def test_calculate_datached_node_name_under_is_under_250(): unique_id="model.my_dbt_project.a_very_short_name", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) assert calculate_detached_node_name(node) == "a_very_short_name_test" @@ -120,7 +126,8 @@ def test_calculate_datached_node_name_under_is_under_250(): unique_id="model.my_dbt_project." + "this_is_a_very_long_name" * 20, # 24 x 20 = 480 characters resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) assert calculate_detached_node_name(node) == "detached_0_test" @@ -128,7 +135,8 @@ def test_calculate_datached_node_name_under_is_under_250(): unique_id="model.my_dbt_project." + "this_is_another_very_long_name" * 20, resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) assert calculate_detached_node_name(node) == "detached_1_test" @@ -202,7 +210,8 @@ def test_create_task_group_for_after_each_supported_nodes(node_type: DbtResource node = DbtNode( unique_id=f"{node_type.value}.{SAMPLE_PROJ_PATH.stem}.dbt_node", resource_type=node_type, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view"}, depends_on=[], @@ -364,7 +373,8 @@ def test_calculate_leaves(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.grandparent", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -372,7 +382,8 @@ def test_calculate_leaves(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent1", resource_type=DbtResourceType.MODEL, depends_on=[grandparent_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -380,7 +391,8 @@ def test_calculate_leaves(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent2", resource_type=DbtResourceType.MODEL, depends_on=[parent1_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -388,7 +400,8 @@ def test_calculate_leaves(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, depends_on=[parent1_node.unique_id, parent2_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -406,7 +419,8 @@ def test_create_task_metadata_unsupported(caplog): unique_id=f"unsupported.{SAMPLE_PROJ_PATH.stem}.unsupported", resource_type="unsupported", depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -434,7 +448,8 @@ def test_create_task_metadata_unsupported(caplog): "unique_id": "model.my_folder.my_model", "resource_type": "model", "depends_on": [], - "file_path": ".", + "file_path": "base_path/original_file_path", + "original_file_path": "original_file_path", "has_non_detached_test": False, "tags": [], "config": {}, @@ -456,7 +471,8 @@ def test_create_task_metadata_unsupported(caplog): "unique_id": "source.my_folder.my_source", "resource_type": "source", "depends_on": [], - "file_path": ".", + "file_path": "base_path/original_file_path", + "original_file_path": "original_file_path", "tags": [], "config": {}, "has_test": False, @@ -477,7 +493,8 @@ def test_create_task_metadata_unsupported(caplog): "unique_id": "snapshot.my_folder.my_snapshot", "resource_type": "snapshot", "depends_on": [], - "file_path": ".", + "file_path": "base_path/original_file_path", + "original_file_path": "original_file_path", "has_non_detached_test": False, "tags": [], "config": {}, @@ -503,7 +520,8 @@ def test_create_task_metadata_model( unique_id=unique_id, resource_type=resource_type, depends_on=[], - file_path=Path(""), + path_base=Path("base_path"), + original_file_path=Path("original_file_path"), tags=[], config={}, has_freshness=True, @@ -524,7 +542,8 @@ def test_create_task_metadata_model_with_versions(caplog): unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model.v1", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -541,7 +560,8 @@ def test_create_task_metadata_model_use_task_group(caplog): unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=Path(""), + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -595,7 +615,8 @@ def test_create_task_metadata_source_with_rendering_options( unique_id=unique_id, resource_type=resource_type, depends_on=[], - file_path=Path(""), + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, has_freshness=has_freshness, @@ -621,7 +642,8 @@ def test_create_task_metadata_seed(caplog, use_task_group): unique_id=f"{DbtResourceType.SEED.value}.my_folder.my_seed", resource_type=DbtResourceType.SEED, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -652,7 +674,8 @@ def test_create_task_metadata_snapshot(caplog): unique_id=f"{DbtResourceType.SNAPSHOT.value}.my_folder.my_snapshot", resource_type=DbtResourceType.SNAPSHOT, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -898,7 +921,8 @@ def test_create_task_metadata_normalize_task_id( unique_id=node_id, resource_type=node_type, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -1000,7 +1024,8 @@ def test_create_test_task_metadata(node_type, node_unique_id, test_indirect_sele unique_id=node_unique_id, resource_type=node_type, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=[], config={}, ) @@ -1066,7 +1091,8 @@ def test_owner(dbt_extra_config, expected_owner): node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", **dbt_extra_config}, depends_on=[], @@ -1209,7 +1235,8 @@ def test_create_task_metadata_disable_owner_inheritance(enable_owner_inheritance node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", "meta": {"owner": node_owner}}, depends_on=[], @@ -1243,7 +1270,8 @@ def test_create_test_task_metadata_disable_owner_inheritance(enable_owner_inheri node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", "meta": {"owner": node_owner}}, depends_on=[], @@ -1290,7 +1318,8 @@ def test_generate_task_or_group_disable_owner_inheritance(enable_owner_inheritan node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.my_model", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", "meta": {"owner": node_owner}}, depends_on=[], @@ -1343,7 +1372,8 @@ def test_build_airflow_graph_disable_owner_inheritance(test_behavior, enable_own node_with_owner = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.model_with_owner", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child"], config={"materialized": "view", "meta": {"owner": "test-owner"}}, depends_on=[], @@ -1408,7 +1438,8 @@ def test_build_airflow_graph_disable_owner_inheritance_with_detached_tests(): parent_node1 = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.parent1", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent1.sql"), config={"materialized": "view", "meta": {"owner": "parent1-owner"}}, depends_on=[], ) @@ -1416,7 +1447,8 @@ def test_build_airflow_graph_disable_owner_inheritance_with_detached_tests(): parent_node2 = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.parent2", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent2.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent2.sql"), config={"materialized": "view", "meta": {"owner": "parent2-owner"}}, depends_on=[], ) @@ -1424,7 +1456,8 @@ def test_build_airflow_graph_disable_owner_inheritance_with_detached_tests(): test_node = DbtNode( unique_id=f"{DbtResourceType.TEST.value}.my_folder.test_both_parents", resource_type=DbtResourceType.TEST, - file_path=SAMPLE_PROJ_PATH / "gen2/tests/test_both_parents.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/tests/test_both_parents.sql"), config={"meta": {"owner": "test-owner"}}, depends_on=[parent_node1.unique_id, parent_node2.unique_id], ) @@ -1612,7 +1645,8 @@ def test_skip_test_task_when_only_detached_tests_exist(): parent_node1 = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.parent1", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent1.sql"), config={"materialized": "view", "meta": {"owner": "parent1-owner"}}, depends_on=[], ) @@ -1620,7 +1654,8 @@ def test_skip_test_task_when_only_detached_tests_exist(): parent_node2 = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.parent2", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent2.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent2.sql"), config={"materialized": "view", "meta": {"owner": "parent2-owner"}}, depends_on=[], ) @@ -1628,7 +1663,8 @@ def test_skip_test_task_when_only_detached_tests_exist(): parent1_test_node = DbtNode( unique_id=f"{DbtResourceType.MODEL.value}.my_folder.test_parent1", resource_type=DbtResourceType.MODEL, - file_path=SAMPLE_PROJ_PATH / "gen2/models/test_parent1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/test_parent1.sql"), config={"materialized": "view", "meta": {"owner": "parent1-owner"}}, depends_on=[parent_node1.unique_id], ) @@ -1636,7 +1672,8 @@ def test_skip_test_task_when_only_detached_tests_exist(): detached_test_node = DbtNode( unique_id=f"{DbtResourceType.TEST.value}.my_folder.test_both_parents", resource_type=DbtResourceType.TEST, - file_path=SAMPLE_PROJ_PATH / "gen2/tests/test_both_parents.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/tests/test_both_parents.sql"), config={"meta": {"owner": "test-owner"}}, depends_on=[parent_node1.unique_id, parent_node2.unique_id], ) @@ -1737,13 +1774,15 @@ def test_exclude_not_set_with_detached_tests(self): unique_id="model.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) detached_test = DbtNode( unique_id="test.my_folder.test_detached", resource_type=DbtResourceType.TEST, depends_on=["model.my_folder.my_model", "model.my_folder.other_model"], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) task_args: dict = {} detached_from_parent = {"model.my_folder.my_model": [detached_test]} @@ -1759,13 +1798,15 @@ def test_exclude_is_string_with_detached_tests(self): unique_id="model.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) detached_test = DbtNode( unique_id="test.my_folder.test_detached", resource_type=DbtResourceType.TEST, depends_on=["model.my_folder.my_model", "model.my_folder.other_model"], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) task_args: dict = {"exclude": "existing_exclude"} detached_from_parent = {"model.my_folder.my_model": [detached_test]} @@ -1781,13 +1822,15 @@ def test_exclude_is_list_with_detached_tests(self): unique_id="model.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) detached_test = DbtNode( unique_id="test.my_folder.test_detached", resource_type=DbtResourceType.TEST, depends_on=["model.my_folder.my_model", "model.my_folder.other_model"], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) task_args: dict = {"exclude": ["existing_exclude"]} detached_from_parent = {"model.my_folder.my_model": [detached_test]} @@ -1803,7 +1846,8 @@ def test_no_detached_tests(self): unique_id="model.my_folder.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) task_args: dict = {} exclude_detached_tests_if_needed(node, task_args, {}) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index e5437587df..6a3e859c23 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -116,7 +116,13 @@ def postgres_profile_config() -> ProfileConfig: ], ) def test_dbt_node_name_and_select(unique_id, expected_name, expected_select): - node = DbtNode(unique_id=unique_id, resource_type=DbtResourceType.MODEL, depends_on=[], file_path="") + node = DbtNode( + unique_id=unique_id, + resource_type=DbtResourceType.MODEL, + depends_on=[], + path_base=Path("."), + original_file_path=Path("."), + ) assert node.name == expected_name assert node.resource_name == expected_select @@ -126,7 +132,8 @@ def test_dbt_node_meta(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": {}}}, ) assert valid_node.meta == {} @@ -135,7 +142,8 @@ def test_dbt_node_meta(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": ""}}, ) with pytest.raises(CosmosLoadDbtException) as exc_info: @@ -150,7 +158,8 @@ def test_dbt_node_operator_kwargs_to_override(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": {"operator_kwargs": {}}}}, ) assert valid_node.operator_kwargs_to_override == {} @@ -159,7 +168,8 @@ def test_dbt_node_operator_kwargs_to_override(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": {"operator_kwargs": ""}}}, ) with pytest.raises(CosmosLoadDbtException) as exc_info: @@ -174,7 +184,8 @@ def test_dbt_profile_config_to_override(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": {"profile_config": {}}}}, ) assert valid_node.profile_config_to_override == {} @@ -183,7 +194,8 @@ def test_dbt_profile_config_to_override(): unique_id="some-id", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={"meta": {"cosmos": {"profile_config": ""}}}, ) with pytest.raises(CosmosLoadDbtException) as exc_info: @@ -202,7 +214,8 @@ def test_dbt_profile_config_to_override(): "unique_id": "model.my_project.customers", "resource_type": "model", "depends_on": [], - "file_path": "", + "file_path": "base_path/original_file_path", + "original_file_path": "original_file_path", "tags": [], "config": {}, "has_test": False, @@ -217,7 +230,8 @@ def test_dbt_profile_config_to_override(): "unique_id": "model.my_project.customers.v1", "resource_type": "model", "depends_on": [], - "file_path": "", + "file_path": "base_path/original_file_path", + "original_file_path": "original_file_path", "tags": [], "config": {}, "has_test": False, @@ -232,7 +246,13 @@ def test_dbt_node_context_dict( unique_id, expected_dict, ): - node = DbtNode(unique_id=unique_id, resource_type=DbtResourceType.MODEL, depends_on=[], file_path="") + node = DbtNode( + unique_id=unique_id, + resource_type=DbtResourceType.MODEL, + depends_on=[], + path_base=Path("base_path"), + original_file_path=Path("original_file_path"), + ) assert node.context_dict == expected_dict @@ -1756,7 +1776,9 @@ def test_parse_dbt_ls_output_real_life_customer_bug(caplog): "model.some_package.some_model": DbtNode( unique_id="model.some_package.some_model", resource_type=DbtResourceType.MODEL, - file_path=Path("some_package/models/some_model.sql"), + depends_on=["source.some_source"], + path_base=Path("some_package"), + original_file_path=Path("models/some_model.sql"), tags=[], config={ "access": "protected", @@ -1789,7 +1811,6 @@ def test_parse_dbt_ls_output_real_life_customer_bug(caplog): "tags": [], "unique_key": None, }, - depends_on=["source.some_source"], package_name="some_package", ), } @@ -1806,10 +1827,11 @@ def test_parse_dbt_ls_output(): "fake-unique-id": DbtNode( unique_id="fake-unique-id", resource_type=DbtResourceType.MODEL, - file_path=Path("fake-project/fake-file-path.sql"), + depends_on=[], + path_base=Path("fake-project"), + original_file_path=Path("fake-file-path.sql"), tags=[], config={}, - depends_on=[], package_name="fake-project", ), } @@ -1825,10 +1847,11 @@ def test_parse_dbt_ls_output_with_json_without_tags_or_config(): "some-unique-id": DbtNode( unique_id="some-unique-id", resource_type=DbtResourceType.MODEL, - file_path=Path("some-project/some-file-path.sql"), + depends_on=[], + path_base=Path("some-project"), + original_file_path=Path("some-file-path.sql"), tags=[], config={}, - depends_on=[], package_name="some-project", ), } @@ -1872,6 +1895,13 @@ def test_parse_dbt_ls_output_does_not_skip_non_model_without_path(caplog): assert "Skipping model" not in caplog.text +def test_parse_dbt_ls_output_raises_when_project_path_is_none(): + """Test that parse_dbt_ls_output raises CosmosLoadDbtException when project_path is None.""" + ls_stdout = '{"resource_type": "model", "name": "x", "package_name": "p", "original_file_path": "x.sql", "unique_id": "model.p.x", "tags": [], "config": {}}' + with pytest.raises(CosmosLoadDbtException, match="project_path is required to parse dbt ls output"): + parse_dbt_ls_output(None, ls_stdout) + + @patch("cosmos.dbt.graph.DbtGraph.should_use_dbt_ls_cache", return_value=False) @patch("cosmos.dbt.graph.Popen") @patch("cosmos.dbt.graph.DbtGraph.update_node_dependency") diff --git a/tests/dbt/test_pruning.py b/tests/dbt/test_pruning.py index 6b1a568ad9..d7d6302f2b 100644 --- a/tests/dbt/test_pruning.py +++ b/tests/dbt/test_pruning.py @@ -40,7 +40,8 @@ def setup_method(self): unique_id="source.test_project.source_used", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_used.yml"), + path_base=Path("/test"), + original_file_path=Path("source_used.yml"), package_name="test_project", ) @@ -48,7 +49,8 @@ def setup_method(self): unique_id="source.test_project.source_orphaned", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_orphaned.yml"), + path_base=Path("/test"), + original_file_path=Path("source_orphaned.yml"), package_name="test_project", ) @@ -56,7 +58,8 @@ def setup_method(self): unique_id="source.test_project.source_multiple", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_multiple.yml"), + path_base=Path("/test"), + original_file_path=Path("source_multiple.yml"), package_name="test_project", ) @@ -64,7 +67,8 @@ def setup_method(self): unique_id="model.test_project.model1", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.source_used"], - file_path=Path("/test/model1.sql"), + path_base=Path("/test"), + original_file_path=Path("model1.sql"), package_name="test_project", ) @@ -72,7 +76,8 @@ def setup_method(self): unique_id="model.test_project.model2", resource_type=DbtResourceType.MODEL, depends_on=["model.test_project.model1"], - file_path=Path("/test/model2.sql"), + path_base=Path("/test"), + original_file_path=Path("model2.sql"), package_name="test_project", ) @@ -80,7 +85,8 @@ def setup_method(self): unique_id="model.test_project.model3", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.source_multiple"], - file_path=Path("/test/model3.sql"), + path_base=Path("/test"), + original_file_path=Path("model3.sql"), package_name="test_project", ) @@ -88,7 +94,8 @@ def setup_method(self): unique_id="model.test_project.model4", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.source_multiple"], - file_path=Path("/test/model4.sql"), + path_base=Path("/test"), + original_file_path=Path("model4.sql"), package_name="test_project", ) @@ -126,7 +133,8 @@ def test_is_source_used_by_filtered_nodes_no_dependencies(self): unique_id="model.test_project.independent", resource_type=DbtResourceType.MODEL, depends_on=[], # No dependencies - file_path=Path("/test/independent.sql"), + path_base=Path("/test"), + original_file_path=Path("independent.sql"), package_name="test_project", ) @@ -148,7 +156,8 @@ def test_is_source_used_by_filtered_nodes_mixed_dependencies(self): "model.test_project.model1", # Model dependency "source.test_project.source_used", # Source dependency ], - file_path=Path("/test/complex.sql"), + path_base=Path("/test"), + original_file_path=Path("complex.sql"), package_name="test_project", ) @@ -188,7 +197,8 @@ def setup_method(self): unique_id="source.test_project.source_with_downstream", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_with_downstream.yml"), + path_base=Path("/test"), + original_file_path=Path("source_with_downstream.yml"), package_name="test_project", has_freshness=True, # Make it renderable ) @@ -197,7 +207,8 @@ def setup_method(self): unique_id="source.test_project.source_orphaned", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_orphaned.yml"), + path_base=Path("/test"), + original_file_path=Path("source_orphaned.yml"), package_name="test_project", has_freshness=True, # Make it renderable ) @@ -206,7 +217,8 @@ def setup_method(self): unique_id="model.test_project.model_using_source", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.source_with_downstream"], - file_path=Path("/test/model_using_source.sql"), + path_base=Path("/test"), + original_file_path=Path("model_using_source.sql"), package_name="test_project", ) @@ -315,7 +327,8 @@ def test_create_task_metadata_source_pruning_edge_cases(self): unique_id="model.test_project.unrelated", resource_type=DbtResourceType.MODEL, depends_on=["some.other.source"], # Doesn't depend on source_orphaned - file_path=Path("/test/unrelated.sql"), + path_base=Path("/test"), + original_file_path=Path("unrelated.sql"), package_name="test_project", ) metadata_unused = create_task_metadata( @@ -384,7 +397,8 @@ def test_source_pruning_with_different_source_rendering_behaviors(self): unique_id="source.test_project.source_no_freshness", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/source_no_freshness.yml"), + path_base=Path("/test"), + original_file_path=Path("source_no_freshness.yml"), package_name="test_project", has_freshness=False, # No freshness ) @@ -430,7 +444,8 @@ def test_non_source_node_with_pruning_function(self): unique_id="model.test_project.some_model", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.some_source"], - file_path=Path("/test/some_model.sql"), + path_base=Path("/test"), + original_file_path=Path("some_model.sql"), package_name="test_project", ) @@ -439,7 +454,8 @@ def test_non_source_node_with_pruning_function(self): unique_id="model.test_project.dependent_model", resource_type=DbtResourceType.MODEL, depends_on=["model.test_project.some_model"], - file_path=Path("/test/dependent_model.sql"), + path_base=Path("/test"), + original_file_path=Path("dependent_model.sql"), package_name="test_project", ) @@ -457,7 +473,8 @@ def test_source_pruning_with_complex_dependencies(self): unique_id="source.test_project.raw_data", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/raw_data.yml"), + path_base=Path("/test"), + original_file_path=Path("raw_data.yml"), package_name="test_project", has_freshness=True, ) @@ -466,7 +483,8 @@ def test_source_pruning_with_complex_dependencies(self): unique_id="model.test_project.stg_data", resource_type=DbtResourceType.MODEL, depends_on=["source.test_project.raw_data"], - file_path=Path("/test/stg_data.sql"), + path_base=Path("/test"), + original_file_path=Path("stg_data.sql"), package_name="test_project", ) @@ -474,7 +492,8 @@ def test_source_pruning_with_complex_dependencies(self): unique_id="model.test_project.mart_data", resource_type=DbtResourceType.MODEL, depends_on=["model.test_project.stg_data"], - file_path=Path("/test/mart_data.sql"), + path_base=Path("/test"), + original_file_path=Path("mart_data.sql"), package_name="test_project", ) @@ -502,7 +521,8 @@ def test_source_pruning_parameter_defaults(self): unique_id="source.test_project.test_source", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=Path("/test/test_source.yml"), + path_base=Path("/test"), + original_file_path=Path("test_source.yml"), package_name="test_project", has_freshness=True, ) diff --git a/tests/dbt/test_selector.py b/tests/dbt/test_selector.py index 38511ab6b7..2b113a257a 100644 --- a/tests/dbt/test_selector.py +++ b/tests/dbt/test_selector.py @@ -41,7 +41,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.grandparent", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "gen1/models/grandparent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen1/models/grandparent.sql"), tags=["has_child"], config={"materialized": "view", "tags": ["has_child"]}, ) @@ -50,7 +51,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.another_grandparent_node", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "gen1/models/another_grandparent_node.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen1/models/another_grandparent_node.sql"), tags=[], config={"meta": {"frequency": "daily"}}, ) @@ -59,7 +61,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent", resource_type=DbtResourceType.MODEL, depends_on=[grandparent_node.unique_id, another_grandparent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent.sql"), tags=["has_child", "is_child"], config={"materialized": "view", "tags": ["has_child", "is_child"]}, ) @@ -69,7 +72,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.child", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/child.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/child.sql"), tags=["nightly", "is_child"], config={"materialized": "table", "tags": ["nightly", "is_child"]}, ) @@ -78,7 +82,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.sibling1", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/sibling1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/sibling1.sql"), tags=["nightly", "deprecated", "test"], config={"materialized": "table", "tags": ["nightly", "deprecated", "test"]}, ) @@ -87,7 +92,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.sibling2", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/sibling2.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/sibling2.sql"), tags=["nightly", "deprecated", "test2"], config={"materialized": "table", "tags": ["nightly", "deprecated", "test2"]}, ) @@ -96,7 +102,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.public.sibling3", resource_type=DbtResourceType.MODEL, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen3/models/public.sibling3.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/public.sibling3.sql"), tags=["nightly", "deprecated", "test3"], config={"materialized": "table", "tags": ["nightly", "deprecated", "test3"]}, ) @@ -105,7 +112,8 @@ def test_is_empty_config(selector_config, paths, tags, config, other, expected): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.orphaned", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "gen3/models/orphaned.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen3/models/orphaned.sql"), tags=[], config={}, ) @@ -144,7 +152,8 @@ def test_node_fqn_str_returns_none_when_missing(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.no_fqn", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "models/no_fqn.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/no_fqn.sql"), tags=[], config={}, fqn=None, @@ -161,7 +170,8 @@ def test_select_nodes_with_fqn_selector(): unique_id=f"{DbtResourceType.MODEL.value}.{project_name}.my_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "models/my_model.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/my_model.sql"), tags=[], config={}, fqn=[project_name, "models", "my_model"], @@ -172,7 +182,8 @@ def test_select_nodes_with_fqn_selector(): unique_id=f"{DbtResourceType.MODEL.value}.{project_name}.other_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "models/other_model.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/other_model.sql"), tags=[], config={}, fqn=[project_name, "models", "other_model"], @@ -183,7 +194,8 @@ def test_select_nodes_with_fqn_selector(): unique_id=f"{DbtResourceType.MODEL.value}.{project_name}.no_fqn", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "models/no_fqn.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/no_fqn.sql"), tags=[], config={}, fqn=None, @@ -231,7 +243,8 @@ def test_select_nodes_by_select_config_meta_nested_property_two_meta_values(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.someone_else", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "gen1/models/someone_else.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen1/models/someone_else.sql"), tags=[], config={"meta": {"frequency": "daily", "dbt_environment": "dev"}}, ) @@ -331,7 +344,8 @@ def test_select_nodes_by_intersection_and_tag_ancestry(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent_sibling", resource_type=DbtResourceType.MODEL, depends_on=[grandparent_node.unique_id, another_grandparent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent_sibling.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent_sibling.sql"), tags=["is_adopted"], config={"materialized": "view", "tags": ["is_adopted"]}, ) @@ -352,7 +366,8 @@ def test_select_nodes_by_tag_ancestry(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent_sibling", resource_type=DbtResourceType.MODEL, depends_on=[grandparent_node.unique_id, another_grandparent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "gen2/models/parent_sibling.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("gen2/models/parent_sibling.sql"), tags=["is_adopted"], config={"materialized": "view", "tags": ["is_adopted"]}, ) @@ -374,7 +389,8 @@ def test_select_nodes_with_test_by_intersection_and_tag_ancestry(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.parent_sibling", resource_type=DbtResourceType.MODEL, depends_on=[grandparent_node.unique_id, another_grandparent_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), tags=["is_adopted"], config={"materialized": "view", "tags": ["is_adopted"]}, ) @@ -382,7 +398,8 @@ def test_select_nodes_with_test_by_intersection_and_tag_ancestry(): unique_id=f"{DbtResourceType.TEST.value}.{SAMPLE_PROJ_PATH.stem}.test", resource_type=DbtResourceType.TEST, depends_on=[parent_node.unique_id, parent_sibling_node.unique_id], - file_path="", + path_base=Path("."), + original_file_path=Path("."), config={}, ) new_sample_nodes = dict(sample_nodes) @@ -588,7 +605,8 @@ def test_select_nodes_by_precursors_with_external_dependency(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.local_staging", resource_type=DbtResourceType.MODEL, depends_on=[external_upstream_id], - file_path=SAMPLE_PROJ_PATH / "models/local_staging.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/local_staging.sql"), tags=[], config={}, ) @@ -596,7 +614,8 @@ def test_select_nodes_by_precursors_with_external_dependency(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.local_marts", resource_type=DbtResourceType.MODEL, depends_on=[local_staging.unique_id], - file_path=SAMPLE_PROJ_PATH / "models/local_marts.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/local_marts.sql"), tags=[], config={}, ) @@ -701,7 +720,8 @@ def test_node_without_depends_on_with_tag_selector_should_not_raise_exception(): depends_on=[], tags=[], config={}, - file_path=SAMPLE_PROJ_PATH / "tests/generic/builtin.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("tests/generic/builtin.sql"), ) nodes = {standalone_test_node.unique_id: standalone_test_node} assert not select_nodes(project_dir=SAMPLE_PROJ_PATH, nodes=nodes, select=["tag:some-tag"]) @@ -714,7 +734,8 @@ def test_should_include_node_without_depends_on(selector_config): depends_on=None, tags=[], config={}, - file_path=SAMPLE_PROJ_PATH / "tests/generic/builtin.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("tests/generic/builtin.sql"), ) selector = NodeSelector({}, selector_config) selector.visited_nodes = set() @@ -947,7 +968,8 @@ def test_select_nodes_by_resource_type_source(): unique_id=f"{DbtResourceType.SOURCE.value}.{SAMPLE_PROJ_PATH.stem}.my_source.my_table", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "sources/my_source.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("sources/my_source.yml"), tags=[], config={}, ) @@ -957,7 +979,8 @@ def test_select_nodes_by_resource_type_source(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.model_from_source", resource_type=DbtResourceType.MODEL, depends_on=[source_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "models/model_from_source.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/model_from_source.sql"), tags=["depends_on_source"], config={"materialized": "table", "tags": ["depends_on_source"]}, ) @@ -984,7 +1007,8 @@ def test_select_nodes_by_exclude_resource_type_model(): unique_id=f"{DbtResourceType.SOURCE.value}.{SAMPLE_PROJ_PATH.stem}.my_source.my_table", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "sources/my_source.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("sources/my_source.yml"), tags=[], config={}, ) @@ -994,7 +1018,8 @@ def test_select_nodes_by_exclude_resource_type_model(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.model_from_source", resource_type=DbtResourceType.MODEL, depends_on=[source_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "models/model_from_source.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/model_from_source.sql"), tags=["depends_on_source"], config={"materialized": "table", "tags": ["depends_on_source"]}, ) @@ -1023,7 +1048,8 @@ def test_select_nodes_by_source_name(): unique_id=f"{DbtResourceType.SOURCE.value}.{SAMPLE_PROJ_PATH.stem}.my_source.my_table", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "sources/my_source.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("sources/my_source.yml"), tags=[], config={}, ) @@ -1049,7 +1075,8 @@ def test_select_nodes_by_exposure_name(): unique_id=f"{DbtResourceType.EXPOSURE.value}.{SAMPLE_PROJ_PATH.stem}.exposure_name", resource_type=DbtResourceType.EXPOSURE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "exposures/my_exposure.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("exposures/my_exposure.yml"), tags=[], config={}, ) @@ -1075,7 +1102,8 @@ def test_select_nodes_by_select_package(): unique_id=f"{DbtResourceType.MODEL.value}.dbt_artifacts.artifact_model_1", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "dbt_packages/dbt_artifacts/models/artifact_model_1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("dbt_packages/dbt_artifacts/models/artifact_model_1.sql"), tags=[], config={}, package_name="dbt_artifacts", @@ -1084,7 +1112,8 @@ def test_select_nodes_by_select_package(): unique_id=f"{DbtResourceType.MODEL.value}.dbt_artifacts.artifact_model_2", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "dbt_packages/dbt_artifacts/models/artifact_model_2.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("dbt_packages/dbt_artifacts/models/artifact_model_2.sql"), tags=[], config={}, package_name="dbt_artifacts", @@ -1112,7 +1141,8 @@ def test_select_nodes_by_exclude_package(): unique_id=f"{DbtResourceType.MODEL.value}.dbt_artifacts.artifact_model_1", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "dbt_packages/dbt_artifacts/models/artifact_model_1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("dbt_packages/dbt_artifacts/models/artifact_model_1.sql"), tags=[], config={}, package_name="dbt_artifacts", @@ -1135,7 +1165,8 @@ def test_select_nodes_by_package_plus_descendants(): unique_id=f"{DbtResourceType.MODEL.value}.dbt_utils.uppercase", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "dbt_packages/dbt_utils/macros/uppercase.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("dbt_packages/dbt_utils/macros/uppercase.sql"), tags=[], config={}, package_name="dbt_utils", @@ -1145,7 +1176,8 @@ def test_select_nodes_by_package_plus_descendants(): unique_id=f"{DbtResourceType.MODEL.value}.{SAMPLE_PROJ_PATH.stem}.uses_utils", resource_type=DbtResourceType.MODEL, depends_on=[pkg_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "models/uses_utils.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/uses_utils.sql"), tags=[], config={}, ) @@ -1179,7 +1211,8 @@ def test_select_nodes_by_exclude_bare_package_name(): unique_id=f"{DbtResourceType.MODEL.value}.dbt_artifacts.artifact_model_1", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "dbt_packages/dbt_artifacts/models/artifact_model_1.sql", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("dbt_packages/dbt_artifacts/models/artifact_model_1.sql"), tags=[], config={}, package_name="dbt_artifacts", @@ -1244,7 +1277,8 @@ def test_select_exposure_nodes_by_graph_ancestry(): unique_id=f"{DbtResourceType.EXPOSURE.value}.{SAMPLE_PROJ_PATH.stem}.exposure_name", resource_type=DbtResourceType.EXPOSURE, depends_on=[parent_node.unique_id], - file_path=SAMPLE_PROJ_PATH / "exposures/my_exposure.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("exposures/my_exposure.yml"), tags=[], config={}, ) @@ -1275,7 +1309,8 @@ def test_exclude_nodes_by_resource_type_seed(): depends_on=[], tags=[], config={}, - file_path=SAMPLE_PROJ_PATH / "seeds/seed.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("seeds/seed.yml"), ) local_nodes[seed_node.unique_id] = seed_node @@ -1296,7 +1331,8 @@ def test_exclude_nodes_by_exclude_resource_type_seed(): depends_on=[], tags=[], config={}, - file_path=SAMPLE_PROJ_PATH / "models/my_seed.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("models/my_seed.yml"), ) local_nodes[seed_node.unique_id] = seed_node @@ -1320,7 +1356,8 @@ def test_source_selector(): unique_id=f"{DbtResourceType.SOURCE.value}.{SAMPLE_PROJ_PATH.stem}.my_source", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "sources/my_source.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("sources/my_source.yml"), tags=[], config={}, ) @@ -1328,7 +1365,8 @@ def test_source_selector(): unique_id=f"{DbtResourceType.SOURCE.value}.{SAMPLE_PROJ_PATH.stem}.another_source", resource_type=DbtResourceType.SOURCE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "sources/another_source.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("sources/another_source.yml"), tags=[], config={}, ) @@ -1364,7 +1402,8 @@ def test_exposure_selector(): unique_id=f"{DbtResourceType.EXPOSURE.value}.{SAMPLE_PROJ_PATH.stem}.my_exposure", resource_type=DbtResourceType.EXPOSURE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "exposures/my_exposure.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("exposures/my_exposure.yml"), tags=[], config={}, ) @@ -1372,7 +1411,8 @@ def test_exposure_selector(): unique_id=f"{DbtResourceType.EXPOSURE.value}.{SAMPLE_PROJ_PATH.stem}.another_exposure", resource_type=DbtResourceType.EXPOSURE, depends_on=[], - file_path=SAMPLE_PROJ_PATH / "exposures/another_exposure.yml", + path_base=SAMPLE_PROJ_PATH, + original_file_path=Path("exposures/another_exposure.yml"), tags=[], config={}, ) diff --git a/tests/test_converter.py b/tests/test_converter.py index 25988d6a8b..69b07418fd 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -238,7 +238,8 @@ def test_validate_arguments_schema_in_task_args(): unique_id=f"{DbtResourceType.SEED}.{SAMPLE_DBT_PROJECT.stem}.seed_parent", resource_type=DbtResourceType.SEED, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) nodes = {"seed_parent": parent_seed} @@ -1025,7 +1026,8 @@ def test_converter_contains_tasks_map(mock_load_dbt_graph, execution_mode, opera unique_id=f"{DbtResourceType.MODEL}.{SAMPLE_DBT_PROJECT.stem}.sample_model", resource_type=DbtResourceType.MODEL, depends_on=[], - file_path="", + path_base=Path("."), + original_file_path=Path("."), ) nodes_with_model = {"sample_model": sample_model}