From 1dfe89ee137fd9b99530704a160ec426ea4f7bfc Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Wed, 25 Oct 2023 15:59:49 +0100 Subject: [PATCH 1/4] Fix default DAG parsing to be dbt ls when ProfileMapping is given --- cosmos/dbt/graph.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index ac92c46f52..c41ee07130 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -121,11 +121,7 @@ def load( if self.project.is_manifest_available(): self.load_from_dbt_manifest() else: - if ( - execution_mode == ExecutionMode.LOCAL - and self.profile_config - and self.profile_config.is_profile_yml_available() - ): + if execution_mode == ExecutionMode.LOCAL and self.profile_config: try: self.load_via_dbt_ls() except FileNotFoundError: From 9113ca496e4d7fbc9e9c1fb001a67dc2f3742013 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Wed, 25 Oct 2023 16:00:50 +0100 Subject: [PATCH 2/4] Make test more resilient to different local testing environments --- tests/operators/test_local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/operators/test_local.py b/tests/operators/test_local.py index b883adea57..580d49e6ce 100644 --- a/tests/operators/test_local.py +++ b/tests/operators/test_local.py @@ -139,7 +139,8 @@ def test_dbt_base_operator_use_indirect_selection(indirect_selection_type) -> No assert cmd[-2] == "--indirect-selection" assert cmd[-1] == indirect_selection_type else: - assert cmd == ["dbt", "run"] + assert cmd[0].endswith("dbt") + assert cmd[1] == "run" @pytest.mark.parametrize( From 0367a8c6f5de7e7f5d9c29fd1f9a2b09a71f2247 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Wed, 25 Oct 2023 17:07:43 +0100 Subject: [PATCH 3/4] Add tests to cover change --- tests/dbt/test_graph.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index f176fa4450..3927bbfdd2 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -82,12 +82,10 @@ def test_load_automatic_manifest_is_available(mock_load_from_dbt_manifest): assert mock_load_from_dbt_manifest.called -@patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", side_effect=FileNotFoundError()) +@patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", side_effect=None) @patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls", return_value=None) -def test_load_automatic_without_manifest(mock_load_via_dbt_ls, mock_load_via_custom_parser): - project_config = ProjectConfig( - dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME, manifest_path="/tmp/manifest.json" - ) +def test_load_automatic_without_manifest_with_profile_yml(mock_load_via_dbt_ls, mock_load_via_custom_parser): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) profile_config = ProfileConfig( profile_name="test", target_name="test", @@ -99,6 +97,24 @@ def test_load_automatic_without_manifest(mock_load_via_dbt_ls, mock_load_via_cus assert not mock_load_via_custom_parser.called +@patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", side_effect=None) +@patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls", return_value=None) +def test_load_automatic_without_manifest_with_profile_mapping(mock_load_via_dbt_ls, mock_load_via_custom_parser): + project_config = ProjectConfig(dbt_project_path=DBT_PROJECTS_ROOT_DIR / DBT_PROJECT_NAME) + profile_config = ProfileConfig( + profile_name="test", + target_name="test", + profile_mapping=PostgresUserPasswordProfileMapping( + conn_id="airflow_db", + profile_args={"schema": "public"}, + ), + ) + dbt_graph = DbtGraph(project=project_config, profile_config=profile_config) + dbt_graph.load(execution_mode=ExecutionMode.LOCAL) + assert mock_load_via_dbt_ls.called + assert not mock_load_via_custom_parser.called + + @patch("cosmos.dbt.graph.DbtGraph.load_via_custom_parser", return_value=None) @patch("cosmos.dbt.graph.DbtGraph.load_via_dbt_ls", side_effect=FileNotFoundError()) def test_load_automatic_without_manifest_and_without_dbt_cmd(mock_load_via_dbt_ls, mock_load_via_custom_parser): From 51601700415bef944a69f9e30d510a398e9d325d Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Wed, 25 Oct 2023 19:07:15 +0100 Subject: [PATCH 4/4] Delete dead code identified by codecov https://app.codecov.io/gh/astronomer/astronomer-cosmos/commit/af09a3e11cecee46b426939a9ffe27614b0f6367/indirect-changes --- cosmos/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cosmos/config.py b/cosmos/config.py index 15f88e0370..610e6e489d 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -163,12 +163,6 @@ def validate_profile(self) -> None: if not self.profiles_yml_filepath and not self.profile_mapping: raise CosmosValueError("Either profiles_yml_filepath or profile_mapping must be set to render a profile") - def is_profile_yml_available(self) -> bool: - """ - Check if the `dbt` profiles.yml file exists. - """ - return Path(self.profiles_yml_filepath).exists() if self.profiles_yml_filepath else False - @contextlib.contextmanager def ensure_profile( self, desired_profile_path: Path | None = None, use_mock_values: bool = False