diff --git a/cosmos/cache.py b/cosmos/cache.py index 096a7fa489..175a709d2f 100644 --- a/cosmos/cache.py +++ b/cosmos/cache.py @@ -306,7 +306,7 @@ def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir: hash_args = hashlib.md5("".join(cmd_args).encode()).hexdigest() elapsed_time = time.perf_counter() - start_time - logger.info( + logger.debug( f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}" ) return f"{dbt_project_hash},{hash_args}" diff --git a/cosmos/converter.py b/cosmos/converter.py index b56c91c668..cfffa758f4 100644 --- a/cosmos/converter.py +++ b/cosmos/converter.py @@ -295,7 +295,7 @@ def __init__( current_time = time.perf_counter() elapsed_time = current_time - previous_time - logger.info( + logger.debug( f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to parse the dbt project for DAG using {self.dbt_graph.load_method}" ) previous_time = current_time @@ -341,7 +341,7 @@ def __init__( current_time = time.perf_counter() elapsed_time = current_time - previous_time - logger.info( + logger.debug( f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to build the Airflow DAG." ) diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 2b1a8c8f19..8406c82d7b 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -576,8 +576,8 @@ def load( self.update_node_dependency() - logger.info("Total nodes: %i", len(self.nodes)) - logger.info("Total filtered nodes: %i", len(self.filtered_nodes)) + logger.debug("Total nodes: %i", len(self.nodes)) + logger.debug("Total filtered nodes: %i", len(self.filtered_nodes)) def run_dbt_ls( self, dbt_cmd: str, project_path: Path, tmp_dir: Path, env_vars: dict[str, str] @@ -640,13 +640,13 @@ def should_use_dbt_ls_cache(self) -> bool: def load_via_dbt_ls_cache(self) -> bool: """(Try to) load dbt ls cache from an Airflow Variable""" - logger.info(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...") + logger.debug(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...") if self.should_use_dbt_ls_cache(): project_path = self.project_path cache_dict = self.get_dbt_ls_cache() if not cache_dict: - logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}") + logger.debug(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}") return False cache_version = cache_dict.get("version") @@ -657,7 +657,7 @@ def load_via_dbt_ls_cache(self) -> bool: ) if dbt_ls_cache and not cache.was_project_modified(cache_version, current_version): - logger.info( + logger.debug( f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.dbt_ls_cache_key} is {len(dbt_ls_cache)}" ) self.load_method = LoadMode.DBT_LS_CACHE @@ -665,9 +665,9 @@ def load_via_dbt_ls_cache(self) -> bool: nodes = parse_dbt_ls_output(project_path=project_path, ls_stdout=dbt_ls_cache) self.nodes = nodes self.filtered_nodes = nodes - logger.info(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}") + logger.debug(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}") return True - logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped") + logger.debug(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped") return False def should_use_partial_parse_cache(self) -> bool: @@ -750,7 +750,7 @@ def load_via_dbt_ls_without_cache(self) -> None: dbt_cmd = self.render_config.dbt_executable_path dbt_cmd = dbt_cmd.as_posix() if isinstance(dbt_cmd, Path) else dbt_cmd - logger.info(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...") + logger.debug(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...") project_path = self.project_path if not self.profile_config: raise CosmosLoadDbtException("Unable to load project via dbt ls without a profile config.") @@ -769,7 +769,7 @@ def load_via_dbt_ls_without_cache(self) -> None: latest_partial_parse = get_partial_parse_path(project_path) if latest_partial_parse is not None and latest_partial_parse.exists(): - logger.info("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse) + logger.debug("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse) cache._copy_partial_parse_to_project(latest_partial_parse, tmpdir_path) with ( @@ -825,7 +825,7 @@ def load_via_dbt_ls_file(self) -> None: to the airflow image. """ self.load_method = LoadMode.DBT_LS_FILE - logger.info("Trying to parse the dbt project `%s` using a dbt ls output file...", self.project.project_name) + logger.debug("Trying to parse the dbt project `%s` using a dbt ls output file...", self.project.project_name) if not self.render_config.is_dbt_ls_file_available(): raise CosmosLoadDbtException(f"Unable to load dbt ls file using {self.render_config.dbt_ls_path}") @@ -858,7 +858,7 @@ def load_via_custom_parser(self) -> None: " be removed in Cosmos 2.0", DeprecationWarning, ) - logger.info("Trying to parse the dbt project `%s` using a custom Cosmos method...", self.project.project_name) + logger.debug("Trying to parse the dbt project `%s` using a custom Cosmos method...", self.project.project_name) if self.render_config.selector: raise CosmosLoadDbtException( @@ -918,7 +918,7 @@ def load_from_dbt_manifest(self) -> None: * self.filtered_nodes """ self.load_method = LoadMode.DBT_MANIFEST - logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) + logger.debug("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name) if self.render_config.selector: raise CosmosLoadDbtException( diff --git a/tests/test_converter.py b/tests/test_converter.py index 8cf22834ae..7ecac30455 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1112,6 +1112,4 @@ def test_dag_versioning_successful_logging(mock_load_dbt_graph, mock_hash_func, execution_config=execution_config, ) - mock_logger.debug.assert_called_once_with( - "Appended dbt project hash test_hash_123 to DAG test_dag_logging documentation" - ) + mock_logger.debug.assert_any_call("Appended dbt project hash test_hash_123 to DAG test_dag_logging documentation")