Skip to content

move debug logs to debug channel #2163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
matthieuauger wants to merge 1 commit into from
Closed

move debug logs to debug channel #2163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cosmos/cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ def _calculate_dbt_ls_cache_current_version(cache_identifier: str, project_dir:
hash_args = hashlib.md5("".join(cmd_args).encode()).hexdigest()

elapsed_time = time.perf_counter() - start_time
logger.info(
logger.debug(
f"Cosmos performance: time to calculate cache identifier {cache_identifier} for current version: {elapsed_time}"
)
return f"{dbt_project_hash},{hash_args}"
Expand Down
4 changes: 2 additions & 2 deletions cosmos/converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ def __init__(

current_time = time.perf_counter()
elapsed_time = current_time - previous_time
logger.info(
logger.debug(
f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to parse the dbt project for DAG using {self.dbt_graph.load_method}"
)
previous_time = current_time
Expand Down Expand Up @@ -341,7 +341,7 @@ def __init__(

current_time = time.perf_counter()
elapsed_time = current_time - previous_time
logger.info(
logger.debug(
f"Cosmos performance ({cache_identifier}) - [{platform.node()}|{os.getpid()}]: It took {elapsed_time:.3}s to build the Airflow DAG."
)

Expand Down
24 changes: 12 additions & 12 deletions cosmos/dbt/graph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -576,8 +576,8 @@ def load(

self.update_node_dependency()

logger.info("Total nodes: %i", len(self.nodes))
logger.info("Total filtered nodes: %i", len(self.filtered_nodes))
logger.debug("Total nodes: %i", len(self.nodes))
logger.debug("Total filtered nodes: %i", len(self.filtered_nodes))
Comment on lines +579 to +580
Copy link
Copy Markdown
Contributor

@pankajastro pankajastro Jan 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can combine both and keep keep as info


def run_dbt_ls(
self, dbt_cmd: str, project_path: Path, tmp_dir: Path, env_vars: dict[str, str]
Expand Down Expand Up @@ -640,13 +640,13 @@ def should_use_dbt_ls_cache(self) -> bool:

def load_via_dbt_ls_cache(self) -> bool:
"""(Try to) load dbt ls cache from an Airflow Variable"""
logger.info(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...")
logger.debug(f"Trying to parse the dbt project using dbt ls cache {self.dbt_ls_cache_key}...")
if self.should_use_dbt_ls_cache():
project_path = self.project_path

cache_dict = self.get_dbt_ls_cache()
if not cache_dict:
logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}")
logger.debug(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key}")
return False

cache_version = cache_dict.get("version")
Expand All @@ -657,17 +657,17 @@ def load_via_dbt_ls_cache(self) -> bool:
)

if dbt_ls_cache and not cache.was_project_modified(cache_version, current_version):
logger.info(
logger.debug(
f"Cosmos performance [{platform.node()}|{os.getpid()}]: The cache size for {self.dbt_ls_cache_key} is {len(dbt_ls_cache)}"
)
self.load_method = LoadMode.DBT_LS_CACHE

nodes = parse_dbt_ls_output(project_path=project_path, ls_stdout=dbt_ls_cache)
self.nodes = nodes
self.filtered_nodes = nodes
logger.info(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}")
logger.debug(f"Cosmos performance: Cache hit for {self.dbt_ls_cache_key} - {current_version}")
return True
logger.info(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped")
logger.debug(f"Cosmos performance: Cache miss for {self.dbt_ls_cache_key} - skipped")
return False

def should_use_partial_parse_cache(self) -> bool:
Expand Down Expand Up @@ -750,7 +750,7 @@ def load_via_dbt_ls_without_cache(self) -> None:
dbt_cmd = self.render_config.dbt_executable_path
dbt_cmd = dbt_cmd.as_posix() if isinstance(dbt_cmd, Path) else dbt_cmd

logger.info(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...")
logger.debug(f"Trying to parse the dbt project in `{self.render_config.project_path}` using dbt ls...")
project_path = self.project_path
if not self.profile_config:
raise CosmosLoadDbtException("Unable to load project via dbt ls without a profile config.")
Expand All @@ -769,7 +769,7 @@ def load_via_dbt_ls_without_cache(self) -> None:
latest_partial_parse = get_partial_parse_path(project_path)

if latest_partial_parse is not None and latest_partial_parse.exists():
logger.info("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse)
logger.debug("Partial parse is enabled and the latest partial parse file is %s", latest_partial_parse)
cache._copy_partial_parse_to_project(latest_partial_parse, tmpdir_path)

with (
Expand Down Expand Up @@ -825,7 +825,7 @@ def load_via_dbt_ls_file(self) -> None:
to the airflow image.
"""
self.load_method = LoadMode.DBT_LS_FILE
logger.info("Trying to parse the dbt project `%s` using a dbt ls output file...", self.project.project_name)
logger.debug("Trying to parse the dbt project `%s` using a dbt ls output file...", self.project.project_name)

if not self.render_config.is_dbt_ls_file_available():
raise CosmosLoadDbtException(f"Unable to load dbt ls file using {self.render_config.dbt_ls_path}")
Expand Down Expand Up @@ -858,7 +858,7 @@ def load_via_custom_parser(self) -> None:
" be removed in Cosmos 2.0",
DeprecationWarning,
)
logger.info("Trying to parse the dbt project `%s` using a custom Cosmos method...", self.project.project_name)
logger.debug("Trying to parse the dbt project `%s` using a custom Cosmos method...", self.project.project_name)

if self.render_config.selector:
raise CosmosLoadDbtException(
Expand Down Expand Up @@ -918,7 +918,7 @@ def load_from_dbt_manifest(self) -> None:
* self.filtered_nodes
"""
self.load_method = LoadMode.DBT_MANIFEST
logger.info("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name)
logger.debug("Trying to parse the dbt project `%s` using a dbt manifest...", self.project.project_name)

if self.render_config.selector:
raise CosmosLoadDbtException(
Expand Down
4 changes: 1 addition & 3 deletions tests/test_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1112,6 +1112,4 @@ def test_dag_versioning_successful_logging(mock_load_dbt_graph, mock_hash_func,
execution_config=execution_config,
)

mock_logger.debug.assert_called_once_with(
"Appended dbt project hash test_hash_123 to DAG test_dag_logging documentation"
)
mock_logger.debug.assert_any_call("Appended dbt project hash test_hash_123 to DAG test_dag_logging documentation")