From 105b6241d105f99fb520e0c2ff62fb2d2dc0ffe4 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Fri, 12 Dec 2025 11:36:06 +0000 Subject: [PATCH 1/2] Fix resolution of `packages-install-path` when it uses `env_var` (#2194) Some dbt projects are generic and are configured in a way that the `packages-install-path` is jinja templated. For example: ``` packages-install-path: 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' ``` When attempting to use this with - `ProjectConfig.install_dbt_deps=True` - `ProjectConfig.copy_dbt_packages=True` Users who have this configuration face the following error: ``` [2025-12-11, 10:54:23 UTC] {local.py:474} INFO - Copying dbt packages to temporary folder. [2025-12-11, 10:54:23 UTC] {project.py:77} INFO - Copying dbt packages to temporary folder... [2025-12-11, 10:54:23 UTC] {taskinstance.py:3336} ERROR - Task failed with exception Traceback (most recent call last): File "/usr/local/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 776, in _execute_task result = _execute_callable(context=context, **execute_callable_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/models/taskinstance.py", line 742, in _execute_callable return ExecutionCallableRunner( ^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/airflow/utils/operator_helpers.py", line 252, in run return self.func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/base.py", line 310, in execute self.build_and_run_cmd(context=context, cmd_flags=self.add_cmd_flags()) File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 842, in build_and_run_cmd result = self.run_command( ^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 601, in run_command self._clone_project(tmp_dir_path) File "/usr/local/lib/python3.11/site-packages/cosmos/operators/local.py", line 475, in _clone_project copy_dbt_packages(Path(self.project_dir), tmp_dir_path) File "/usr/local/lib/python3.11/site-packages/cosmos/dbt/project.py", line 91, in copy_dbt_packages shutil.copy2(src_path, dst_path) File "/usr/local/lib/python3.11/shutil.py", line 448, in copy2 copyfile(src, dst, follow_symlinks=follow_symlinks) File "/usr/local/lib/python3.11/shutil.py", line 256, in copyfile with open(src, 'rb') as fsrc: ^^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: '/usr/local/airflow/dags/dbt/pr_3531_13_dbt_vip_enrichment/dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' ``` This is because when we implemented https://github.com/astronomer/astronomer-cosmos/pull/1768 we did not forsee the parameter could be a template. This PR aims to solve this issue. Observation: we'll need to release this not only as part of 1.12.0, but also cherry-pick it and, exceptionally, backport it to v1.10.3 and 1.11.3. (cherry picked from commit 4a906cdeb7bf6bb685a7f9f5186b6677f276bccf) --- cosmos/dbt/project.py | 24 +++++++++++++++++- tests/dbt/test_project.py | 52 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 1 deletion(-) diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py index 8da6eeb8d1..9f5504eeba 100644 --- a/cosmos/dbt/project.py +++ b/cosmos/dbt/project.py @@ -7,6 +7,7 @@ from typing import Generator import yaml +from jinja2 import Template from cosmos.constants import ( DBT_DEFAULT_PACKAGES_FOLDER, @@ -40,6 +41,27 @@ def has_non_empty_dependencies_file(project_path: Path) -> bool: return False +def _resolve_env_var(template_str: str) -> str: + """ + Given a Jinja template string, resolve the environment variables, declared using the dbt syntax, + and return the rendered string. + + Example: + - template_str = '/usr/local/airflow/dags/dbt/dbt_packages{{ "_" + env_var("env","") if env_var("env","")!="" }}' + - environment variable `env` is set to "test" + + Then, the rendered string will be: + '/usr/local/airflow/dags/dbt/dbt_packages_test' + """ + + def env_var(name: str, default: str = "") -> str: + return os.getenv(name, default) + + template = Template(template_str) + rendered = template.render(env_var=env_var) + return rendered + + def get_dbt_packages_subpath(source_folder: Path) -> str: """ Return the dbt project's package installation sub path. @@ -64,7 +86,7 @@ def get_dbt_packages_subpath(source_folder: Path) -> str: logger.info(f"Unable to read the {DBT_PROJECT_FILENAME} file") else: subpath = dbt_project_file_content.get("packages-install-path", DBT_DEFAULT_PACKAGES_FOLDER) - return subpath + return _resolve_env_var(subpath) def copy_dbt_packages(source_folder: Path, target_folder: Path) -> None: diff --git a/tests/dbt/test_project.py b/tests/dbt/test_project.py index 7038d3b75d..2d323f0018 100644 --- a/tests/dbt/test_project.py +++ b/tests/dbt/test_project.py @@ -7,6 +7,7 @@ from cosmos.constants import DBT_DEFAULT_PACKAGES_FOLDER, DBT_PROJECT_FILENAME, PACKAGE_LOCKFILE_YML from cosmos.dbt.project import ( + _resolve_env_var, change_working_directory, copy_dbt_packages, copy_manifest_file_if_exists, @@ -78,6 +79,57 @@ def test_returns_custom_path_when_defined(tmpdir): assert result == "custom_dbt_packages" +@patch.dict(os.environ, {"MY_PATH": "custom_packages"}) +def test_resolve_env_var_with_simple_env_var(): + """Test _resolve_env_var with and without a simple env_var reference.""" + + result = _resolve_env_var("dbt_packages") + assert result == "dbt_packages" + + result = _resolve_env_var('{{ env_var("MY_PATH") }}') + assert result == "custom_packages" + + +@patch.dict(os.environ, {}, clear=False) +def test_resolve_env_var_with_default_value(): + """Test _resolve_env_var with env_var default when variable is not set.""" + # Ensure the variable is not set + os.environ.pop("NONEXISTENT_VAR", None) + result = _resolve_env_var('{{ env_var("NONEXISTENT_VAR", "default_path") }}') + assert result == "default_path" + + +@patch.dict(os.environ, {"dbt_packages_suffix": "test"}) +def test_resolve_env_var_with_complex_template(): + """Test _resolve_env_var with complex conditional templates.""" + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages_test" + + os.environ.pop("dbt_packages_suffix", None) + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages" + + +@patch.dict(os.environ, {}, clear=False) +def test_resolve_env_var_with_complex_template_unset_var(): + """Test _resolve_env_var with a complex conditional template when variable is not set.""" + if "dbt_packages_suffix" in os.environ: + del os.environ["dbt_packages_suffix"] + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages" + + +@patch.dict(os.environ, {"ENV_SUFFIX": "prod"}) +def test_get_dbt_packages_subpath_with_env_var_template(tmpdir): + """Test get_dbt_packages_subpath with env_var in packages-install-path.""" + write_dbt_project_yml(tmpdir, {"packages-install-path": 'dbt_packages_{{ env_var("ENV_SUFFIX") }}'}) + result = get_dbt_packages_subpath(tmpdir) + assert result == "dbt_packages_prod" + + def test_create_symlinks(tmp_path): """Tests that symlinks are created for expected files in the dbt project directory.""" tmp_dir = tmp_path / "dbt-project" From 69d7dd00f40cc8a50cf9b53303f71b60337451e1 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Tue, 16 Dec 2025 08:57:40 -0500 Subject: [PATCH 2/2] Release 1.11.3 --- CHANGELOG.rst | 8 ++++++++ cosmos/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7adfff214..f1082a419d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,14 @@ Changelog Changelog ========= +1.11.3 (2025-12-16) +------------------- + +Bug Fixes + +* (back-ported) Fix resolution of ``packages-install-path`` when it uses ``env_var`` by @tatiana in #2194 + + 1.11.2 (2025-11-24) -------------------- diff --git a/cosmos/__init__.py b/cosmos/__init__.py index 21c6434366..2386267cb5 100644 --- a/cosmos/__init__.py +++ b/cosmos/__init__.py @@ -9,7 +9,7 @@ from cosmos import settings -__version__ = "1.11.2" +__version__ = "1.11.3" if not settings.enable_memory_optimised_imports: from cosmos.airflow.dag import DbtDag