diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7adfff214..f1082a419d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,14 @@ Changelog Changelog ========= +1.11.3 (2025-12-16) +------------------- + +Bug Fixes + +* (back-ported) Fix resolution of ``packages-install-path`` when it uses ``env_var`` by @tatiana in #2194 + + 1.11.2 (2025-11-24) -------------------- diff --git a/cosmos/__init__.py b/cosmos/__init__.py index 21c6434366..2386267cb5 100644 --- a/cosmos/__init__.py +++ b/cosmos/__init__.py @@ -9,7 +9,7 @@ from cosmos import settings -__version__ = "1.11.2" +__version__ = "1.11.3" if not settings.enable_memory_optimised_imports: from cosmos.airflow.dag import DbtDag diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py index 8da6eeb8d1..9f5504eeba 100644 --- a/cosmos/dbt/project.py +++ b/cosmos/dbt/project.py @@ -7,6 +7,7 @@ from typing import Generator import yaml +from jinja2 import Template from cosmos.constants import ( DBT_DEFAULT_PACKAGES_FOLDER, @@ -40,6 +41,27 @@ def has_non_empty_dependencies_file(project_path: Path) -> bool: return False +def _resolve_env_var(template_str: str) -> str: + """ + Given a Jinja template string, resolve the environment variables, declared using the dbt syntax, + and return the rendered string. + + Example: + - template_str = '/usr/local/airflow/dags/dbt/dbt_packages{{ "_" + env_var("env","") if env_var("env","")!="" }}' + - environment variable `env` is set to "test" + + Then, the rendered string will be: + '/usr/local/airflow/dags/dbt/dbt_packages_test' + """ + + def env_var(name: str, default: str = "") -> str: + return os.getenv(name, default) + + template = Template(template_str) + rendered = template.render(env_var=env_var) + return rendered + + def get_dbt_packages_subpath(source_folder: Path) -> str: """ Return the dbt project's package installation sub path. @@ -64,7 +86,7 @@ def get_dbt_packages_subpath(source_folder: Path) -> str: logger.info(f"Unable to read the {DBT_PROJECT_FILENAME} file") else: subpath = dbt_project_file_content.get("packages-install-path", DBT_DEFAULT_PACKAGES_FOLDER) - return subpath + return _resolve_env_var(subpath) def copy_dbt_packages(source_folder: Path, target_folder: Path) -> None: diff --git a/tests/dbt/test_project.py b/tests/dbt/test_project.py index 7038d3b75d..2d323f0018 100644 --- a/tests/dbt/test_project.py +++ b/tests/dbt/test_project.py @@ -7,6 +7,7 @@ from cosmos.constants import DBT_DEFAULT_PACKAGES_FOLDER, DBT_PROJECT_FILENAME, PACKAGE_LOCKFILE_YML from cosmos.dbt.project import ( + _resolve_env_var, change_working_directory, copy_dbt_packages, copy_manifest_file_if_exists, @@ -78,6 +79,57 @@ def test_returns_custom_path_when_defined(tmpdir): assert result == "custom_dbt_packages" +@patch.dict(os.environ, {"MY_PATH": "custom_packages"}) +def test_resolve_env_var_with_simple_env_var(): + """Test _resolve_env_var with and without a simple env_var reference.""" + + result = _resolve_env_var("dbt_packages") + assert result == "dbt_packages" + + result = _resolve_env_var('{{ env_var("MY_PATH") }}') + assert result == "custom_packages" + + +@patch.dict(os.environ, {}, clear=False) +def test_resolve_env_var_with_default_value(): + """Test _resolve_env_var with env_var default when variable is not set.""" + # Ensure the variable is not set + os.environ.pop("NONEXISTENT_VAR", None) + result = _resolve_env_var('{{ env_var("NONEXISTENT_VAR", "default_path") }}') + assert result == "default_path" + + +@patch.dict(os.environ, {"dbt_packages_suffix": "test"}) +def test_resolve_env_var_with_complex_template(): + """Test _resolve_env_var with complex conditional templates.""" + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages_test" + + os.environ.pop("dbt_packages_suffix", None) + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages" + + +@patch.dict(os.environ, {}, clear=False) +def test_resolve_env_var_with_complex_template_unset_var(): + """Test _resolve_env_var with a complex conditional template when variable is not set.""" + if "dbt_packages_suffix" in os.environ: + del os.environ["dbt_packages_suffix"] + template = 'dbt_packages{{ "_" + env_var("dbt_packages_suffix","") if env_var("dbt_packages_suffix","")!="" }}' + result = _resolve_env_var(template) + assert result == "dbt_packages" + + +@patch.dict(os.environ, {"ENV_SUFFIX": "prod"}) +def test_get_dbt_packages_subpath_with_env_var_template(tmpdir): + """Test get_dbt_packages_subpath with env_var in packages-install-path.""" + write_dbt_project_yml(tmpdir, {"packages-install-path": 'dbt_packages_{{ env_var("ENV_SUFFIX") }}'}) + result = get_dbt_packages_subpath(tmpdir) + assert result == "dbt_packages_prod" + + def test_create_symlinks(tmp_path): """Tests that symlinks are created for expected files in the dbt project directory.""" tmp_dir = tmp_path / "dbt-project"