diff --git a/cosmos/config.py b/cosmos/config.py index 1e5226e773..ff36bee829 100644 --- a/cosmos/config.py +++ b/cosmos/config.py @@ -147,7 +147,7 @@ class ProjectConfig: :param dbt_project_path: The path to the dbt project directory. Example: /path/to/dbt/project. Defaults to None :param install_dbt_deps: Run dbt deps during DAG parsing and task execution. Defaults to True. - :param copy_dbt_packages: Copy dbt_packages directory, if it exists, instead of creating a symbolic link. If not set, fetches the value from [cosmos]default_copy_dbt_packages_value (False by default). + :param copy_dbt_packages: Copy dbt_packages directory, if it exists, instead of creating a symbolic link. If not set, fetches the value from [cosmos]default_copy_dbt_packages (False by default). :param models_relative_path: The relative path to the dbt models directory within the project. Defaults to models :param seeds_relative_path: The relative path to the dbt seeds directory within the project. Defaults to seeds :param snapshots_relative_path: The relative path to the dbt snapshots directory within the project. Defaults to @@ -168,7 +168,7 @@ class ProjectConfig: dbt_project_path: Path | None = None install_dbt_deps: bool = True - copy_dbt_packages: bool = settings.default_copy_dbt_packages_value + copy_dbt_packages: bool = settings.default_copy_dbt_packages manifest_path: Path | None = None models_path: Path | None = None seeds_path: Path | None = None @@ -179,7 +179,7 @@ def __init__( self, dbt_project_path: str | Path | None = None, install_dbt_deps: bool = True, - copy_dbt_packages: bool = settings.default_copy_dbt_packages_value, + copy_dbt_packages: bool = settings.default_copy_dbt_packages, models_relative_path: str | Path = "models", seeds_relative_path: str | Path = "seeds", snapshots_relative_path: str | Path = "snapshots", diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 776c98f545..686d8fce78 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -154,7 +154,7 @@ def __init__( profile_config: ProfileConfig, invocation_mode: InvocationMode | None = None, install_deps: bool = True, - copy_dbt_packages: bool = settings.default_copy_dbt_packages_value, + copy_dbt_packages: bool = settings.default_copy_dbt_packages, callback: Callable[[str], None] | None = None, callback_args: dict[str, Any] | None = None, should_store_compiled_sql: bool = True, diff --git a/cosmos/settings.py b/cosmos/settings.py index 4be18b209f..9f9298d640 100644 --- a/cosmos/settings.py +++ b/cosmos/settings.py @@ -29,7 +29,7 @@ enable_cache_profile = conf.getboolean("cosmos", "enable_cache_profile", fallback=True) dbt_profile_cache_dir_name = conf.get("cosmos", "profile_cache_dir_name", fallback="profile") virtualenv_max_retries_lock = conf.getint("cosmos", "virtualenv_max_retries_lock", fallback=120) -default_copy_dbt_packages_value = conf.getboolean("cosmos", "default_copy_dbt_packages_value", fallback=False) +default_copy_dbt_packages = conf.getboolean("cosmos", "default_copy_dbt_packages", fallback=False) # Experimentally adding `remote_cache_dir` as a separate entity in the Cosmos 1.6 release to gather feedback. # This will be merged with the `cache_dir` config parameter in upcoming releases. diff --git a/docs/configuration/cosmos-conf.rst b/docs/configuration/cosmos-conf.rst index 89a11b2229..b055813f83 100644 --- a/docs/configuration/cosmos-conf.rst +++ b/docs/configuration/cosmos-conf.rst @@ -78,6 +78,16 @@ This page lists all available Airflow configurations that affect ``astronomer-co - Default: ``None`` - Environment Variable: ``AIRFLOW__COSMOS__DBT_DOCS_CONN_ID`` +.. _default_copy_dbt_packages: + +`default_copy_dbt_packages`_: + (Introduced in Cosmos 1.10.0): By default, Cosmos 1.x either installs ``dbt deps`` or creates a symbolic link to the original ``dbt_packages`` folder. + This configuration changes this behaviour, by copying the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally. + Can be overridden at a ``DbtDag`` and ``DbtTaskGroup``, via ``ProjectConfig.copy_dbt_packages``, or at an operator level, via ``operator_args={"copy_dbt_packages"}``. + + - Default: ``False`` + - Environment Variable: ``AIRFLOW__COSMOS__DEFAULT_COPY_DBT_PACKAGES`` + .. _enable_cache_profile: `enable_cache_profile`_: diff --git a/docs/configuration/operator-args.rst b/docs/configuration/operator-args.rst index 2458c0d7c0..bca4d48999 100644 --- a/docs/configuration/operator-args.rst +++ b/docs/configuration/operator-args.rst @@ -95,8 +95,10 @@ dbt-related - ``vars``: Supply dbt variables to run the task using dbt project. This argument overrides variables defined in the ``dbt_project.yml`` and any values set in ``ProjectConfig.dbt_vars``. Arguments set as dbt ``vars`` in ``operators_args`` will not be used to render the DAG when using ``LoadMode.DBT_LS``. Use ``ProjectConfig.dbt_vars`` instead for this use-case. - ``warn_error``: convert ``dbt`` warnings into errors. - ``full_refresh``: If True, then full refresh the node. This only applies to model and seed nodes. +- ``copy_dbt_packages``: (new in v1.10) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, copy the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally. - ``install_deps``: (deprecated in v1.9, use ``ProjectConfig.install_dbt_deps`` onwards) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, run ``dbt deps`` every time a task is executed. + Airflow-related ............... diff --git a/docs/configuration/project-config.rst b/docs/configuration/project-config.rst index e5d4620f89..7eb771f1f3 100644 --- a/docs/configuration/project-config.rst +++ b/docs/configuration/project-config.rst @@ -25,6 +25,7 @@ variables that should be used for rendering and execution. It takes the followin - ``env_vars``: (new in v1.3) A dictionary of environment variables used for rendering and execution. Rendering with env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode. - ``install_dbt_deps``: (new in v1.9) Run dbt deps during DAG parsing and task execution if True (default). +- ``copy_dbt_packages``: (new in v1.10) Copy the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally. - ``partial_parse``: (new in v1.4) If True, then attempt to use the ``partial_parse.msgpack`` if it exists. This is only used for the ``LoadMode.DBT_LS`` load mode, and for the ``ExecutionMode.LOCAL`` and ``ExecutionMode.VIRTUALENV`` execution modes. Due to the way that dbt `partial parsing works `_, it does not work with Cosmos profile mapping classes. To benefit from this feature, users have to set the ``profiles_yml_filepath`` argument in ``ProfileConfig``.