Add documentation related to copy_dbt_packages configurations

cosmos/config.py

@@ -147,7 +147,7 @@ class ProjectConfig:
 
     :param dbt_project_path: The path to the dbt project directory. Example: /path/to/dbt/project. Defaults to None
     :param install_dbt_deps: Run dbt deps during DAG parsing and task execution. Defaults to True.
-    :param copy_dbt_packages: Copy dbt_packages directory, if it exists, instead of creating a symbolic link. If not set, fetches the value from [cosmos]default_copy_dbt_packages_value (False by default).
+    :param copy_dbt_packages: Copy dbt_packages directory, if it exists, instead of creating a symbolic link. If not set, fetches the value from [cosmos]default_copy_dbt_packages (False by default).
     :param models_relative_path: The relative path to the dbt models directory within the project. Defaults to models
     :param seeds_relative_path: The relative path to the dbt seeds directory within the project. Defaults to seeds
     :param snapshots_relative_path: The relative path to the dbt snapshots directory within the project. Defaults to
@@ -168,7 +168,7 @@ class ProjectConfig:
 
     dbt_project_path: Path | None = None
     install_dbt_deps: bool = True
-    copy_dbt_packages: bool = settings.default_copy_dbt_packages_value
+    copy_dbt_packages: bool = settings.default_copy_dbt_packages
     manifest_path: Path | None = None
     models_path: Path | None = None
     seeds_path: Path | None = None
@@ -179,7 +179,7 @@ def __init__(
         self,
         dbt_project_path: str | Path | None = None,
         install_dbt_deps: bool = True,
-        copy_dbt_packages: bool = settings.default_copy_dbt_packages_value,
+        copy_dbt_packages: bool = settings.default_copy_dbt_packages,
         models_relative_path: str | Path = "models",
         seeds_relative_path: str | Path = "seeds",
         snapshots_relative_path: str | Path = "snapshots",

cosmos/operators/local.py

@@ -154,7 +154,7 @@ def __init__(
         profile_config: ProfileConfig,
         invocation_mode: InvocationMode | None = None,
         install_deps: bool = True,
-        copy_dbt_packages: bool = settings.default_copy_dbt_packages_value,
+        copy_dbt_packages: bool = settings.default_copy_dbt_packages,
         callback: Callable[[str], None] | None = None,
         callback_args: dict[str, Any] | None = None,
         should_store_compiled_sql: bool = True,

cosmos/settings.py

@@ -29,7 +29,7 @@
 enable_cache_profile = conf.getboolean("cosmos", "enable_cache_profile", fallback=True)
 dbt_profile_cache_dir_name = conf.get("cosmos", "profile_cache_dir_name", fallback="profile")
 virtualenv_max_retries_lock = conf.getint("cosmos", "virtualenv_max_retries_lock", fallback=120)
-default_copy_dbt_packages_value = conf.getboolean("cosmos", "default_copy_dbt_packages_value", fallback=False)
+default_copy_dbt_packages = conf.getboolean("cosmos", "default_copy_dbt_packages", fallback=False)
 
 # Experimentally adding `remote_cache_dir` as a separate entity in the Cosmos 1.6 release to gather feedback.
 # This will be merged with the `cache_dir` config parameter in upcoming releases.

docs/configuration/cosmos-conf.rst

@@ -78,6 +78,16 @@ This page lists all available Airflow configurations that affect ``astronomer-co
     - Default: ``None``
     - Environment Variable: ``AIRFLOW__COSMOS__DBT_DOCS_CONN_ID``
 
+.. _default_copy_dbt_packages:
+
+`default_copy_dbt_packages`_:
+    (Introduced in Cosmos 1.10.0):  By default, Cosmos 1.x either installs ``dbt deps`` or creates a symbolic link to the original ``dbt_packages`` folder.
+    This configuration changes this behaviour, by copying the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally.
+    Can be overridden at a ``DbtDag`` and ``DbtTaskGroup``, via ``ProjectConfig.copy_dbt_packages``, or at an operator level, via ``operator_args={"copy_dbt_packages"}``.
+
+    - Default: ``False``
+    - Environment Variable: ``AIRFLOW__COSMOS__DEFAULT_COPY_DBT_PACKAGES``
+
 .. _enable_cache_profile:
 
 `enable_cache_profile`_:

docs/configuration/operator-args.rst

@@ -95,8 +95,10 @@ dbt-related
 - ``vars``: Supply dbt variables to run the task using dbt project. This argument overrides variables defined in the ``dbt_project.yml`` and any values set in ``ProjectConfig.dbt_vars``. Arguments set as dbt ``vars`` in ``operators_args`` will not be used to render the DAG when using ``LoadMode.DBT_LS``. Use  ``ProjectConfig.dbt_vars`` instead for this use-case.
 - ``warn_error``: convert ``dbt`` warnings into errors.
 - ``full_refresh``: If True, then full refresh the node. This only applies to model and seed nodes.
+- ``copy_dbt_packages``: (new in v1.10) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, copy the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally.
 - ``install_deps``: (deprecated in v1.9, use ``ProjectConfig.install_dbt_deps`` onwards) When using ``ExecutionMode.LOCAL`` or ``ExecutionMode.VIRTUALENV``, run ``dbt deps`` every time a task is executed.
 
+
 Airflow-related
 ...............
 

docs/configuration/project-config.rst

@@ -25,6 +25,7 @@ variables that should be used for rendering and execution. It takes the followin
 - ``env_vars``: (new in v1.3) A dictionary of environment variables used for rendering and execution. Rendering with
   env vars is only supported when using ``RenderConfig.LoadMode.DBT_LS`` load mode.
 - ``install_dbt_deps``: (new in v1.9) Run dbt deps during DAG parsing and task execution if True (default).
+- ``copy_dbt_packages``: (new in v1.10) Copy the dbt project ``dbt_packages`` instead of creating symbolic links, so Cosmos can run ``dbt deps`` incrementally.
 - ``partial_parse``: (new in v1.4) If True, then attempt to use the ``partial_parse.msgpack`` if it exists. This is only used
   for the ``LoadMode.DBT_LS`` load mode, and for the ``ExecutionMode.LOCAL`` and ``ExecutionMode.VIRTUALENV``
   execution modes. Due to the way that dbt `partial parsing works <https://docs.getdbt.com/reference/parsing#known-limitations>`_, it does not work with Cosmos profile mapping classes. To benefit from this feature, users have to set the ``profiles_yml_filepath`` argument in ``ProfileConfig``.