diff --git a/.gitignore b/.gitignore index fb6538814f..e2c14d9eba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # cosmos-specific ignores # these files get autogenerated -docs/profiles/* +docs/reference/profiles/* # actionlint binary (downloaded when running validate-workflows or locally) actionlint diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 08c143a69a..16e69f5d42 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1083,7 +1083,7 @@ Features * Add cosmos/propagate_logs Airflow config support for disabling log propagation by @agreenburg in #648 (`documentation `_). * Add operator_args ``full_refresh`` as a templated field by @joppevos in #623 * Expose environment variables and dbt variables in ``ProjectConfig`` by @jbandoro in #735 (`documentation `_). -* Support disabling event tracking when using Cosmos profile mapping by @jbandoro in #768 (`documentation `_). +* Support disabling event tracking when using Cosmos profile mapping by @jbandoro in #768 (`documentation `_). Enhancements diff --git a/docs/conf.py b/docs/conf.py index 268a246c76..8750fcb187 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # -- Begin docs redirect section # -- To test redirects in a local build, paste the redirect source, and append .html to the end. -# -- For example, "airflow3_compatibility/index" redirect must be tested using "airflow3_compatibility/index.html" +# -- For example, "airflow3_compatibility/index" source redirect must be tested using "airflow3_compatibility/index.html" # -- https://documatt.com/sphinx-reredirects/usage/ redirects = { "airflow3_compatibility/index": "../policy/airflow3-compatibility.html", @@ -77,7 +77,7 @@ "configuration/operator-args": "../guides/run_dbt/customization/operator-args.html", "configuration/parsing-methods": "../guides/translate_dbt_to_airflow/parsing-methods.html", "configuration/partial-parsing": "../guides/run_dbt/customization/partial-parsing.html", - "configuration/profile-config": "../reference/configs/profile-config.html", + "configuration/profile-config": "../guides/connect_database/index.html", "configuration/project-config": "../reference/configs/project-config.html", "configuration/render-config": "../guides/translate_dbt_to_airflow/render-config.html", "configuration/scheduling": "../guides/run_dbt/customization/scheduling.html", @@ -100,4 +100,30 @@ "getting_started/operators": "../guides/run_dbt/operators/operators.html", "getting_started/watcher-execution-mode": "../guides/run_dbt/airflow-worker/watcher-execution-mode.html", "getting_started/watcher-kubernetes-execution-mode": "../guides/run_dbt/container/watcher-kubernetes-execution-mode.html", + "profiles/AthenaAccessKey": "../reference/profiles/AthenaAccessKey.html", + "profiles/ClickhouseUserPassword": "../reference/profiles/ClickhouseUserPassword.html", + "profiles/DatabricksOauth": "../reference/profiles/DatabricksOauth.html", + "profiles/DatabricksToken": "../reference/profiles/DatabricksToken.html", + "profiles/DuckDBUserPassword": "../reference/profiles/DuckDBUserPassword.html", + "profiles/ExasolUserPassword": "../reference/profiles/ExasolUserPassword.html", + "profiles/GoogleCloudOauth": "../reference/profiles/GoogleCloudOauth.html", + "profiles/GoogleCloudServiceAccountDict": "../reference/profiles/GoogleCloudServiceAccountDict.html", + "profiles/GoogleCloudServiceAccountFile": "../reference/profiles/GoogleCloudServiceAccountFile.html", + "profiles/index": "../reference/profiles/index.html", + "profiles/MysqlUserPassword": "../reference/profiles/MysqlUserPassword.html", + "profiles/OracleUserPassword": "../reference/profiles/OracleUserPassword.html", + "profiles/PostgresUserPassword": "../reference/profiles/PostgresUserPassword.html", + "profiles/RedshiftUserPassword": "../reference/profiles/RedshiftUserPassword.html", + "profiles/SnowflakeEncryptedPrivateKeyFilePem": "../reference/profiles/SnowflakeEncryptedPrivateKeyFilePem.html", + "profiles/SnowflakeEncryptedPrivateKeyPem": "../reference/profiles/SnowflakeEncryptedPrivateKeyPem.html", + "profiles/SnowflakePrivateKeyPem": "../reference/profiles/SnowflakePrivateKeyPem.html", + "profiles/SnowflakeUserPassword": "../reference/profiles/SnowflakeUserPassword.html", + "profiles/SparkThrift": "../reference/profiles/SparkThrift.html", + "profiles/StandardSQLServerAuth": "../reference/profiles/StandardSQLServerAuth.html", + "profiles/StarrocksUserPassword": "../reference/profiles/StarrocksUserPassword.html", + "profiles/TeradataUserPassword": "../reference/profiles/TeradataUserPassword.html", + "profiles/TrinoCertificate": "../reference/profiles/TrinoCertificate.html", + "profiles/TrinoJWT": "../reference/profiles/TrinoJWT.html", + "profiles/TrinoLDAP": "../reference/profiles/TrinoLDAP.html", + "profiles/VerticaUserPassword": "../reference/profiles/VerticaUserPassword.html", } diff --git a/docs/generate_mappings.py b/docs/generate_mappings.py index 52a7b1a787..bc825a896e 100644 --- a/docs/generate_mappings.py +++ b/docs/generate_mappings.py @@ -42,8 +42,8 @@ def get_fields_from_mapping(mapping: type[BaseProfileMapping]) -> list[Field]: def generate_mapping_docs( - templates_dir: str = "./templates", - output_dir: str = "./profiles", + templates_dir: str = "./reference/templates", + output_dir: str = "./reference/profiles", ) -> None: """ Generate a dedicated docs page per profile mapping. diff --git a/docs/guides/connect_database/index.rst b/docs/guides/connect_database/index.rst new file mode 100644 index 0000000000..985c4451a2 --- /dev/null +++ b/docs/guides/connect_database/index.rst @@ -0,0 +1,34 @@ +.. _connect_database: + +Connect to your database +=============================== + +Cosmos supports two methods of authenticating with your database: + +- using your own `dbt profiles.yml <../../guides/connect_database/use-your-profiles-yml.html>`_ file +- using Airflow connections via Cosmos' `profile mappings <../../guides/connect_database/use-profile-mapping.html>`_ + +If you're already interacting with your database from Airflow and have a connection set up, it's recommended +to use a profile mapping to translate that Airflow connection to a dbt profile. This is because it's easier to +maintain a single connection object in Airflow than it is to maintain a connection object in Airflow and a dbt profile +in your dbt project. + +If you don't already have an Airflow connection, or if there's no readily-available profile mapping for your database, +you can use your own dbt profiles.yml file. + +Regardless of which method you use, you'll need to tell Cosmos which profile and target name it should use. Profile config +is set in the ``cosmos.config.ProfileConfig`` object, like so: + +.. code-block:: python + + from cosmos.config import ProfileConfig + + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + # choose one of the following + profile_mapping=..., + profiles_yml_filepath=..., + ) + + dag = DbtDag(profile_config=profile_config, ...) diff --git a/docs/templates/index.rst.jinja2 b/docs/guides/connect_database/profile-customise-per-node.rst similarity index 50% rename from docs/templates/index.rst.jinja2 rename to docs/guides/connect_database/profile-customise-per-node.rst index 87285565c3..7f5f8dfcc7 100644 --- a/docs/templates/index.rst.jinja2 +++ b/docs/guides/connect_database/profile-customise-per-node.rst @@ -1,113 +1,8 @@ -.. - This file is autogenerated by `docs/scripts/generate_mappings.py`. Do not edit by hand. - -.. toctree:: - :caption: Profiles - - self - {% for profile in profile_mapping_names %}{{ profile }} - {% endfor %} - -Profiles Overview -================== - -Cosmos supports two methods of authenticating with your database: - -- using your own dbt profiles.yml file -- using Airflow connections via Cosmos' profile mappings - -If you're already interacting with your database from Airflow and have a connection set up, it's recommended -to use a profile mapping to translate that Airflow connection to a dbt profile. This is because it's easier to -maintain a single connection object in Airflow than it is to maintain a connection object in Airflow and a dbt profile -in your dbt project. - -If you don't already have an Airflow connection, or if there's no readily-available profile mapping for your database, -you can use your own dbt profiles.yml file. - -Regardless of which method you use, you'll need to tell Cosmos which profile and target name it should use. Profile config -is set in the ``cosmos.config.ProfileConfig`` object, like so: - -.. code-block:: python - - from cosmos.config import ProfileConfig - - profile_config = ProfileConfig( - profile_name="my_profile_name", - target_name="my_target_name", - - # choose one of the following - profile_mapping=..., - profiles_yml_filepath=..., - ) - - dag = DbtDag(profile_config=profile_config, ...) - - -Using your own profiles.yml file -++++++++++++++++++++++++++++++++ - -If you don't want to use Airflow connections, or if there's no readily-available profile mapping for your database, -you can use your own dbt profiles.yml file. To do so, you'll need to pass the path to your profiles.yml file to the -``profiles_yml_filepath`` argument in ``ProfileConfig``. - -For example, the code snippet below points Cosmos at a ``profiles.yml`` file and instructs Cosmos to use the -``my_snowflake_profile`` profile and ``dev`` target: - -.. code-block:: python - - from cosmos.config import ProfileConfig - - profile_config = ProfileConfig( - profile_name="my_snowflake_profile", - target_name="dev", - profiles_yml_filepath="/path/to/profiles.yml", - ) - - dag = DbtDag(profile_config=profile_config, ...) - -Using a profile mapping -+++++++++++++++++++++++ - -Profile mappings are utilities provided by Cosmos that translate Airflow connections to dbt profiles. This means that -you can use the same connection objects you use in Airflow to authenticate with your database in dbt. To do so, there's -a class in Cosmos for each Airflow connection to dbt profile mapping. - -You can find the available profile mappings on the left-hand side of this page. Each profile mapping is imported from -``cosmos.profiles`` and takes two arguments: - -* ``conn_id``: the Airflow connection ID to use. -* ``profile_args``: a dictionary of additional arguments to pass to the dbt profile. This is useful for specifying - values that are not in the Airflow connection. This also acts as an override for any values that are in the Airflow - connection but should be overridden. - -Below is an example of using the Snowflake profile mapping, where we take most arguments from the Airflow connection -but override the ``database`` and ``schema`` values: - -.. code-block:: python - - from cosmos.profiles import SnowflakeUserPasswordProfileMapping - - profile_config = ProfileConfig( - profile_name="my_profile_name", - target_name="my_target_name", - profile_mapping=SnowflakeUserPasswordProfileMapping( - conn_id="my_snowflake_conn_id", - profile_args={ - "database": "my_snowflake_database", - "schema": "my_snowflake_schema", - }, - ), - ) - - dag = DbtDag(profile_config=profile_config, ...) - -Note that when using a profile mapping, the profiles.yml file gets generated with the profile name and target name -you specify in ``ProfileConfig``. - .. _profile-customise-per-node: Customising the profile config per dbt node -+++++++++++++++++++++++++++++++++++++++++++ +=========================================== + .. versionadded:: 1.9.0 @@ -184,7 +79,7 @@ The parts of ``profiles.yml``, which aren't specific to a particular data platfo write_json=True, warn_error=True, warn_error_options={"include": "all"}, - log_format='text', + log_format="text", debug=True, version_check=True, ), @@ -195,7 +90,7 @@ The parts of ``profiles.yml``, which aren't specific to a particular data platfo Disabling dbt event tracking -++++++++++++++++++++++++++++ +---------------------------- .. note: Deprecated in v.1.4 and will be removed in v2.0.0. Use dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=False) instead. diff --git a/docs/guides/connect_database/use-profile-mapping.rst b/docs/guides/connect_database/use-profile-mapping.rst new file mode 100644 index 0000000000..da6d5b5d9a --- /dev/null +++ b/docs/guides/connect_database/use-profile-mapping.rst @@ -0,0 +1,40 @@ +.. _use-profile-mapping: + +Use a profile mapping +======================= + +Profile mappings are utilities provided by Cosmos that translate Airflow connections to dbt profiles. This means that +you can use the same connection objects you use in Airflow to authenticate with your database in dbt. To do so, there's +a class in Cosmos for each Airflow connection to dbt profile mapping. + +You can find the available profile mappings on the left-hand side of this page. Each profile mapping is imported from +``cosmos.profiles`` and takes two arguments: + +* ``conn_id``: the Airflow connection ID to use. +* ``profile_args``: a dictionary of additional arguments to pass to the dbt profile. This is useful for specifying + values that are not in the Airflow connection. This also acts as an override for any values that are in the Airflow + connection but should be overridden. + +Below is an example of using the Snowflake profile mapping, where we take most arguments from the Airflow connection +but override the ``database`` and ``schema`` values: + +.. code-block:: python + + from cosmos.profiles import SnowflakeUserPasswordProfileMapping + + profile_config = ProfileConfig( + profile_name="my_profile_name", + target_name="my_target_name", + profile_mapping=SnowflakeUserPasswordProfileMapping( + conn_id="my_snowflake_conn_id", + profile_args={ + "database": "my_snowflake_database", + "schema": "my_snowflake_schema", + }, + ), + ) + + dag = DbtDag(profile_config=profile_config, ...) + +Note that when using a profile mapping, the profiles.yml file gets generated with the profile name and target name +you specify in ``ProfileConfig``. diff --git a/docs/guides/connect_database/use-your-profiles-yml.rst b/docs/guides/connect_database/use-your-profiles-yml.rst new file mode 100644 index 0000000000..c5f8d8745e --- /dev/null +++ b/docs/guides/connect_database/use-your-profiles-yml.rst @@ -0,0 +1,23 @@ +.. _use-your-profiles-yml: + +Use your own profiles.yml file +============================== + +If you don't want to use Airflow connections, or if there's no readily-available profile mapping for your database, +you can use your own dbt profiles.yml file. To do so, you'll need to pass the path to your profiles.yml file to the +``profiles_yml_filepath`` argument in ``ProfileConfig``. + +For example, the code snippet below points Cosmos at a ``profiles.yml`` file and instructs Cosmos to use the +``my_snowflake_profile`` profile and ``dev`` target: + +.. code-block:: python + + from cosmos.config import ProfileConfig + + profile_config = ProfileConfig( + profile_name="my_snowflake_profile", + target_name="dev", + profiles_yml_filepath="/path/to/profiles.yml", + ) + + dag = DbtDag(profile_config=profile_config, ...) diff --git a/docs/guides/index.rst b/docs/guides/index.rst index 353d37bd97..b16685da0c 100644 --- a/docs/guides/index.rst +++ b/docs/guides/index.rst @@ -12,6 +12,16 @@ Cosmos offers a number of configuration options to customize its behavior. For m dbt_setup/dbt-fusion +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Connect to your database + + Connection options + connect_database/use-your-profiles-yml + connect_database/use-profile-mapping + connect_database/profile-customise-per-node + .. toctree:: :maxdepth: 1 :hidden: diff --git a/docs/guides/run_dbt/container/kubernetes.rst b/docs/guides/run_dbt/container/kubernetes.rst index d200589429..4ff074ec41 100644 --- a/docs/guides/run_dbt/container/kubernetes.rst +++ b/docs/guides/run_dbt/container/kubernetes.rst @@ -160,8 +160,8 @@ The Kubernetes execution mode has the following limitations: - Does not emit OpenLineage events (there is an `open ticket #496 `__ to address this) - Does not emit Airflow datasets, assets, and dataset aliases (there is an `open ticket #2329 `__ to address this) - Does not handle installing dbt deps for users (there is an `open ticket #679 `__ to address this) -- Does not support `ProfileMapping `_ (there is an `open ticket #749 `__ to address this) -- Does not support `Callbacks `_ (there is an `open ticket #1575 `__ to address this) -- Does not expose Compiled SQL as a `templated field `_ -- Does not benefit from `Cosmos caching mechanisms `_ -- Does not support `generating dbt docs & uploading to an object store `_ (there is a `PR `_ to solve this for S3) +- Does not support `ProfileMapping `_ (there is an `open ticket #749 `__ to address this) +- Does not support `Callbacks `_ (there is an `open ticket #1575 `__ to address this) +- Does not expose Compiled SQL as a `templated field `_ +- Does not benefit from `Cosmos caching mechanisms `_ +- Does not support `generating dbt docs & uploading to an object store `_ (there is a `PR `_ to solve this for S3) diff --git a/docs/index.rst b/docs/index.rst index 8c24731043..cd79000a16 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,6 @@ Getting Started Guides Optimize Performance - Profiles Reference Project Policies diff --git a/docs/reference/configs/index.rst b/docs/reference/configs/index.rst deleted file mode 100644 index 741b7a76bd..0000000000 --- a/docs/reference/configs/index.rst +++ /dev/null @@ -1,13 +0,0 @@ - -# Configurations - Uncomment this section to turn the page into a dropdown navigation -# ============== - -.. toctree:: - :maxdepth: 1 - :hidden: - :caption: Configurations - - cosmos-conf - execution-config - profile-config - project-config diff --git a/docs/reference/configs/profile-config.rst b/docs/reference/configs/profile-config.rst deleted file mode 100644 index dcc5c784ab..0000000000 --- a/docs/reference/configs/profile-config.rst +++ /dev/null @@ -1,4 +0,0 @@ -Profile Config -================ - -Cosmos has multiple methods for supplying profiles. For more information, click on the Profiles tab on the top navbar. diff --git a/docs/reference/configs/profiles-config.rst b/docs/reference/configs/profiles-config.rst new file mode 100644 index 0000000000..095bbe51c7 --- /dev/null +++ b/docs/reference/configs/profiles-config.rst @@ -0,0 +1,11 @@ +Profiles Config +================ + +Cosmos supports two methods of authenticating with your database: + +- Using your `dbt profiles.yml <../../guides/connect_database/use-your-profiles-yml.html>`_ file +- using Airflow connections with the Cosmos `profile_mapping <../../guides/connect_database/use-profile-mapping.html>`_ + +You can also customize the profiles used in each `dbt node <../../guides/connect_database/profile-customise-per-node.html>`_. + +The `Profiles reference <../profiles/index.html>`_ contains information about the different profiles available for you to use in your database connections. diff --git a/docs/reference/index.rst b/docs/reference/index.rst index 4c430eb4d8..a1bd7e0f3e 100644 --- a/docs/reference/index.rst +++ b/docs/reference/index.rst @@ -1,9 +1,42 @@ Reference ========= +.. toctree:: + :maxdepth: 0 + :hidden: + + self + .. toctree:: :maxdepth: 1 :hidden: :caption: Configurations - configs/index + configs/project-config + configs/execution-config + configs/cosmos-conf + configs/profiles-config + +.. toctree:: + :maxdepth: 1 + :hidden: + :caption: Profiles + + profiles/index + +Configurations +~~~~~~~~~~~~~~~ + +There are different configurations and profiles that you can use to configure how Cosmos works. + +- `ProjectConfig `_: The ``ProjectConfig`` contains information about which dbt project a Cosmos Dag or task group is going to execute, as well as configurations that apply to both rendering and execution. +- `ExecutionConfig `_: The ``ExecutionConfig`` determines where and how the dbt commands are run within Cosmos. +- `CosmosConfig `_: This page lists available Airflow configurations that affect ``astronomer-cosmos`` behavior. You can set them in the ``airflow.cfg`` file or using environment variables. +- `ProfileConfig `_: The ``ProfileConfig`` class determines which data warehouse Cosmos connects to when it executes the dbt SQL. These docs include reference documentation for connecting to popular data warehouses you might use in your dbt code. + +Profiles +~~~~~~~~~ + +The **Profiles** reference provides information about the different kinds of profile mappings available in Cosmos. These profile mappings are Airflow operators that map Airflow connections to dbt profiles, allowing you to work with resources in your data warehouses. + +- `Profile mapping <./profiles/index.html>`_ diff --git a/docs/reference/templates/index.rst.jinja2 b/docs/reference/templates/index.rst.jinja2 new file mode 100644 index 0000000000..bbc61b04ad --- /dev/null +++ b/docs/reference/templates/index.rst.jinja2 @@ -0,0 +1,12 @@ +.. + This file is autogenerated by ``docs/scripts/generate_mappings.py``. Do not edit by hand. + +Profile mapping reference +========================= + +.. toctree:: + :maxdepth: 1 + :caption: Profiles + + {% for profile in profile_mapping_names %}{{ profile }} + {% endfor %} diff --git a/docs/templates/profile_mapping.rst.jinja2 b/docs/reference/templates/profile_mapping.rst.jinja2 similarity index 96% rename from docs/templates/profile_mapping.rst.jinja2 rename to docs/reference/templates/profile_mapping.rst.jinja2 index c5b25b48b1..2154f778ed 100644 --- a/docs/templates/profile_mapping.rst.jinja2 +++ b/docs/reference/templates/profile_mapping.rst.jinja2 @@ -1,5 +1,5 @@ .. - This file is autogenerated by `docs/scripts/generate_mappings.py`. Do not edit by hand. + This file is autogenerated by ``docs/scripts/generate_mappings.py``. Do not edit by hand. {{ mapping_name }}