Reference documentation restructure + refactor

.gitignore

@@ -1,6 +1,6 @@
 # cosmos-specific ignores
 # these files get autogenerated
-docs/profiles/*
+docs/reference/profiles/*
 
 # actionlint binary (downloaded when running validate-workflows or locally)
 actionlint

CHANGELOG.rst

@@ -1083,7 +1083,7 @@ Features
 * Add cosmos/propagate_logs Airflow config support for disabling log propagation by @agreenburg in #648 (`documentation <https://astronomer.github.io/astronomer-cosmos/configuration/logging.html>`_).
 * Add operator_args ``full_refresh`` as a templated field by @joppevos in #623
 * Expose environment variables and dbt variables in ``ProjectConfig`` by @jbandoro in #735 (`documentation <https://astronomer.github.io/astronomer-cosmos/configuration/project-config.html#project-config-example>`_).
-* Support disabling event tracking when using Cosmos profile mapping by @jbandoro in #768 (`documentation <https://astronomer.github.io/astronomer-cosmos/profiles/index.html#disabling-dbt-event-tracking>`_).
+* Support disabling event tracking when using Cosmos profile mapping by @jbandoro in #768 (`documentation <https://astronomer.github.io/astronomer-cosmos/guides/connect_database/profile-customise-per-node.html#disabling-dbt-event-trackingg>`_).
 
 Enhancements
 

docs/conf.py

@@ -56,7 +56,7 @@
 
 # -- Begin docs redirect section
 # -- To test redirects in a local build, paste the redirect source, and append .html to the end.
-# -- For example, "airflow3_compatibility/index" redirect must be tested using "airflow3_compatibility/index.html"
+# -- For example, "airflow3_compatibility/index" source redirect must be tested using "airflow3_compatibility/index.html"
 # --  https://documatt.com/sphinx-reredirects/usage/
 redirects = {
     "airflow3_compatibility/index": "../policy/airflow3-compatibility.html",
@@ -77,7 +77,7 @@
     "configuration/operator-args": "../guides/run_dbt/customization/operator-args.html",
     "configuration/parsing-methods": "../guides/translate_dbt_to_airflow/parsing-methods.html",
     "configuration/partial-parsing": "../guides/run_dbt/customization/partial-parsing.html",
-    "configuration/profile-config": "../reference/configs/profile-config.html",
+    "configuration/profile-config": "../guides/connect_database/index.html",
     "configuration/project-config": "../reference/configs/project-config.html",
     "configuration/render-config": "../guides/translate_dbt_to_airflow/render-config.html",
     "configuration/scheduling": "../guides/run_dbt/customization/scheduling.html",
@@ -100,4 +100,30 @@
     "getting_started/operators": "../guides/run_dbt/operators/operators.html",
     "getting_started/watcher-execution-mode": "../guides/run_dbt/airflow-worker/watcher-execution-mode.html",
     "getting_started/watcher-kubernetes-execution-mode": "../guides/run_dbt/container/watcher-kubernetes-execution-mode.html",
+    "profiles/AthenaAccessKey": "../reference/profiles/AthenaAccessKey.html",
+    "profiles/ClickhouseUserPassword": "../reference/profiles/ClickhouseUserPassword.html",
+    "profiles/DatabricksOauth": "../reference/profiles/DatabricksOauth.html",
+    "profiles/DatabricksToken": "../reference/profiles/DatabricksToken.html",
+    "profiles/DuckDBUserPassword": "../reference/profiles/DuckDBUserPassword.html",
+    "profiles/ExasolUserPassword": "../reference/profiles/ExasolUserPassword.html",
+    "profiles/GoogleCloudOauth": "../reference/profiles/GoogleCloudOauth.html",
+    "profiles/GoogleCloudServiceAccountDict": "../reference/profiles/GoogleCloudServiceAccountDict.html",
+    "profiles/GoogleCloudServiceAccountFile": "../reference/profiles/GoogleCloudServiceAccountFile.html",
+    "profiles/index": "../reference/profiles/index.html",
+    "profiles/MysqlUserPassword": "../reference/profiles/MysqlUserPassword.html",
+    "profiles/OracleUserPassword": "../reference/profiles/OracleUserPassword.html",
+    "profiles/PostgresUserPassword": "../reference/profiles/PostgresUserPassword.html",
+    "profiles/RedshiftUserPassword": "../reference/profiles/RedshiftUserPassword.html",
+    "profiles/SnowflakeEncryptedPrivateKeyFilePem": "../reference/profiles/SnowflakeEncryptedPrivateKeyFilePem.html",
+    "profiles/SnowflakeEncryptedPrivateKeyPem": "../reference/profiles/SnowflakeEncryptedPrivateKeyPem.html",
+    "profiles/SnowflakePrivateKeyPem": "../reference/profiles/SnowflakePrivateKeyPem.html",
+    "profiles/SnowflakeUserPassword": "../reference/profiles/SnowflakeUserPassword.html",
+    "profiles/SparkThrift": "../reference/profiles/SparkThrift.html",
+    "profiles/StandardSQLServerAuth": "../reference/profiles/StandardSQLServerAuth.html",
+    "profiles/StarrocksUserPassword": "../reference/profiles/StarrocksUserPassword.html",
+    "profiles/TeradataUserPassword": "../reference/profiles/TeradataUserPassword.html",
+    "profiles/TrinoCertificate": "../reference/profiles/TrinoCertificate.html",
+    "profiles/TrinoJWT": "../reference/profiles/TrinoJWT.html",
+    "profiles/TrinoLDAP": "../reference/profiles/TrinoLDAP.html",
+    "profiles/VerticaUserPassword": "../reference/profiles/VerticaUserPassword.html",
 }

docs/generate_mappings.py

@@ -42,8 +42,8 @@ def get_fields_from_mapping(mapping: type[BaseProfileMapping]) -> list[Field]:
 
 
 def generate_mapping_docs(
-    templates_dir: str = "./templates",
-    output_dir: str = "./profiles",
+    templates_dir: str = "./reference/templates",
+    output_dir: str = "./reference/profiles",
 ) -> None:
     """
     Generate a dedicated docs page per profile mapping.

docs/guides/connect_database/index.rst

@@ -0,0 +1,34 @@
+.. _connect_database:
+
+Connect to your database
+===============================
+
+Cosmos supports two methods of authenticating with your database:
+
+- using your own `dbt profiles.yml <../../guides/connect_database/use-your-profiles-yml.html>`_ file
+- using Airflow connections via Cosmos' `profile mappings <../../guides/connect_database/use-profile-mapping.html>`_
+
+If you're already interacting with your database from Airflow and have a connection set up, it's recommended
+to use a profile mapping to translate that Airflow connection to a dbt profile. This is because it's easier to
+maintain a single connection object in Airflow than it is to maintain a connection object in Airflow and a dbt profile
+in your dbt project.
+
+If you don't already have an Airflow connection, or if there's no readily-available profile mapping for your database,
+you can use your own dbt profiles.yml file.
+
+Regardless of which method you use, you'll need to tell Cosmos which profile and target name it should use. Profile config
+is set in the ``cosmos.config.ProfileConfig`` object, like so:
+
+.. code-block:: python
+
+    from cosmos.config import ProfileConfig
+
+    profile_config = ProfileConfig(
+        profile_name="my_profile_name",
+        target_name="my_target_name",
+        # choose one of the following
+        profile_mapping=...,
+        profiles_yml_filepath=...,
+    )
+
+    dag = DbtDag(profile_config=profile_config, ...)

docs/templates/index.rst.jinja2 → docs/guides/connect_database/profile-customise-per-node.rst

@@ -1,113 +1,8 @@
-..
-  This file is autogenerated by `docs/scripts/generate_mappings.py`. Do not edit by hand.
-
-.. toctree::
-   :caption: Profiles
-
-   self
-   {% for profile in profile_mapping_names %}{{ profile }}
-   {% endfor %}
-
-Profiles Overview
-==================
-
-Cosmos supports two methods of authenticating with your database:
-
-- using your own dbt profiles.yml file
-- using Airflow connections via Cosmos' profile mappings
-
-If you're already interacting with your database from Airflow and have a connection set up, it's recommended
-to use a profile mapping to translate that Airflow connection to a dbt profile. This is because it's easier to
-maintain a single connection object in Airflow than it is to maintain a connection object in Airflow and a dbt profile
-in your dbt project.
-
-If you don't already have an Airflow connection, or if there's no readily-available profile mapping for your database,
-you can use your own dbt profiles.yml file.
-
-Regardless of which method you use, you'll need to tell Cosmos which profile and target name it should use. Profile config
-is set in the ``cosmos.config.ProfileConfig`` object, like so:
-
-.. code-block:: python
-
-    from cosmos.config import ProfileConfig
-
-    profile_config = ProfileConfig(
-        profile_name="my_profile_name",
-        target_name="my_target_name",
-
-        # choose one of the following
-        profile_mapping=...,
-        profiles_yml_filepath=...,
-    )
-
-    dag = DbtDag(profile_config=profile_config, ...)
-
-
-Using your own profiles.yml file
-++++++++++++++++++++++++++++++++
-
-If you don't want to use Airflow connections, or if there's no readily-available profile mapping for your database,
-you can use your own dbt profiles.yml file. To do so, you'll need to pass the path to your profiles.yml file to the
-``profiles_yml_filepath`` argument in ``ProfileConfig``.
-
-For example, the code snippet below points Cosmos at a ``profiles.yml`` file and instructs Cosmos to use the
-``my_snowflake_profile`` profile and ``dev`` target:
-
-.. code-block:: python
-
-    from cosmos.config import ProfileConfig
-
-    profile_config = ProfileConfig(
-        profile_name="my_snowflake_profile",
-        target_name="dev",
-        profiles_yml_filepath="/path/to/profiles.yml",
-    )
-
-    dag = DbtDag(profile_config=profile_config, ...)
-
-Using a profile mapping
-+++++++++++++++++++++++
-
-Profile mappings are utilities provided by Cosmos that translate Airflow connections to dbt profiles. This means that
-you can use the same connection objects you use in Airflow to authenticate with your database in dbt. To do so, there's
-a class in Cosmos for each Airflow connection to dbt profile mapping.
-
-You can find the available profile mappings on the left-hand side of this page. Each profile mapping is imported from
-``cosmos.profiles`` and takes two arguments:
-
-* ``conn_id``: the Airflow connection ID to use.
-* ``profile_args``: a dictionary of additional arguments to pass to the dbt profile. This is useful for specifying
-  values that are not in the Airflow connection. This also acts as an override for any values that are in the Airflow
-  connection but should be overridden.
-
-Below is an example of using the Snowflake profile mapping, where we take most arguments from the Airflow connection
-but override the ``database`` and ``schema`` values:
-
-.. code-block:: python
-
-    from cosmos.profiles import SnowflakeUserPasswordProfileMapping
-
-    profile_config = ProfileConfig(
-        profile_name="my_profile_name",
-        target_name="my_target_name",
-        profile_mapping=SnowflakeUserPasswordProfileMapping(
-            conn_id="my_snowflake_conn_id",
-            profile_args={
-                "database": "my_snowflake_database",
-                "schema": "my_snowflake_schema",
-            },
-        ),
-    )
-
-    dag = DbtDag(profile_config=profile_config, ...)
-
-Note that when using a profile mapping, the profiles.yml file gets generated with the profile name and target name
-you specify in ``ProfileConfig``.
-
 .. _profile-customise-per-node:
 
 Customising the profile config per dbt node
-+++++++++++++++++++++++++++++++++++++++++++
+===========================================
+
 .. versionadded:: 1.9.0
 
 
@@ -184,7 +79,7 @@ The parts of ``profiles.yml``, which aren't specific to a particular data platfo
                 write_json=True,
                 warn_error=True,
                 warn_error_options={"include": "all"},
-                log_format='text',
+                log_format="text",
                 debug=True,
                 version_check=True,
             ),
@@ -195,7 +90,7 @@ The parts of ``profiles.yml``, which aren't specific to a particular data platfo
 
 
 Disabling dbt event tracking
-++++++++++++++++++++++++++++
+----------------------------
 
 .. note:
    Deprecated in v.1.4 and will be removed in v2.0.0. Use dbt_config_vars=DbtProfileConfigVars(send_anonymous_usage_stats=False) instead.

docs/guides/connect_database/use-profile-mapping.rst

@@ -0,0 +1,40 @@
+.. _use-profile-mapping:
+
+Use a profile mapping
+=======================
+
+Profile mappings are utilities provided by Cosmos that translate Airflow connections to dbt profiles. This means that
+you can use the same connection objects you use in Airflow to authenticate with your database in dbt. To do so, there's
+a class in Cosmos for each Airflow connection to dbt profile mapping.
+
+You can find the available profile mappings on the left-hand side of this page. Each profile mapping is imported from
+``cosmos.profiles`` and takes two arguments:
+
+* ``conn_id``: the Airflow connection ID to use.
+* ``profile_args``: a dictionary of additional arguments to pass to the dbt profile. This is useful for specifying
+  values that are not in the Airflow connection. This also acts as an override for any values that are in the Airflow
+  connection but should be overridden.
+
+Below is an example of using the Snowflake profile mapping, where we take most arguments from the Airflow connection
+but override the ``database`` and ``schema`` values:
+
+.. code-block:: python
+
+    from cosmos.profiles import SnowflakeUserPasswordProfileMapping
+
+    profile_config = ProfileConfig(
+        profile_name="my_profile_name",
+        target_name="my_target_name",
+        profile_mapping=SnowflakeUserPasswordProfileMapping(
+            conn_id="my_snowflake_conn_id",
+            profile_args={
+                "database": "my_snowflake_database",
+                "schema": "my_snowflake_schema",
+            },
+        ),
+    )
+
+    dag = DbtDag(profile_config=profile_config, ...)
+
+Note that when using a profile mapping, the profiles.yml file gets generated with the profile name and target name
+you specify in ``ProfileConfig``.

docs/guides/connect_database/use-your-profiles-yml.rst

@@ -0,0 +1,23 @@
+.. _use-your-profiles-yml:
+
+Use your own profiles.yml file
+==============================
+
+If you don't want to use Airflow connections, or if there's no readily-available profile mapping for your database,
+you can use your own dbt profiles.yml file. To do so, you'll need to pass the path to your profiles.yml file to the
+``profiles_yml_filepath`` argument in ``ProfileConfig``.
+
+For example, the code snippet below points Cosmos at a ``profiles.yml`` file and instructs Cosmos to use the
+``my_snowflake_profile`` profile and ``dev`` target:
+
+.. code-block:: python
+
+    from cosmos.config import ProfileConfig
+
+    profile_config = ProfileConfig(
+        profile_name="my_snowflake_profile",
+        target_name="dev",
+        profiles_yml_filepath="/path/to/profiles.yml",
+    )
+
+    dag = DbtDag(profile_config=profile_config, ...)

docs/guides/index.rst

@@ -12,6 +12,16 @@ Cosmos offers a number of configuration options to customize its behavior. For m
 
    dbt_setup/dbt-fusion
 
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Connect to your database
+
+   Connection options <connect_database/index>
+   connect_database/use-your-profiles-yml
+   connect_database/use-profile-mapping
+   connect_database/profile-customise-per-node
+
 .. toctree::
    :maxdepth: 1
    :hidden:

docs/guides/run_dbt/container/kubernetes.rst

@@ -160,8 +160,8 @@ The Kubernetes execution mode has the following limitations:
 - Does not emit OpenLineage events (there is an `open ticket #496 <https://github.com/astronomer/astronomer-cosmos/issues/496>`__ to address this)
 - Does not emit Airflow datasets, assets, and dataset aliases (there is an `open ticket #2329 <https://github.com/astronomer/astronomer-cosmos/issues/2329>`__ to address this)
 - Does not handle installing dbt deps for users (there is an `open ticket #679 <https://github.com/astronomer/astronomer-cosmos/issues/679>`__ to address this)
-- Does not support `ProfileMapping <https://astronomer.github.io/astronomer-cosmos/profiles/index.html#using-a-profile-mapping>`_ (there is an `open ticket #749 <https://github.com/astronomer/astronomer-cosmos/issues/749>`__ to address this)
-- Does not support `Callbacks <https://astronomer.github.io/astronomer-cosmos/guides/callbacks.html>`_ (there is an `open ticket #1575 <https://github.com/astronomer/astronomer-cosmos/issues/1575>`__ to address this)
-- Does not expose Compiled SQL as a `templated field <https://astronomer.github.io/astronomer-cosmos/guides/compiled-sql.html>`_
-- Does not benefit from `Cosmos caching mechanisms <https://astronomer.github.io/astronomer-cosmos/guides/caching.html>`_
-- Does not support `generating dbt docs & uploading to an object store <https://astronomer.github.io/astronomer-cosmos/guides/generating-docs.html>`_ (there is a `PR <https://github.com/astronomer/astronomer-cosmos/pull/2058>`_ to solve this for S3)
+- Does not support `ProfileMapping <https://astronomer.github.io/astronomer-cosmos/guides/connect_database/use-profile-mapping.html>`_ (there is an `open ticket #749 <https://github.com/astronomer/astronomer-cosmos/issues/749>`__ to address this)
+- Does not support `Callbacks <https://astronomer.github.io/astronomer-cosmos/guides/callbacks/callbacks.html>`_ (there is an `open ticket #1575 <https://github.com/astronomer/astronomer-cosmos/issues/1575>`__ to address this)
+- Does not expose Compiled SQL as a `templated field <https://astronomer.github.io/astronomer-cosmos/guides/cosmos_devex/compiled-sql.html>`_
+- Does not benefit from `Cosmos caching mechanisms <https://astronomer.github.io/astronomer-cosmos/optimize_performance/caching.html>`_
+- Does not support `generating dbt docs & uploading to an object store <https://astronomer.github.io/astronomer-cosmos/guides/dbt_docs/generating-docs.html>`_ (there is a `PR <https://github.com/astronomer/astronomer-cosmos/pull/2058>`_ to solve this for S3)

docs/index.rst

@@ -9,7 +9,6 @@
    Getting Started <getting_started/index>
    Guides <guides/index>
    Optimize Performance <optimize_performance/index>
-   Profiles <profiles/index>
    Reference <reference/index>
    Project Policies <policy/index>
 

docs/reference/configs/profiles-config.rst

@@ -0,0 +1,11 @@
+Profiles Config
+================
+
+Cosmos supports two methods of authenticating with your database:
+
+- Using your `dbt profiles.yml <../../guides/connect_database/use-your-profiles-yml.html>`_ file
+- using Airflow connections with the Cosmos `profile_mapping <../../guides/connect_database/use-profile-mapping.html>`_
+
+You can also customize the profiles used in each `dbt node <../../guides/connect_database/profile-customise-per-node.html>`_.
+
+The `Profiles reference <../profiles/index.html>`_ contains information about the different profiles available for you to use in your database connections.