[Draft] Restructure reference section

docs/conf.py

@@ -53,3 +53,56 @@
 }
 
 generate_mapping_docs()
+
+# -- Begin docs redirect section
+# -- To test redirects in a local build, paste the redirect source, and append .html to the end.
+# -- For example, "airflow3_compatibility/index" redirect must be tested using "airflow3_compatibility/index.html"
+# --  https://documatt.com/sphinx-reredirects/usage/
+redirects = {
+    "configuration/caching": "../optimize_performance/caching.html",
+    "configuration/cosmos-conf": "../reference/configs/cosmos-conf.html",
+    "configuration/execution-config": "../reference/configs/execution-config.html",
+    "configuration/memory_optimization": "../optimize_performance/memory_optimization.html",
+    "configuration/partial-parsing": "../optimize_performance/partial-parsing.html",
+    "configuration/profile-config": "../reference/profiles/index.html",
+    "configuration/project-config": "../reference/configs/project-config.html",
+    "configuration/selecting-excluding": "../optimize_performance/selecting-excluding.html",
+    "getting_started/async-execution-mode": "../guides/run_dbt/airflow-worker/async-execution-mode.html",
+    "getting_started/aws-container-run-job": "../guides/run_dbt/airflow-worker/async-execution-mode.html",
+    "getting_started/azure-container-instance": "../guides/run_dbt/container/azure-container-instance.html",
+    "getting_started/custom-airflow-properties": "../run_dbt/airflow-worker/custom-airflow-properties.html",
+    "getting_started/docker": "../guides/run_dbt/container/docker.html",
+    "getting_started/execution-modes-local-conflicts": "../guides/run_dbt/airflow-worker/execution-modes-local-conflicts.html",
+    "getting_started/execution-modes": "../guides/run_dbt/execution-modes.html",
+    "getting_started/gcp-cloud-run-job": "../guides/run_dbt/container/gcp-cloud-run-job.html",
+    "getting_started/kubernetes": "../guides/run_dbt/container/kubernetes.html",
+    "getting_started/operators": "../guides/run_dbt/operators/operators.html",
+    "getting_started/watcher-execution-mode": "../guides/run_dbt/airflow-worker/watcher-execution-mode.html",
+    "getting_started/watcher-kubernetes-execution-mode": "../guides/run_dbt/container/watcher-kubernetes-execution-mode.html",
+    "profiles/AthenaAccessKey": "../reference/profiles/AthenaAccessKey.html",
+    "profiles/ClickhouseUserPassword": "../reference/profiles/ClickhouseUserPassword.html",
+    "profiles/DatabricksOauth": "../reference/profiles/DatabricksOauth.html",
+    "profiles/DatabricksToken": "../reference/profiles/DatabricksToken.html",
+    "profiles/DuckDBUserPassword": "../reference/profiles/DuckDBUserPassword.html",
+    "profiles/ExasolUserPassword": "../reference/profiles/ExasolUserPassword.html",
+    "profiles/GoogleCloudOauth": "../reference/profiles/GoogleCloudOauth.html",
+    "profiles/GoogleCloudServiceAccountDict": "../reference/profiles/GoogleCloudServiceAccountDict.html",
+    "profiles/GoogleCloudServiceAccountFile": "../reference/profiles/GoogleCloudServiceAccountFile.html",
+    "profiles/index": "../reference/profiles/index.html",
+    "profiles/MysqlUserPassword": "../reference/profiles/MysqlUserPassword.html",
+    "profiles/OracleUserPassword": "../reference/profiles/OracleUserPassword.html",
+    "profiles/PostgresUserPassword": "../reference/profiles/PostgresUserPassword.html",
+    "profiles/RedshiftUserPassword": "../reference/profiles/RedshiftUserPassword.html",
+    "profiles/SnowflakeEncryptedPrivateKeyFilePem": "../reference/profiles/SnowflakeEncryptedPrivateKeyFilePem.html",
+    "profiles/SnowflakeEncryptedPrivateKeyPem": "../reference/profiles/SnowflakeEncryptedPrivateKeyPem.html",
+    "profiles/SnowflakePrivateKeyPem": "../reference/profiles/SnowflakePrivateKeyPem.html",
+    "profiles/SnowflakeUserPassword": "../reference/profiles/SnowflakeUserPassword.html",
+    "profiles/SparkThrift": "../reference/profiles/SparkThrift.html",
+    "profiles/StandardSQLServerAuth": "../reference/profiles/StandardSQLServerAuth.html",
+    "profiles/StarrocksUserPassword": "../reference/profiles/StarrocksUserPassword.html",
+    "profiles/TeradataUserPassword": "../reference/profiles/TeradataUserPassword.html",
+    "profiles/TrinoCertificate": "../reference/profiles/TrinoCertificate.html",
+    "profiles/TrinoJWT": "../reference/profiles/TrinoJWT.html",
+    "profiles/TrinoLDAP": "../reference/profiles/TrinoLDAP.html",
+    "profiles/VerticaUserPassword": "../reference/profiles/VerticaUserPassword.html",
+}

docs/contributing.rst

@@ -155,17 +155,19 @@ To run the checks manually, run:
 Writing Docs
 ____________
 
-`Hatch <https://hatch.pypa.io/latest/>`_ is a unified command-line tool for managing dependencies and environment isolation for Python developers. In Cosmos, we use a Hatchto declare the dependencies required for the project itself, as well as for tests and documentation builds.
+`Hatch <https://hatch.pypa.io/latest/>`_ is a unified command-line tool for managing dependencies and environment isolation for Python developers. In Cosmos, we use a Hatch to declare the dependencies required for the project itself, as well as for tests and documentation builds.
 
 If you don’t already have Hatch installed, please `install it <https://hatch.pypa.io/latest/install/>`_ before proceeding. As an example, on macOS, you can do so with:
 
 .. code-block:: bash
+
     brew install hatch
 
 
 You can run the docs locally by running the following:
 
 .. code-block:: bash
+
     hatch run docs:serve
 
 

docs/generate_mappings.py

@@ -42,8 +42,8 @@ def get_fields_from_mapping(mapping: type[BaseProfileMapping]) -> list[Field]:
 
 
 def generate_mapping_docs(
-    templates_dir: str = "./templates",
-    output_dir: str = "./profiles",
+    templates_dir: str = "./reference/templates",
+    output_dir: str = "./reference/profiles",
 ) -> None:
     """
     Generate a dedicated docs page per profile mapping.

docs/getting_started/astro.rst

@@ -1,7 +1,7 @@
 .. _astro:
 
-Getting Started on Astro
-========================
+Getting Started with Cosmos on Astro
+====================================
 
 While it is possible to use Cosmos on Astro with all :ref:`Execution Modes <execution-modes>`, we recommend using the ``local`` execution mode. It's the simplest to set up and use.
 

docs/getting_started/dbt-airflow-concepts.rst

@@ -1,7 +1,7 @@
 .. _dbt-airflow-concepts:
 
-Similar dbt & Airflow concepts
-==============================
+Similar dbt and Airflow concepts
+================================
 
 While dbt is an open source tool for data transformations and analysis, using SQL, Airflow focuses on being a platform
 for the development, scheduling and monitoring of batch-oriented workflows, using Python. Although both tools have many

docs/getting_started/index.rst

@@ -1,27 +1,28 @@
 .. _getting-started:
 
 .. toctree::
+   :maxdepth: 1
    :hidden:
-   :caption: Contents:
+   :caption: Cosmos Fundamentals
+
+   Similar dbt and Airflow concepts <dbt-airflow-concepts>
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Quickstart
 
    Astro CLI quickstart <astro-cli-quickstart>
-   Astro <astro>
-   MWAA <mwaa>
-   GCC <gcc>
-   Open-Source <open-source>
-   Execution Modes <execution-modes>
-   Docker Execution Mode <docker>
-   Kubernetes Execution Mode <kubernetes>
-   Azure Container Instance Execution Mode <azure-container-instance>
-   AWS Container Run Job Execution Mode <aws-container-run-job>
-   GCP Cloud Run Job Execution Mode <gcp-cloud-run-job>
-   Airflow Async Execution Mode <async-execution-mode>
-   Watcher Execution Mode <watcher-execution-mode>
-   Watcher Kubernetes Execution Mode <watcher-kubernetes-execution-mode>
-   dbt and Airflow Similar Concepts <dbt-airflow-concepts>
-   Operators <operators>
-   Custom Airflow Properties <custom-airflow-properties>
 
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Get started with Cosmos
+
+   Open-source Airflow <open-source>
+   Astro <astro>
+   Google Cloud Composer (GCC) <gcc>
+   Amazon Managed Workflows for Apache Airflow (MWAA) <mwaa>
 
 Getting Started
 ===============
@@ -46,11 +47,11 @@ For more customization, check out the different execution modes that Cosmos supp
 
 For specific guides, see the following:
 
-- `Executing dbt DAGs with Docker Operators <docker.html>`__
-- `Executing dbt DAGs with KubernetesPodOperators <kubernetes.html>`__
-- `Executing dbt DAGs with Watcher Kubernetes Mode <watcher-kubernetes-execution-mode.html>`__
-- `Executing dbt DAGs with AzureContainerInstancesOperators <azure-container-instance.html>`__
-- `Executing dbt DAGs with GcpCloudRunExecuteJobOperators <gcp-cloud-run-job.html>`__
+- `Executing dbt DAGs with DockerOperators <../../guides/run_dbt/container/docker.html>`__
+- `Executing dbt DAGs with KubernetesPodOperators <../../guides/run_dbt/container/kubernetes.html>`__
+- `Executing dbt DAGs with Watcher Kubernetes Mode <../../guides/run_dbt/container/watcher-kubernetes-execution-mode.html>`__
+- `Executing dbt DAGs with AzureContainerInstancesOperators <../../guides/run_dbt/container/azure-container-instance.html>`__
+- `Executing dbt DAGs with GcpCloudRunExecuteJobOperators <../../guides/run_dbt/container/gcp-cloud-run-job.html>`__
 
 
 Concepts Overview

docs/getting_started/mwaa.rst

@@ -1,7 +1,7 @@
 .. _mwaa:
 
-Getting Started on MWAA
-=======================
+Getting Started with Cosmos on Amazon Managed Workflows
+=======================================================
 
 Users can face Python dependency issues when trying to use the Cosmos `Local Execution Mode <execution-modes.html#local>`_ in Amazon Managed Workflows for `Apache Airflow® <https://airflow.apache.org/>`_ (MWAA).
 

docs/getting_started/open-source.rst

@@ -1,7 +1,7 @@
 .. _open-source:
 
-Getting Started on Open Source Airflow
-======================================
+Getting Started with Cosmos on Open-source Airflow
+==================================================
 
 When running open-source Airflow, your setup may vary. This guide assumes you have access to edit the underlying image.
 

docs/guides/cosmos_devex/index.rst

@@ -0,0 +1,14 @@
+.. _cosmos_devex:
+
+
+Cosmos DevEx
+============
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Cosmos DevEx
+
+   lineage
+   compiled-sql
+   logging
+   task-display-name

docs/guides/index.rst

@@ -0,0 +1,54 @@
+.. _guides:
+
+Guides
+======
+
+Cosmos offers a number of configuration options to customize its behavior. For more info, check out the links on the left or the table of contents below.
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Set up dbt with Airflow
+
+   dbt_setup/dbt-fusion
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Translating dbt into Airflow
+
+   translate_dbt_to_airflow/index
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+   :caption: How Cosmos runs dbt
+
+   run_dbt/execution-modes
+   run_dbt/airflow-worker/index
+   run_dbt/container/index
+   run_dbt/callbacks/callbacks
+   run_dbt/operators/operators
+   run_dbt/customization/index
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Multi-project Setups
+
+   multi_project/multi-project
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Documentation
+
+   dbt_docs/generating-docs
+   dbt_docs/hosting-docs
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Cosmos DevEx
+
+   cosmos_devex/index

docs/configuration/multi-project.rst → docs/guides/multi_project/multi-project.rst

@@ -169,7 +169,7 @@ You can use either separate DAGs or a combined DAG with task groups.
 
 **Option 1: Combined DAG with Task Groups using dbt ls Load Mode (Recommended)**
 
-.. literalinclude:: ../../dev/dags/cross_project_dbt_ls_dag.py
+.. literalinclude:: ../../../dev/dags/cross_project_dbt_ls_dag.py
     :language: python
     :start-after: [START cross_project_dbt_ls_dag]
     :end-before: [END cross_project_dbt_ls_dag]
@@ -178,7 +178,7 @@ You can use either separate DAGs or a combined DAG with task groups.
 
 This option uses pre-generated ``manifest.json`` files for faster DAG parsing (no ``dbt ls`` execution required).
 
-.. literalinclude:: ../../dev/dags/cross_project_manifest_dag.py
+.. literalinclude:: ../../../dev/dags/cross_project_manifest_dag.py
     :language: python
     :start-after: [START cross_project_manifest_dag]
     :end-before: [END cross_project_manifest_dag]

docs/getting_started/async-execution-mode.rst → docs/guides/run_dbt/airflow-worker/async-execution-mode.rst

@@ -1,7 +1,5 @@
 .. _async-execution-mode:
 
-.. title:: Getting Started with Deferrable Operator
-
 Airflow Async Execution Mode
 ============================
 

docs/getting_started/execution-modes-local-conflicts.rst → docs/guides/run_dbt/airflow-worker/execution-modes-local-conflicts.rst

@@ -10,8 +10,8 @@ When using the `Local Execution Mode <execution-modes.html#local>`__, users may
 
 If you find errors, we recommend users isolating the installation of dbt from the Airflow installation.
 With the `Local Execution Mode <execution-modes.html#local>`__, this can be accomplished by installing dbt in a separate
-Python virtualenv and setting the `ExecutionConfig.dbt_executable_path <../configuration/execution-config.html>`_  and
-`RenderConfig.dbt_executable_path <../configuration/render-config.html>`_ parameters.
+Python virtualenv and setting the `ExecutionConfig.dbt_executable_path <../guides/execution-config.html>`_  and
+`RenderConfig.dbt_executable_path <../guides/render-config.html>`_ parameters.
 
 The page `execution modes <execution-modes.html>`__ describes many other methods that support isolating dbt from Airflow.
 

docs/guides/run_dbt/airflow-worker/index.rst

@@ -0,0 +1,9 @@
+Run dbt in an Airflow worker
+============================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Run dbt in an Airflow worker
+
+   async-execution-mode
+   watcher-execution-mode

docs/getting_started/watcher-execution-mode.rst → docs/guides/run_dbt/airflow-worker/watcher-execution-mode.rst

@@ -144,7 +144,7 @@ Example 1 — Using ``DbtDag`` with ``ExecutionMode.WATCHER``
 You can enable WATCHER mode directly in your ``DbtDag`` configuration.
 This approach is best when your Airflow DAG is fully dedicated to a dbt project.
 
-.. literalinclude:: ../../dev/dags/example_watcher.py
+.. literalinclude:: ../../../../dev/dags/example_watcher.py
     :language: python
     :start-after: [START example_watcher]
     :end-before: [END example_watcher]
@@ -370,7 +370,7 @@ Source freshness nodes
 
 Since Cosmos 1.6, it `supports the rendering of source nodes <https://www.astronomer.io/blog/native-support-for-source-node-rendering-in-cosmos/>`_.
 
-We noticed some Cosmos users use this feature alongside `overriding Cosmos source nodes <https://astronomer.github.io/astronomer-cosmos/configuration/render-config.html#customizing-how-nodes-are-rendered-experimental>`_ as sensors or another operator that allows them to skip the following branch of the DAG if the source is not fresh.
+We noticed some Cosmos users use this feature alongside `overriding Cosmos source nodes <https://astronomer.github.io/astronomer-cosmos/guides/render-config.html#customizing-how-nodes-are-rendered-experimental>`_ as sensors or another operator that allows them to skip the following branch of the DAG if the source is not fresh.
 
 This use case is not currently supported by the ``ExecutionMode.WATCHER``, since the ``dbt build`` command does not run `source freshness checks <https://docs.getdbt.com/reference/commands/build#source-freshness-checks>`_.
 
@@ -451,7 +451,7 @@ Asynchronous sensor execution
 
 To disable asynchronous execution, set the ``deferrable`` flag to ``False`` in the ``operator_args``.
 
-.. literalinclude:: ../../dev/dags/example_watcher.py
+.. literalinclude:: ../../../../dev/dags/example_watcher.py
    :language: python
    :start-after: [START example_watcher_synchronous]
    :end-before: [END example_watcher_synchronous]

docs/configuration/callbacks.rst → docs/guides/run_dbt/callbacks/callbacks.rst

@@ -34,7 +34,7 @@ Example: Using Callbacks with a Single Operator
 To demonstrate how to specify a callback function for uploading files from the target directory, here’s an example
 using a single operator in an Airflow DAG:
 
-.. literalinclude:: ../../dev/dags/example_operators.py
+.. literalinclude:: ../../../../dev/dags/example_operators.py
     :language: python
     :start-after: [START single_operator_callback]
     :end-before: [END single_operator_callback]
@@ -46,7 +46,7 @@ You can leverage the :ref:`remote_target_path` configuration to upload files
 from the target directory to a remote storage. Below is an example of how to define a callback helper function in your
 ``DbtDag`` that utilizes this configuration:
 
-.. literalinclude:: ../../dev/dags/cosmos_callback_dag.py
+.. literalinclude:: ../../../../dev/dags/cosmos_callback_dag.py
     :language: python
     :start-after: [START cosmos_callback_example]
     :end-before: [END cosmos_callback_example]