[DRAFT] Add Reference docs section

docs/conf.py

@@ -53,3 +53,56 @@
 }
 
 generate_mapping_docs()
+
+# -- Begin docs redirect section
+# -- To test redirects in a local build, paste the redirect source, and append .html to the end.
+# -- For example, "airflow3_compatibility/index" redirect must be tested using "airflow3_compatibility/index.html"
+# --  https://documatt.com/sphinx-reredirects/usage/
+redirects = {
+    "configuration/caching": "../optimize_performance/caching.html",
+    "configuration/cosmos-conf": "../reference/configs/cosmos-conf.html",
+    "configuration/execution-config": "../reference/configs/execution-config.html",
+    "configuration/memory_optimization": "../optimize_performance/memory_optimization.html",
+    "configuration/partial-parsing": "../optimize_performance/partial-parsing.html",
+    "configuration/profile-config": "../reference/configs/profile-config.html",
+    "configuration/project-config": "../reference/configs/project-config.html",
+    "configuration/selecting-excluding": "../optimize_performance/selecting-excluding.html",
+    "getting_started/async-execution-mode": "../guides/run_dbt/airflow-worker/async-execution-mode.html",
+    "getting_started/aws-container-run-job": "../guides/run_dbt/airflow-worker/async-execution-mode.html",
+    "getting_started/azure-container-instance": "../guides/run_dbt/container/azure-container-instance.html",
+    "getting_started/custom-airflow-properties": "../run_dbt/airflow-worker/custom-airflow-properties.html",
+    "getting_started/docker": "../guides/run_dbt/container/docker.html",
+    "getting_started/execution-modes-local-conflicts": "../guides/run_dbt/airflow-worker/execution-modes-local-conflicts.html",
+    "getting_started/execution-modes": "../guides/run_dbt/execution-modes.html",
+    "getting_started/gcp-cloud-run-job": "../guides/run_dbt/container/gcp-cloud-run-job.html",
+    "getting_started/kubernetes": "../guides/run_dbt/container/kubernetes.html",
+    "getting_started/operators": "../guides/run_dbt/operators/operators.html",
+    "getting_started/watcher-execution-mode": "../guides/run_dbt/airflow-worker/watcher-execution-mode.html",
+    "getting_started/watcher-kubernetes-execution-mode": "../guides/run_dbt/container/watcher-kubernetes-execution-mode.html",
+    "profiles/AthenaAccessKey": "../reference/profiles/AthenaAccessKey.html",
+    "profiles/ClickhouseUserPassword": "../reference/profiles/ClickhouseUserPassword.html",
+    "profiles/DatabricksOauth": "../reference/profiles/DatabricksOauth.html",
+    "profiles/DatabricksToken": "../reference/profiles/DatabricksToken.html",
+    "profiles/DuckDBUserPassword": "../reference/profiles/DuckDBUserPassword.html",
+    "profiles/ExasolUserPassword": "../reference/profiles/ExasolUserPassword.html",
+    "profiles/GoogleCloudOauth": "../reference/profiles/GoogleCloudOauth.html",
+    "profiles/GoogleCloudServiceAccountDict": "../reference/profiles/GoogleCloudServiceAccountDict.html",
+    "profiles/GoogleCloudServiceAccountFile": "../reference/profiles/GoogleCloudServiceAccountFile.html",
+    "profiles/index": "../reference/profiles/index.html",
+    "profiles/MysqlUserPassword": "../reference/profiles/MysqlUserPassword.html",
+    "profiles/OracleUserPassword": "../reference/profiles/OracleUserPassword.html",
+    "profiles/PostgresUserPassword": "../reference/profiles/PostgresUserPassword.html",
+    "profiles/RedshiftUserPassword": "../reference/profiles/RedshiftUserPassword.html",
+    "profiles/SnowflakeEncryptedPrivateKeyFilePem": "../reference/profiles/SnowflakeEncryptedPrivateKeyFilePem.html",
+    "profiles/SnowflakeEncryptedPrivateKeyPem": "../reference/profiles/SnowflakeEncryptedPrivateKeyPem.html",
+    "profiles/SnowflakePrivateKeyPem": "../reference/profiles/SnowflakePrivateKeyPem.html",
+    "profiles/SnowflakeUserPassword": "../reference/profiles/SnowflakeUserPassword.html",
+    "profiles/SparkThrift": "../reference/profiles/SparkThrift.html",
+    "profiles/StandardSQLServerAuth": "../reference/profiles/StandardSQLServerAuth.html",
+    "profiles/StarrocksUserPassword": "../reference/profiles/StarrocksUserPassword.html",
+    "profiles/TeradataUserPassword": "../reference/profiles/TeradataUserPassword.html",
+    "profiles/TrinoCertificate": "../reference/profiles/TrinoCertificate.html",
+    "profiles/TrinoJWT": "../reference/profiles/TrinoJWT.html",
+    "profiles/TrinoLDAP": "../reference/profiles/TrinoLDAP.html",
+    "profiles/VerticaUserPassword": "../reference/profiles/VerticaUserPassword.html",
+}

docs/contributing.rst

@@ -155,17 +155,19 @@ To run the checks manually, run:
 Writing Docs
 ____________
 
-`Hatch <https://hatch.pypa.io/latest/>`_ is a unified command-line tool for managing dependencies and environment isolation for Python developers. In Cosmos, we use a Hatchto declare the dependencies required for the project itself, as well as for tests and documentation builds.
+`Hatch <https://hatch.pypa.io/latest/>`_ is a unified command-line tool for managing dependencies and environment isolation for Python developers. In Cosmos, we use a Hatch to declare the dependencies required for the project itself, as well as for tests and documentation builds.
 
 If you don’t already have Hatch installed, please `install it <https://hatch.pypa.io/latest/install/>`_ before proceeding. As an example, on macOS, you can do so with:
 
 .. code-block:: bash
+
     brew install hatch
 
 
 You can run the docs locally by running the following:
 
 .. code-block:: bash
+
     hatch run docs:serve
 
 

docs/generate_mappings.py

@@ -42,8 +42,8 @@ def get_fields_from_mapping(mapping: type[BaseProfileMapping]) -> list[Field]:
 
 
 def generate_mapping_docs(
-    templates_dir: str = "./templates",
-    output_dir: str = "./profiles",
+    templates_dir: str = "./reference/templates",
+    output_dir: str = "./reference/profiles",
 ) -> None:
     """
     Generate a dedicated docs page per profile mapping.

docs/getting_started/astro.rst

@@ -1,7 +1,7 @@
 .. _astro:
 
-Getting Started on Astro
-========================
+Getting Started with Cosmos on Astro
+====================================
 
 While it is possible to use Cosmos on Astro with all :ref:`Execution Modes <execution-modes>`, we recommend using the ``local`` execution mode. It's the simplest to set up and use.
 

docs/getting_started/dbt-airflow-concepts.rst

@@ -1,7 +1,7 @@
 .. _dbt-airflow-concepts:
 
-Similar dbt & Airflow concepts
-==============================
+Similar dbt and Airflow concepts
+================================
 
 While dbt is an open source tool for data transformations and analysis, using SQL, Airflow focuses on being a platform
 for the development, scheduling and monitoring of batch-oriented workflows, using Python. Although both tools have many

docs/getting_started/index.rst

@@ -1,27 +1,28 @@
 .. _getting-started:
 
 .. toctree::
+   :maxdepth: 1
    :hidden:
-   :caption: Contents:
+   :caption: Cosmos Fundamentals
+
+   Similar dbt and Airflow concepts <dbt-airflow-concepts>
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Quickstart
 
    Astro CLI quickstart <astro-cli-quickstart>
-   Astro <astro>
-   MWAA <mwaa>
-   GCC <gcc>
-   Open-Source <open-source>
-   Execution Modes <execution-modes>
-   Docker Execution Mode <docker>
-   Kubernetes Execution Mode <kubernetes>
-   Azure Container Instance Execution Mode <azure-container-instance>
-   AWS Container Run Job Execution Mode <aws-container-run-job>
-   GCP Cloud Run Job Execution Mode <gcp-cloud-run-job>
-   Airflow Async Execution Mode <async-execution-mode>
-   Watcher Execution Mode <watcher-execution-mode>
-   Watcher Kubernetes Execution Mode <watcher-kubernetes-execution-mode>
-   dbt and Airflow Similar Concepts <dbt-airflow-concepts>
-   Operators <operators>
-   Custom Airflow Properties <custom-airflow-properties>
 
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Get started with Cosmos
+
+   Open-source Airflow <open-source>
+   Astro <astro>
+   Google Cloud Composer (GCC) <gcc>
+   Amazon Managed Workflows for Apache Airflow (MWAA) <mwaa>
 
 Getting Started
 ===============
@@ -46,11 +47,11 @@ For more customization, check out the different execution modes that Cosmos supp
 
 For specific guides, see the following:
 
-- `Executing dbt DAGs with Docker Operators <docker.html>`__
-- `Executing dbt DAGs with KubernetesPodOperators <kubernetes.html>`__
-- `Executing dbt DAGs with Watcher Kubernetes Mode <watcher-kubernetes-execution-mode.html>`__
-- `Executing dbt DAGs with AzureContainerInstancesOperators <azure-container-instance.html>`__
-- `Executing dbt DAGs with GcpCloudRunExecuteJobOperators <gcp-cloud-run-job.html>`__
+- `Executing dbt DAGs with DockerOperators <../../guides/run_dbt/container/docker.html>`__
+- `Executing dbt DAGs with KubernetesPodOperators <../../guides/run_dbt/container/kubernetes.html>`__
+- `Executing dbt DAGs with Watcher Kubernetes Mode <../../guides/run_dbt/container/watcher-kubernetes-execution-mode.html>`__
+- `Executing dbt DAGs with AzureContainerInstancesOperators <../../guides/run_dbt/container/azure-container-instance.html>`__
+- `Executing dbt DAGs with GcpCloudRunExecuteJobOperators <../../guides/run_dbt/container/gcp-cloud-run-job.html>`__
 
 
 Concepts Overview

docs/getting_started/mwaa.rst

@@ -1,7 +1,7 @@
 .. _mwaa:
 
-Getting Started on MWAA
-=======================
+Getting Started with Cosmos on Amazon Managed Workflows
+=======================================================
 
 Users can face Python dependency issues when trying to use the Cosmos `Local Execution Mode <execution-modes.html#local>`_ in Amazon Managed Workflows for `Apache Airflow® <https://airflow.apache.org/>`_ (MWAA).
 

docs/getting_started/open-source.rst

@@ -1,7 +1,7 @@
 .. _open-source:
 
-Getting Started on Open Source Airflow
-======================================
+Getting Started with Cosmos on Open-source Airflow
+==================================================
 
 When running open-source Airflow, your setup may vary. This guide assumes you have access to edit the underlying image.
 

docs/guides/cosmos_devex/index.rst

@@ -0,0 +1,14 @@
+.. _cosmos_devex:
+
+
+Cosmos DevEx
+============
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Cosmos DevEx
+
+   lineage
+   compiled-sql
+   logging
+   task-display-name

docs/guides/index.rst

@@ -0,0 +1,61 @@
+.. _guides:
+
+Guides
+======
+
+Cosmos offers a number of configuration options to customize its behavior. For more info, check out the links on the left or the table of contents below.
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Set up dbt with Airflow
+
+   dbt_setup/dbt-fusion
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Translating dbt into Airflow
+
+   translate_dbt_to_airflow/index
+
+.. toctree::
+   :maxdepth: 3
+   :hidden:
+   :caption: How Cosmos runs dbt
+
+   run_dbt/execution-modes
+   run_dbt/airflow-worker/index
+   run_dbt/container/index
+   run_dbt/callbacks/callbacks
+   run_dbt/operators/operators
+   run_dbt/customization/index
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Multi-project Setups
+
+   multi_project/multi-project
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Documentation
+
+   dbt_docs/generating-docs
+   dbt_docs/hosting-docs
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Cosmos DevEx
+
+   cosmos_devex/index
+
+.. toctree::
+   :maxdepth: 1
+   :hidden:
+   :caption: Optimizing Performance
+
+   optimize_performance/index

docs/configuration/multi-project.rst → docs/guides/multi_project/multi-project.rst

@@ -169,7 +169,7 @@ You can use either separate DAGs or a combined DAG with task groups.
 
 **Option 1: Combined DAG with Task Groups using dbt ls Load Mode (Recommended)**
 
-.. literalinclude:: ../../dev/dags/cross_project_dbt_ls_dag.py
+.. literalinclude:: ../../../dev/dags/cross_project_dbt_ls_dag.py
     :language: python
     :start-after: [START cross_project_dbt_ls_dag]
     :end-before: [END cross_project_dbt_ls_dag]
@@ -178,7 +178,7 @@ You can use either separate DAGs or a combined DAG with task groups.
 
 This option uses pre-generated ``manifest.json`` files for faster DAG parsing (no ``dbt ls`` execution required).
 
-.. literalinclude:: ../../dev/dags/cross_project_manifest_dag.py
+.. literalinclude:: ../../../dev/dags/cross_project_manifest_dag.py
     :language: python
     :start-after: [START cross_project_manifest_dag]
     :end-before: [END cross_project_manifest_dag]