From 4c32700f2fead8ab492bc5de4d132aacc854c3c8 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 14 Jan 2025 09:12:52 -0500
Subject: [PATCH 01/13] WIP

---
 docs/README.md                          |  43 +++++++++
 docs/_static/custom.css                 |  32 +++++++
 docs/conf/conf.py                       |  78 ++++++++++++++++
 docs/dev-guide/index.md                 |   0
 docs/{ => dev-guide}/testing.md         |   2 +-
 docs/index.md                           |   9 ++
 docs/requirements.txt                   |   6 ++
 docs/tasks.yaml                         | 116 ++++++++++++++++++++++++
 docs/user-guide/index.md                |   0
 docs/{ => user-guide}/quick-start.md    |   0
 taskfile.yaml                           |   3 +-
 tools/scripts/find-broken-docs-links.py | 105 +++++++++++++++++++++
 12 files changed, 392 insertions(+), 2 deletions(-)
 create mode 100644 docs/README.md
 create mode 100644 docs/_static/custom.css
 create mode 100644 docs/conf/conf.py
 create mode 100644 docs/dev-guide/index.md
 rename docs/{ => dev-guide}/testing.md (97%)
 create mode 100644 docs/index.md
 create mode 100644 docs/requirements.txt
 create mode 100644 docs/tasks.yaml
 create mode 100644 docs/user-guide/index.md
 rename docs/{ => user-guide}/quick-start.md (100%)
 create mode 100644 tools/scripts/find-broken-docs-links.py

diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 000000000..4458f6d8e
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,43 @@
+# Docs
+
+This directory contains the files necessary to generate a Sphinx-based documentation website for
+this project:
+
+* `conf` - Configuration files
+* `src` - The actual docs
+
+## Requirements
+
+* [Node.js] >= 16 to be able to [view the output](#viewing-the-output)
+* Python 3.10 or higher
+* [Task] 3.40.0 or higher
+
+## Build commands
+
+* Build the site incrementally:
+
+  ```shell
+  task docs:site
+  ```
+
+    * The output of the build will be in `../build/docs/html`.
+
+* Clean up the build:
+
+  ```shell
+  task docs:clean
+  ```
+
+## Viewing the output
+
+```shell
+task docs:serve
+```
+
+The command above will install [http-server] and serve the built docs site; `http-server` will print
+the address it binds to (usually http://localhost:8080).
+
+[git-lfs]: https://git-lfs.com
+[http-server]: https://www.npmjs.com/package/http-server
+[Node.js]: https://nodejs.org/en/download/current
+[Task]: https://taskfile.dev/
diff --git a/docs/_static/custom.css b/docs/_static/custom.css
new file mode 100644
index 000000000..f25a6694b
--- /dev/null
+++ b/docs/_static/custom.css
@@ -0,0 +1,32 @@
+html[data-theme="dark"], html[data-theme="light"] {
+    --pst-color-primary: #3399ff;
+    --pst-color-secondary: #9580ff;
+}
+
+a {
+    text-decoration: none;
+}
+
+a:hover {
+    text-decoration: underline;
+}
+
+/*
+Use the bottom border that's used for indicating the current page as the hover style (for a more
+cohesive look).
+NOTE: This selector matches the one in pydata-sphinx-theme
+pydata/pydata-sphinx-theme@v0.14.4/src/pydata_sphinx_theme/assets/styles/sections/_header.scss#L86
+*/
+.bd-header .navbar-nav li a.nav-link:hover {
+    border-bottom: max(3px,.1875rem,.12em) solid var(--pst-color-secondary);
+    text-decoration: none;
+}
+
+/*
+Remove margin from sidebar-primary-items__end so that we don't have an unnecessary scrollbar. We're
+not using the end items currently.
+*/
+.bd-sidebar-primary .sidebar-primary-items__end {
+    margin-top: 0;
+    margin-bottom: 0;
+}
diff --git a/docs/conf/conf.py b/docs/conf/conf.py
new file mode 100644
index 000000000..df3119784
--- /dev/null
+++ b/docs/conf/conf.py
@@ -0,0 +1,78 @@
+# -- Project information -------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "Spider"
+
+# NOTE: We don't include a period after "Inc" since the theme adds one already.
+copyright = "2024-2025 YScope Inc"
+
+# -- General configuration -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "myst_parser",
+    "sphinx_copybutton",
+    "sphinx_design",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.viewcode",
+    "sphinxcontrib.mermaid",
+]
+
+# -- MyST extensions -----------------------------------------------------------
+# https://myst-parser.readthedocs.io/en/stable/syntax/optional.html
+myst_enable_extensions = [
+    "attrs_block",
+    "colon_fence",
+]
+
+myst_heading_anchors = 4
+
+# -- Sphinx autodoc options ----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
+
+autoclass_content = "class"
+autodoc_class_signature = "separated"
+autodoc_typehints = "description"
+
+# -- HTML output options -------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_favicon = "https://docs.yscope.com/_static/favicon.ico"
+html_title = project
+html_show_copyright = True
+
+html_static_path = ["../src/_static"]
+
+html_theme = "pydata_sphinx_theme"
+
+# -- Theme options -------------------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html
+
+html_theme_options = {
+    "footer_start": ["copyright"],
+    "footer_center": [],
+    "footer_end": ["theme-version"],
+    "navbar_start": ["navbar-logo"],
+    "navbar_end": ["navbar-icon-links", "theme-switcher"],
+    "primary_sidebar_end": [],
+    "secondary_sidebar_items": ["page-toc", "edit-this-page"],
+    "show_prev_next": False,
+    "use_edit_page_button": True,
+}
+
+# -- Theme source buttons ------------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/source-buttons.html
+
+html_context = {
+    "github_user": "y-scope",
+    "github_repo": "spider",
+    "github_version": "main",
+    "doc_path": "docs/src",
+}
+
+# -- Theme custom CSS and JS ---------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/static_assets.html
+
+
+def setup(app):
+    app.add_css_file("custom.css")
diff --git a/docs/dev-guide/index.md b/docs/dev-guide/index.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/testing.md b/docs/dev-guide/testing.md
similarity index 97%
rename from docs/testing.md
rename to docs/dev-guide/testing.md
index c133e865a..2678db747 100644
--- a/docs/testing.md
+++ b/docs/dev-guide/testing.md
@@ -57,4 +57,4 @@ You can use the following tasks to run integration tests.
 | `test:integration`            | Runs all integration tests.                                       |
 
 
-[gh-workflow-unit-tests]: ../.github/workflows/unit-tests.yaml
+[gh-workflow-unit-tests]: ../../.github/workflows/unit-tests.yaml
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 000000000..40b5df9fa
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,9 @@
+Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
+latency, high throughput, and robust fault tolerance.
+
+:::{toctree}
+:hidden:
+
+user-guide/index
+dev-guide/index
+:::
\ No newline at end of file
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 000000000..9170c116a
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,6 @@
+myst-parser>=4.0.0
+pydata-sphinx-theme>=0.16.1
+sphinx_design>=0.6.1
+sphinx-copybutton>=0.5.2
+sphinx>=8.1.3
+sphinxcontrib-mermaid>=1.0.0
diff --git a/docs/tasks.yaml b/docs/tasks.yaml
new file mode 100644
index 000000000..251641c6d
--- /dev/null
+++ b/docs/tasks.yaml
@@ -0,0 +1,116 @@
+version: "3"
+
+vars:
+  # Paths
+  G_DOCS_BUILD_DIR: "{{.G_BUILD_DIR}}/docs/html"
+  G_DOCS_VENV_DIR: "{{.G_BUILD_DIR}}/docs-venv"
+  G_NODE_DEPS_DIR: "{{.G_BUILD_DIR}}/docs-node"
+
+  # Target checksum files
+  G_DOCS_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/docs#docs-venv.md5"
+
+tasks:
+  clean:
+    cmds:
+      - "rm -rf '{{.G_DOCS_BUILD_DIR}}'"
+
+  serve:
+    deps:
+      - "http-server"
+      - "site"
+    cmds:
+      - "npm --prefix '{{.G_NODE_DEPS_DIR}}' exec http-server '{{.G_DOCS_BUILD_DIR}}' -c-1"
+
+  site:
+    vars:
+      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5"
+      OUTPUT_DIR: "{{.G_DOCS_BUILD_DIR}}"
+    dir: "{{.TASKFILE_DIR}}"
+    deps:
+      - ":init"
+      - task: ":utils:validate-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+      - "docs-venv"
+    cmds:
+      # Call `clean` before building since `sphinx-build --write-all --fresh-env` isn't always
+      # equivalent to building from scratch.
+      - task: "clean"
+      - "python3 '{{.ROOT_DIR}}/tools/scripts/find-broken-docs-links.py'"
+      - |-
+        . "{{.G_DOCS_VENV_DIR}}/bin/activate"
+        sphinx-build \
+          --write-all \
+          --fresh-env \
+          --conf-dir conf \
+          --nitpicky \
+          --fail-on-warning \
+          --keep-going \
+          --builder html \
+          src "{{.OUTPUT_DIR}}"
+      # This command must be last
+      - task: ":utils:compute-checksum"
+        vars:
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+    sources:
+      - "{{.G_DOCS_VENV_CHECKSUM_FILE}}"
+      - "{{.ROOT_DIR}}/taskfile.yaml"
+      - "{{.TASKFILE}}"
+      - "conf/**/*"
+      - "src/**/*"
+    generates: ["{{.CHECKSUM_FILE}}"]
+
+  docs-venv:
+    internal: true
+    vars:
+      CHECKSUM_FILE: "{{.G_DOCS_VENV_CHECKSUM_FILE}}"
+      OUTPUT_DIR: "{{.G_DOCS_VENV_DIR}}"
+      REQUIREMENTS_FILE: "docs/requirements.txt"
+    deps:
+      - ":init"
+      - task: ":utils:validate-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+    cmds:
+      - task: ":utils:create-venv"
+        vars:
+          LABEL: "docs"
+          OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+          REQUIREMENTS_FILE: "{{.REQUIREMENTS_FILE}}"
+      # This command must be last
+      - task: ":utils:compute-checksum"
+        vars:
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+    sources:
+      - "{{.REQUIREMENTS_FILE}}"
+      - "{{.ROOT_DIR}}/taskfile.yaml"
+      - "{{.TASKFILE}}"
+    generates: ["{{.CHECKSUM_FILE}}"]
+
+  http-server:
+    internal: true
+    vars:
+      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5"
+      OUTPUT_DIR: "{{.G_NODE_DEPS_DIR}}"
+    deps:
+      - ":init"
+      - task: ":utils:validate-checksum"
+        vars:
+          CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+    cmds:
+      - "rm -rf '{{.OUTPUT_DIR}}'"
+      - "npm --prefix '{{.OUTPUT_DIR}}' install http-server"
+      # This command must be last
+      - task: ":utils:compute-checksum"
+        vars:
+          DATA_DIR: "{{.OUTPUT_DIR}}"
+          OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+    sources:
+      - "{{.ROOT_DIR}}/taskfile.yaml"
+      - "{{.TASKFILE}}"
+    generates: ["{{.CHECKSUM_FILE}}"]
diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md
new file mode 100644
index 000000000..e69de29bb
diff --git a/docs/quick-start.md b/docs/user-guide/quick-start.md
similarity index 100%
rename from docs/quick-start.md
rename to docs/user-guide/quick-start.md
diff --git a/taskfile.yaml b/taskfile.yaml
index b398bfcf5..8b8778fde 100644
--- a/taskfile.yaml
+++ b/taskfile.yaml
@@ -1,8 +1,9 @@
 version: "3"
 
 includes:
-  deps: "dep-tasks.yaml"
   build: "build-tasks.yaml"
+  deps: "dep-tasks.yaml"
+  docs: "docs/tasks.yaml"
   lint: "lint-tasks.yaml"
   test: "test-tasks.yaml"
   utils: "tools/yscope-dev-utils/taskfiles/utils.yml"
diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py
new file mode 100644
index 000000000..4a89ad1ee
--- /dev/null
+++ b/tools/scripts/find-broken-docs-links.py
@@ -0,0 +1,105 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def main(argv):
+    repo_root = _get_repo_root()
+
+    found_violation = False
+
+    # Check for docs.yscope.com links with ".md" suffixes
+    if _check_tracked_files(
+            r"docs\.yscope\.com/.+\.md",
+            repo_root,
+            repo_root,
+            'docs.yscope.com links cannot have ".md" suffixes.',
+    ):
+        found_violation = True
+
+    # Check for sphinx :link: attributes that have ".md" suffixes
+    if _check_tracked_files(
+            r":link:[[:space:]]*.+\.md",
+            repo_root,
+            repo_root / "docs",
+            'sphinx :link: attributes cannot have ".md" suffixes',
+    ):
+        found_violation = True
+
+    if found_violation:
+        return 1
+
+    return 0
+
+
+def _get_repo_root() -> Path:
+    path_str = subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True
+    )
+    return Path(path_str.strip())
+
+
+def _check_tracked_files(
+        pattern: str, repo_root: Path, dir_to_search: Path, error_msg: str
+) -> bool:
+    """
+    Check for a pattern in all tracked files in the repo (except this script).
+    :param pattern: The pattern to search for.
+    :param repo_root: The root of the repository.
+    :param dir_to_search: The directory to search in.
+    :param error_msg: Error message if the pattern is found.
+    :return: Whether the pattern was found in any file.
+    """
+    found_matches = False
+
+    # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0')
+    for path_str in subprocess.check_output(
+            [
+                "git",
+                "ls-files",
+                "--cached",
+                "--exclude-standard",
+                "-z",
+                str(dir_to_search.relative_to(repo_root)),
+            ],
+            cwd=repo_root,
+            text=True,
+    ).split("\0"):
+        path = Path(path_str)
+
+        # Skip directories and this script
+        if path == __file__ or (repo_root / path).is_dir():
+            continue
+
+        try:
+            for match in subprocess.check_output(
+                    ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path],
+                    cwd=repo_root,
+                    text=True,
+            ).splitlines():
+                _parse_and_print_match(match, error_msg)
+                found_matches = True
+        except subprocess.CalledProcessError:
+            pass
+
+    return found_matches
+
+
+def _parse_and_print_match(match: str, error_msg: str):
+    """
+    Parses and prints grep matches in a format relevant to the current environment.
+    :param match: The match to parse and print.
+    :param error_msg: Error message if the pattern is found.
+    """
+    if os.getenv("GITHUB_ACTIONS") == "true":
+        # Print a GitHub Actions error annotation
+        file, line, _ = match.split(":", 2)
+        print(f"::error file={file},line={line}::{error_msg}")
+    else:
+        print(error_msg, file=sys.stderr)
+        print(match, file=sys.stderr)
+
+
+if "__main__" == __name__:
+    sys.exit(main(sys.argv))

From a7b9ccc58684bce07f48f2af34aee138de7d5cd6 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Wed, 15 Jan 2025 14:13:52 -0500
Subject: [PATCH 02/13] Move docs into proper locations; Add titles.

---
 docs/dev-guide/index.md                  |  0
 docs/index.md                            |  9 -----
 docs/{ => src}/_static/custom.css        |  0
 docs/src/dev-guide/index.md              |  7 ++++
 docs/{ => src}/dev-guide/testing.md      |  2 +-
 docs/src/index.md                        | 31 +++++++++++++++
 docs/src/user-guide/index.md             |  7 ++++
 docs/{ => src}/user-guide/quick-start.md | 48 ++++++++++++++----------
 docs/user-guide/index.md                 |  0
 9 files changed, 75 insertions(+), 29 deletions(-)
 delete mode 100644 docs/dev-guide/index.md
 delete mode 100644 docs/index.md
 rename docs/{ => src}/_static/custom.css (100%)
 create mode 100644 docs/src/dev-guide/index.md
 rename docs/{ => src}/dev-guide/testing.md (97%)
 create mode 100644 docs/src/index.md
 create mode 100644 docs/src/user-guide/index.md
 rename docs/{ => src}/user-guide/quick-start.md (85%)
 delete mode 100644 docs/user-guide/index.md

diff --git a/docs/dev-guide/index.md b/docs/dev-guide/index.md
deleted file mode 100644
index e69de29bb..000000000
diff --git a/docs/index.md b/docs/index.md
deleted file mode 100644
index 40b5df9fa..000000000
--- a/docs/index.md
+++ /dev/null
@@ -1,9 +0,0 @@
-Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
-latency, high throughput, and robust fault tolerance.
-
-:::{toctree}
-:hidden:
-
-user-guide/index
-dev-guide/index
-:::
\ No newline at end of file
diff --git a/docs/_static/custom.css b/docs/src/_static/custom.css
similarity index 100%
rename from docs/_static/custom.css
rename to docs/src/_static/custom.css
diff --git a/docs/src/dev-guide/index.md b/docs/src/dev-guide/index.md
new file mode 100644
index 000000000..5748fd398
--- /dev/null
+++ b/docs/src/dev-guide/index.md
@@ -0,0 +1,7 @@
+# Developer guide
+
+:::{toctree}
+:hidden:
+
+testing
+:::
diff --git a/docs/dev-guide/testing.md b/docs/src/dev-guide/testing.md
similarity index 97%
rename from docs/dev-guide/testing.md
rename to docs/src/dev-guide/testing.md
index 2678db747..3f8ce5a53 100644
--- a/docs/dev-guide/testing.md
+++ b/docs/src/dev-guide/testing.md
@@ -57,4 +57,4 @@ You can use the following tasks to run integration tests.
 | `test:integration`            | Runs all integration tests.                                       |
 
 
-[gh-workflow-unit-tests]: ../../.github/workflows/unit-tests.yaml
+[gh-workflow-unit-tests]: ../../../.github/workflows/unit-tests.yaml
diff --git a/docs/src/index.md b/docs/src/index.md
new file mode 100644
index 000000000..09da0f6f7
--- /dev/null
+++ b/docs/src/index.md
@@ -0,0 +1,31 @@
+# Spider
+
+Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
+latency, high throughput, and robust fault tolerance.
+
+Spider's docs are separated into two categories:
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: user-guide/index
+🧑 User guide
+^^^
+Docs for those interested in using and operating Spider.
+:::
+
+:::{grid-item-card}
+:link: dev-guide/index
+🛠 Developer guide
+^^^
+Docs for those interested in developing Spider.
+:::
+::::
+
+:::{toctree}
+:hidden:
+
+user-guide/index
+dev-guide/index
+:::
diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md
new file mode 100644
index 000000000..912dff962
--- /dev/null
+++ b/docs/src/user-guide/index.md
@@ -0,0 +1,7 @@
+# User guide
+
+:::{toctree}
+:hidden:
+
+quick-start
+:::
diff --git a/docs/user-guide/quick-start.md b/docs/src/user-guide/quick-start.md
similarity index 85%
rename from docs/user-guide/quick-start.md
rename to docs/src/user-guide/quick-start.md
index 96cc07cce..b4d50d52a 100644
--- a/docs/user-guide/quick-start.md
+++ b/docs/src/user-guide/quick-start.md
@@ -15,9 +15,12 @@ you'll need to:
 
 The example source code for this guide is in `examples/quick-start`.
 
-> [!NOTE] In the rest of this guide:
-> 1. we specify source file paths relative to `examples/quick-start`.
-> 2. all CMake commands should be run from inside `examples/quick-start`.
+:::{note}
+In the rest of this guide:
+
+1. we specify source file paths relative to `examples/quick-start`.
+2. all CMake commands should be run from inside `examples/quick-start`.
+:::
 
 # Requirements
 
@@ -39,15 +42,18 @@ In Spider, a task is a C++ function that satisfies the following conditions:
   * All other parameters must have types that conform to the `Serializable` or `Data` interfaces.
 * It returns a value that conforms to the `Serializable` or `Data` interfaces.
 
-> [!NOTE]
-> You don't immediately need to understand the TaskContext, Serializable, or Data types as we'll
-> explain them in other guides.
+:::{note}
+You don't immediately need to understand the TaskContext, Serializable, or Data types as we'll
+explain them in other guides.
+:::
+
 
 For example, the task in `src/tasks.cpp` computes and returns the sum of two integers.
 
-> [!NOTE]
-> The task is split into a header file and an implementation file so that it can be loaded as a
-> library in the worker, as we'll see in later sections.
+:::{note}
+The task is split into a header file and an implementation file so that it can be loaded as a
+library in the worker, as we'll see in later sections.
+:::
 
 The integer parameters and return value are `Serializable` values.
 
@@ -81,8 +87,9 @@ verifies its result.
 When we submit a task to Spider, Spider returns a `Job`, which represents a scheduled, running, or
 completed task (or `TaskGraph`) in a Spider cluster.
 
-> [!NOTE]
-> `Job`s and `TaskGraph`s will be explained in another guide.
+:::{note}
+`Job`s and `TaskGraph`s will be explained in another guide.
+:::
 
 # Building the client
 
@@ -122,12 +129,14 @@ docker run \
         --publish 3306:3306 mariadb:latest
 ```
 
-> [!WARNING]
-> When the container above is stopped, the database will be deleted. In production, you should set
-> up a database instance with some form of data persistence.
+:::{warning}
+When the container above is stopped, the database will be deleted. In production, you should set up
+a database instance with some form of data persistence.
+:::
 
-> [!WARNING]
-> The container above is using hardcoded default credentials that shouldn't be used in production.
+:::{warning}
+The container above is using hardcoded default credentials that shouldn't be used in production.
+:::
 
 Alternatively, if you have an existing MySQL/MariaDB instance, you can use that as well. Simply
 create a database and authorize a user to access it.
@@ -183,9 +192,10 @@ NOTE:
 * You can specify multiple task libraries to load. The task libraries must be built with linkage
   to the Spider client library. 
 
-> [!TIP]
-> You can start multiple workers to increase the number of concurrent tasks that can be run on the
-> cluster.
+:::{tip}
+You can start multiple workers to increase the number of concurrent tasks that can be run on the
+cluster.
+:::
 
 # Running the client
 
diff --git a/docs/user-guide/index.md b/docs/user-guide/index.md
deleted file mode 100644
index e69de29bb..000000000

From 972781f2af9b801ae158717142a493e0df8971c6 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 17:51:01 -0500
Subject: [PATCH 03/13] Rename user/dev guide -> user/dev docs.

---
 docs/src/{dev-guide => dev-docs}/index.md         |  2 +-
 docs/src/{dev-guide => dev-docs}/testing.md       |  0
 docs/src/index.md                                 | 12 ++++++------
 docs/src/{user-guide => user-docs}/index.md       |  2 +-
 docs/src/{user-guide => user-docs}/quick-start.md |  0
 5 files changed, 8 insertions(+), 8 deletions(-)
 rename docs/src/{dev-guide => dev-docs}/index.md (66%)
 rename docs/src/{dev-guide => dev-docs}/testing.md (100%)
 rename docs/src/{user-guide => user-docs}/index.md (75%)
 rename docs/src/{user-guide => user-docs}/quick-start.md (100%)

diff --git a/docs/src/dev-guide/index.md b/docs/src/dev-docs/index.md
similarity index 66%
rename from docs/src/dev-guide/index.md
rename to docs/src/dev-docs/index.md
index 5748fd398..a7601d8a9 100644
--- a/docs/src/dev-guide/index.md
+++ b/docs/src/dev-docs/index.md
@@ -1,4 +1,4 @@
-# Developer guide
+# Developer docs
 
 :::{toctree}
 :hidden:
diff --git a/docs/src/dev-guide/testing.md b/docs/src/dev-docs/testing.md
similarity index 100%
rename from docs/src/dev-guide/testing.md
rename to docs/src/dev-docs/testing.md
diff --git a/docs/src/index.md b/docs/src/index.md
index 09da0f6f7..c0bed8213 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -9,15 +9,15 @@ Spider's docs are separated into two categories:
 :gutter: 2
 
 :::{grid-item-card}
-:link: user-guide/index
-🧑 User guide
+:link: user-docs/index
+🧑 User docs 
 ^^^
 Docs for those interested in using and operating Spider.
 :::
 
 :::{grid-item-card}
-:link: dev-guide/index
-🛠 Developer guide
+:link: dev-docs/index
+🛠 Developer docs
 ^^^
 Docs for those interested in developing Spider.
 :::
@@ -26,6 +26,6 @@ Docs for those interested in developing Spider.
 :::{toctree}
 :hidden:
 
-user-guide/index
-dev-guide/index
+user-docs/index.md
+dev-docs/index.md
 :::
diff --git a/docs/src/user-guide/index.md b/docs/src/user-docs/index.md
similarity index 75%
rename from docs/src/user-guide/index.md
rename to docs/src/user-docs/index.md
index 912dff962..f8a22969f 100644
--- a/docs/src/user-guide/index.md
+++ b/docs/src/user-docs/index.md
@@ -1,4 +1,4 @@
-# User guide
+# User docs
 
 :::{toctree}
 :hidden:
diff --git a/docs/src/user-guide/quick-start.md b/docs/src/user-docs/quick-start.md
similarity index 100%
rename from docs/src/user-guide/quick-start.md
rename to docs/src/user-docs/quick-start.md

From 9506c9ade5acd557740dc21028da87df665473a6 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:01:19 -0500
Subject: [PATCH 04/13] Fill out user and dev sections.

---
 docs/src/dev-docs/index.md                    | 14 ++++++++++++++
 docs/src/user-docs/guides-overview.md         | 14 ++++++++++++++
 .../{quick-start.md => guides-quick-start.md} |  3 ---
 docs/src/user-docs/index.md                   | 19 ++++++++++++++++++-
 4 files changed, 46 insertions(+), 4 deletions(-)
 create mode 100644 docs/src/user-docs/guides-overview.md
 rename docs/src/user-docs/{quick-start.md => guides-quick-start.md} (97%)

diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md
index a7601d8a9..637fdad1d 100644
--- a/docs/src/dev-docs/index.md
+++ b/docs/src/dev-docs/index.md
@@ -1,5 +1,19 @@
 # Developer docs
 
+This section contains docs for developing Spider. Choose one of the sections below or use the left
+sidebar (if it's hidden, click the <i class="fa fa-bars"></i> icon) to navigate to specific docs.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: testing
+Testing
+^^^
+How to test Spider.
+:::
+::::
+
 :::{toctree}
 :hidden:
 
diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md
new file mode 100644
index 000000000..1e57e4bd4
--- /dev/null
+++ b/docs/src/user-docs/guides-overview.md
@@ -0,0 +1,14 @@
+# Overview
+
+The tutorials below guide you on how to use and operate Spider.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: guides-quick-start
+Quick start
+^^^
+How to get started with running a task on Spider.
+:::
+::::
diff --git a/docs/src/user-docs/quick-start.md b/docs/src/user-docs/guides-quick-start.md
similarity index 97%
rename from docs/src/user-docs/quick-start.md
rename to docs/src/user-docs/guides-quick-start.md
index b4d50d52a..1fd4c696a 100644
--- a/docs/src/user-docs/quick-start.md
+++ b/docs/src/user-docs/guides-quick-start.md
@@ -1,8 +1,5 @@
 # Quick start
 
-Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
-latency, high throughput, and robust fault tolerance.
-
 The guide below briefly describes how to get started with running a task on Spider. At a high-level,
 you'll need to:
 
diff --git a/docs/src/user-docs/index.md b/docs/src/user-docs/index.md
index f8a22969f..d56056307 100644
--- a/docs/src/user-docs/index.md
+++ b/docs/src/user-docs/index.md
@@ -1,7 +1,24 @@
 # User docs
 
+This section contains docs for using and operating Spider. Choose one of the sections below or use
+the left sidebar (if it's hidden, click the <i class="fa fa-bars"></i> icon) to navigate to specific
+docs.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: guides-overview
+Guides
+^^^
+Guides for using and operating Spider.
+:::
+::::
+
 :::{toctree}
 :hidden:
+:caption: Guides
 
-quick-start
+guides-overview.md
+guides-quick-start.md
 :::

From 2ce4659dccac7c21a6b8216a132763b52f727d37 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:02:15 -0500
Subject: [PATCH 05/13] Remove unnecessary link.

---
 docs/README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/README.md b/docs/README.md
index 4458f6d8e..67d439c94 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -37,7 +37,6 @@ task docs:serve
 The command above will install [http-server] and serve the built docs site; `http-server` will print
 the address it binds to (usually http://localhost:8080).
 
-[git-lfs]: https://git-lfs.com
 [http-server]: https://www.npmjs.com/package/http-server
 [Node.js]: https://nodejs.org/en/download/current
 [Task]: https://taskfile.dev/

From 290e919d3d3dd76e744a80b798284b1e78b00d3f Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:05:42 -0500
Subject: [PATCH 06/13] Link to the example source code on GitHub.

---
 docs/src/user-docs/guides-quick-start.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/src/user-docs/guides-quick-start.md b/docs/src/user-docs/guides-quick-start.md
index 1fd4c696a..c290e7490 100644
--- a/docs/src/user-docs/guides-quick-start.md
+++ b/docs/src/user-docs/guides-quick-start.md
@@ -10,7 +10,7 @@ you'll need to:
 * Set up a Spider cluster
 * Run the client
 
-The example source code for this guide is in `examples/quick-start`.
+The example source code for this guide is in [examples/quick-start].
 
 :::{note}
 In the rest of this guide:
@@ -214,3 +214,4 @@ support for fault tolerance.
 
 [Docker]: https://docs.docker.com/engine/install/
 [docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
+[examples/quick-start]: https://github.com/y-scope/spider/tree/main/examples/quick-start

From 130d711b5f3adae44785da2ecba43fb54cf8fa94 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:06:41 -0500
Subject: [PATCH 07/13] Add new taskfile to YAML linting.

---
 lint-tasks.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lint-tasks.yaml b/lint-tasks.yaml
index aa8804992..6897b7f1e 100644
--- a/lint-tasks.yaml
+++ b/lint-tasks.yaml
@@ -171,6 +171,7 @@ tasks:
           .github/ \
           build-tasks.yaml \
           dep-tasks.yaml \
+          docs/tasks.yaml \
           lint-tasks.yaml \
           taskfile.yaml \
           test-tasks.yaml

From 92766e897d60115cb9b4adfb72fdf3c54f1c6f17 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:14:45 -0500
Subject: [PATCH 08/13] Remove unnecessary mermaid package.

---
 docs/conf/conf.py     | 1 -
 docs/requirements.txt | 1 -
 2 files changed, 2 deletions(-)

diff --git a/docs/conf/conf.py b/docs/conf/conf.py
index df3119784..1f9c46f58 100644
--- a/docs/conf/conf.py
+++ b/docs/conf/conf.py
@@ -15,7 +15,6 @@
     "sphinx_design",
     "sphinx.ext.autodoc",
     "sphinx.ext.viewcode",
-    "sphinxcontrib.mermaid",
 ]
 
 # -- MyST extensions -----------------------------------------------------------
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 9170c116a..843d45bba 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -3,4 +3,3 @@ pydata-sphinx-theme>=0.16.1
 sphinx_design>=0.6.1
 sphinx-copybutton>=0.5.2
 sphinx>=8.1.3
-sphinxcontrib-mermaid>=1.0.0

From 781f38f221c4003224574b68dde46bf7e9879162 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:15:03 -0500
Subject: [PATCH 09/13] Update file link to GH link to avoid unintentional
 downloads.

---
 docs/src/dev-docs/testing.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md
index 3f8ce5a53..873ef7b67 100644
--- a/docs/src/dev-docs/testing.md
+++ b/docs/src/dev-docs/testing.md
@@ -57,4 +57,4 @@ You can use the following tasks to run integration tests.
 | `test:integration`            | Runs all integration tests.                                       |
 
 
-[gh-workflow-unit-tests]: ../../../.github/workflows/unit-tests.yaml
+[gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml

From 7313dbedf31c5ca37d47e50f5f4d1d7781682053 Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:25:52 -0500
Subject: [PATCH 10/13] Minor improvement.

---
 tools/scripts/find-broken-docs-links.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py
index 4a89ad1ee..fb9c4a6a6 100644
--- a/tools/scripts/find-broken-docs-links.py
+++ b/tools/scripts/find-broken-docs-links.py
@@ -2,9 +2,13 @@
 import subprocess
 import sys
 from pathlib import Path
+from typing import List
 
 
-def main(argv):
+def main(argv: List[str] = None):
+    if argv is None:
+        argv = sys.argv
+
     repo_root = _get_repo_root()
 
     found_violation = False

From 3529bb8f700f3371d500e58f93809cbde9dfb74a Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 20:12:25 -0500
Subject: [PATCH 11/13] Fix bullet indent.

---
 docs/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/README.md b/docs/README.md
index 67d439c94..6954274f4 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -20,7 +20,7 @@ this project:
   task docs:site
   ```
 
-    * The output of the build will be in `../build/docs/html`.
+  * The output of the build will be in `../build/docs/html`.
 
 * Clean up the build:
 

From 1d090dccd5a096b98eed72a9a1fdf1e5aeb2c3aa Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 20:14:53 -0500
Subject: [PATCH 12/13] Log grep process failure.

---
 tools/scripts/find-broken-docs-links.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py
index fb9c4a6a6..1a1d16806 100644
--- a/tools/scripts/find-broken-docs-links.py
+++ b/tools/scripts/find-broken-docs-links.py
@@ -84,8 +84,9 @@ def _check_tracked_files(
             ).splitlines():
                 _parse_and_print_match(match, error_msg)
                 found_matches = True
-        except subprocess.CalledProcessError:
-            pass
+        except subprocess.CalledProcessError as ex:
+            if ex.returncode != 1:
+                print(f"Failed to grep '{path}' - exit status {ex.returncode}.", file=sys.stderr)
 
     return found_matches
 

From 0b21e830ece3f19be509fe6f4d6f9a4355701e5b Mon Sep 17 00:00:00 2001
From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com>
Date: Tue, 28 Jan 2025 22:48:39 -0500
Subject: [PATCH 13/13] Embed snippets of example source into guide.

---
 docs/src/user-docs/guides-quick-start.md | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/docs/src/user-docs/guides-quick-start.md b/docs/src/user-docs/guides-quick-start.md
index c290e7490..68486611d 100644
--- a/docs/src/user-docs/guides-quick-start.md
+++ b/docs/src/user-docs/guides-quick-start.md
@@ -44,8 +44,15 @@ You don't immediately need to understand the TaskContext, Serializable, or Data
 explain them in other guides.
 :::
 
-
-For example, the task in `src/tasks.cpp` computes and returns the sum of two integers.
+For example, the task in `src/tasks.cpp` computes and returns the sum of two integers:
+
+:::{literalinclude} ../../../examples/quick-start/src/tasks.cpp
+:caption: src/tasks.cpp: The example task.
+:language: cpp
+:lines: 5-12
+:lineno-start: 5
+:linenos: true
+:::
 
 :::{note}
 The task is split into a header file and an implementation file so that it can be loaded as a
@@ -79,7 +86,15 @@ To make Spider to run a task, we first need to write a client application. Gener
 4. and then handles the result.
 
 For example, the client in `src/client.cpp` runs the `sum` task from the previous section and
-verifies its result.
+verifies its result:
+
+:::{literalinclude} ../../../examples/quick-start/src/client.cpp
+:caption: src/client.cpp: A snippet of the example client.
+:language: cpp
+:lines: 24-35
+:lineno-start: 24
+:linenos: true
+:::
 
 When we submit a task to Spider, Spider returns a `Job`, which represents a scheduled, running, or
 completed task (or `TaskGraph`) in a Spider cluster.