diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 000000000..6954274f4
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,42 @@
+# Docs
+
+This directory contains the files necessary to generate a Sphinx-based documentation website for
+this project:
+
+* `conf` - Configuration files
+* `src` - The actual docs
+
+## Requirements
+
+* [Node.js] >= 16 to be able to [view the output](#viewing-the-output)
+* Python 3.10 or higher
+* [Task] 3.40.0 or higher
+
+## Build commands
+
+* Build the site incrementally:
+
+ ```shell
+ task docs:site
+ ```
+
+ * The output of the build will be in `../build/docs/html`.
+
+* Clean up the build:
+
+ ```shell
+ task docs:clean
+ ```
+
+## Viewing the output
+
+```shell
+task docs:serve
+```
+
+The command above will install [http-server] and serve the built docs site; `http-server` will print
+the address it binds to (usually http://localhost:8080).
+
+[http-server]: https://www.npmjs.com/package/http-server
+[Node.js]: https://nodejs.org/en/download/current
+[Task]: https://taskfile.dev/
diff --git a/docs/conf/conf.py b/docs/conf/conf.py
new file mode 100644
index 000000000..1f9c46f58
--- /dev/null
+++ b/docs/conf/conf.py
@@ -0,0 +1,77 @@
+# -- Project information -------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "Spider"
+
+# NOTE: We don't include a period after "Inc" since the theme adds one already.
+copyright = "2024-2025 YScope Inc"
+
+# -- General configuration -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+ "myst_parser",
+ "sphinx_copybutton",
+ "sphinx_design",
+ "sphinx.ext.autodoc",
+ "sphinx.ext.viewcode",
+]
+
+# -- MyST extensions -----------------------------------------------------------
+# https://myst-parser.readthedocs.io/en/stable/syntax/optional.html
+myst_enable_extensions = [
+ "attrs_block",
+ "colon_fence",
+]
+
+myst_heading_anchors = 4
+
+# -- Sphinx autodoc options ----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration
+
+autoclass_content = "class"
+autodoc_class_signature = "separated"
+autodoc_typehints = "description"
+
+# -- HTML output options -------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_favicon = "https://docs.yscope.com/_static/favicon.ico"
+html_title = project
+html_show_copyright = True
+
+html_static_path = ["../src/_static"]
+
+html_theme = "pydata_sphinx_theme"
+
+# -- Theme options -------------------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html
+
+html_theme_options = {
+ "footer_start": ["copyright"],
+ "footer_center": [],
+ "footer_end": ["theme-version"],
+ "navbar_start": ["navbar-logo"],
+ "navbar_end": ["navbar-icon-links", "theme-switcher"],
+ "primary_sidebar_end": [],
+ "secondary_sidebar_items": ["page-toc", "edit-this-page"],
+ "show_prev_next": False,
+ "use_edit_page_button": True,
+}
+
+# -- Theme source buttons ------------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/source-buttons.html
+
+html_context = {
+ "github_user": "y-scope",
+ "github_repo": "spider",
+ "github_version": "main",
+ "doc_path": "docs/src",
+}
+
+# -- Theme custom CSS and JS ---------------------------------------------------
+# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/static_assets.html
+
+
+def setup(app):
+ app.add_css_file("custom.css")
diff --git a/docs/requirements.txt b/docs/requirements.txt
new file mode 100644
index 000000000..843d45bba
--- /dev/null
+++ b/docs/requirements.txt
@@ -0,0 +1,5 @@
+myst-parser>=4.0.0
+pydata-sphinx-theme>=0.16.1
+sphinx_design>=0.6.1
+sphinx-copybutton>=0.5.2
+sphinx>=8.1.3
diff --git a/docs/src/_static/custom.css b/docs/src/_static/custom.css
new file mode 100644
index 000000000..f25a6694b
--- /dev/null
+++ b/docs/src/_static/custom.css
@@ -0,0 +1,32 @@
+html[data-theme="dark"], html[data-theme="light"] {
+ --pst-color-primary: #3399ff;
+ --pst-color-secondary: #9580ff;
+}
+
+a {
+ text-decoration: none;
+}
+
+a:hover {
+ text-decoration: underline;
+}
+
+/*
+Use the bottom border that's used for indicating the current page as the hover style (for a more
+cohesive look).
+NOTE: This selector matches the one in pydata-sphinx-theme
+pydata/pydata-sphinx-theme@v0.14.4/src/pydata_sphinx_theme/assets/styles/sections/_header.scss#L86
+*/
+.bd-header .navbar-nav li a.nav-link:hover {
+ border-bottom: max(3px,.1875rem,.12em) solid var(--pst-color-secondary);
+ text-decoration: none;
+}
+
+/*
+Remove margin from sidebar-primary-items__end so that we don't have an unnecessary scrollbar. We're
+not using the end items currently.
+*/
+.bd-sidebar-primary .sidebar-primary-items__end {
+ margin-top: 0;
+ margin-bottom: 0;
+}
diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md
new file mode 100644
index 000000000..637fdad1d
--- /dev/null
+++ b/docs/src/dev-docs/index.md
@@ -0,0 +1,21 @@
+# Developer docs
+
+This section contains docs for developing Spider. Choose one of the sections below or use the left
+sidebar (if it's hidden, click the icon) to navigate to specific docs.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: testing
+Testing
+^^^
+How to test Spider.
+:::
+::::
+
+:::{toctree}
+:hidden:
+
+testing
+:::
diff --git a/docs/testing.md b/docs/src/dev-docs/testing.md
similarity index 95%
rename from docs/testing.md
rename to docs/src/dev-docs/testing.md
index c133e865a..873ef7b67 100644
--- a/docs/testing.md
+++ b/docs/src/dev-docs/testing.md
@@ -57,4 +57,4 @@ You can use the following tasks to run integration tests.
| `test:integration` | Runs all integration tests. |
-[gh-workflow-unit-tests]: ../.github/workflows/unit-tests.yaml
+[gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml
diff --git a/docs/src/index.md b/docs/src/index.md
new file mode 100644
index 000000000..c0bed8213
--- /dev/null
+++ b/docs/src/index.md
@@ -0,0 +1,31 @@
+# Spider
+
+Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
+latency, high throughput, and robust fault tolerance.
+
+Spider's docs are separated into two categories:
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: user-docs/index
+🧑 User docs
+^^^
+Docs for those interested in using and operating Spider.
+:::
+
+:::{grid-item-card}
+:link: dev-docs/index
+🛠Developer docs
+^^^
+Docs for those interested in developing Spider.
+:::
+::::
+
+:::{toctree}
+:hidden:
+
+user-docs/index.md
+dev-docs/index.md
+:::
diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md
new file mode 100644
index 000000000..1e57e4bd4
--- /dev/null
+++ b/docs/src/user-docs/guides-overview.md
@@ -0,0 +1,14 @@
+# Overview
+
+The tutorials below guide you on how to use and operate Spider.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: guides-quick-start
+Quick start
+^^^
+How to get started with running a task on Spider.
+:::
+::::
diff --git a/docs/quick-start.md b/docs/src/user-docs/guides-quick-start.md
similarity index 77%
rename from docs/quick-start.md
rename to docs/src/user-docs/guides-quick-start.md
index 96cc07cce..68486611d 100644
--- a/docs/quick-start.md
+++ b/docs/src/user-docs/guides-quick-start.md
@@ -1,8 +1,5 @@
# Quick start
-Spider is a distributed system for executing user-defined tasks. It is designed to achieve low
-latency, high throughput, and robust fault tolerance.
-
The guide below briefly describes how to get started with running a task on Spider. At a high-level,
you'll need to:
@@ -13,11 +10,14 @@ you'll need to:
* Set up a Spider cluster
* Run the client
-The example source code for this guide is in `examples/quick-start`.
+The example source code for this guide is in [examples/quick-start].
+
+:::{note}
+In the rest of this guide:
-> [!NOTE] In the rest of this guide:
-> 1. we specify source file paths relative to `examples/quick-start`.
-> 2. all CMake commands should be run from inside `examples/quick-start`.
+1. we specify source file paths relative to `examples/quick-start`.
+2. all CMake commands should be run from inside `examples/quick-start`.
+:::
# Requirements
@@ -39,15 +39,25 @@ In Spider, a task is a C++ function that satisfies the following conditions:
* All other parameters must have types that conform to the `Serializable` or `Data` interfaces.
* It returns a value that conforms to the `Serializable` or `Data` interfaces.
-> [!NOTE]
-> You don't immediately need to understand the TaskContext, Serializable, or Data types as we'll
-> explain them in other guides.
+:::{note}
+You don't immediately need to understand the TaskContext, Serializable, or Data types as we'll
+explain them in other guides.
+:::
-For example, the task in `src/tasks.cpp` computes and returns the sum of two integers.
+For example, the task in `src/tasks.cpp` computes and returns the sum of two integers:
-> [!NOTE]
-> The task is split into a header file and an implementation file so that it can be loaded as a
-> library in the worker, as we'll see in later sections.
+:::{literalinclude} ../../../examples/quick-start/src/tasks.cpp
+:caption: src/tasks.cpp: The example task.
+:language: cpp
+:lines: 5-12
+:lineno-start: 5
+:linenos: true
+:::
+
+:::{note}
+The task is split into a header file and an implementation file so that it can be loaded as a
+library in the worker, as we'll see in later sections.
+:::
The integer parameters and return value are `Serializable` values.
@@ -76,13 +86,22 @@ To make Spider to run a task, we first need to write a client application. Gener
4. and then handles the result.
For example, the client in `src/client.cpp` runs the `sum` task from the previous section and
-verifies its result.
+verifies its result:
+
+:::{literalinclude} ../../../examples/quick-start/src/client.cpp
+:caption: src/client.cpp: A snippet of the example client.
+:language: cpp
+:lines: 24-35
+:lineno-start: 24
+:linenos: true
+:::
When we submit a task to Spider, Spider returns a `Job`, which represents a scheduled, running, or
completed task (or `TaskGraph`) in a Spider cluster.
-> [!NOTE]
-> `Job`s and `TaskGraph`s will be explained in another guide.
+:::{note}
+`Job`s and `TaskGraph`s will be explained in another guide.
+:::
# Building the client
@@ -122,12 +141,14 @@ docker run \
--publish 3306:3306 mariadb:latest
```
-> [!WARNING]
-> When the container above is stopped, the database will be deleted. In production, you should set
-> up a database instance with some form of data persistence.
+:::{warning}
+When the container above is stopped, the database will be deleted. In production, you should set up
+a database instance with some form of data persistence.
+:::
-> [!WARNING]
-> The container above is using hardcoded default credentials that shouldn't be used in production.
+:::{warning}
+The container above is using hardcoded default credentials that shouldn't be used in production.
+:::
Alternatively, if you have an existing MySQL/MariaDB instance, you can use that as well. Simply
create a database and authorize a user to access it.
@@ -183,9 +204,10 @@ NOTE:
* You can specify multiple task libraries to load. The task libraries must be built with linkage
to the Spider client library.
-> [!TIP]
-> You can start multiple workers to increase the number of concurrent tasks that can be run on the
-> cluster.
+:::{tip}
+You can start multiple workers to increase the number of concurrent tasks that can be run on the
+cluster.
+:::
# Running the client
@@ -207,3 +229,4 @@ support for fault tolerance.
[Docker]: https://docs.docker.com/engine/install/
[docker-non-root]: https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user
+[examples/quick-start]: https://github.com/y-scope/spider/tree/main/examples/quick-start
diff --git a/docs/src/user-docs/index.md b/docs/src/user-docs/index.md
new file mode 100644
index 000000000..d56056307
--- /dev/null
+++ b/docs/src/user-docs/index.md
@@ -0,0 +1,24 @@
+# User docs
+
+This section contains docs for using and operating Spider. Choose one of the sections below or use
+the left sidebar (if it's hidden, click the icon) to navigate to specific
+docs.
+
+::::{grid} 1 1 2 2
+:gutter: 2
+
+:::{grid-item-card}
+:link: guides-overview
+Guides
+^^^
+Guides for using and operating Spider.
+:::
+::::
+
+:::{toctree}
+:hidden:
+:caption: Guides
+
+guides-overview.md
+guides-quick-start.md
+:::
diff --git a/docs/tasks.yaml b/docs/tasks.yaml
new file mode 100644
index 000000000..251641c6d
--- /dev/null
+++ b/docs/tasks.yaml
@@ -0,0 +1,116 @@
+version: "3"
+
+vars:
+ # Paths
+ G_DOCS_BUILD_DIR: "{{.G_BUILD_DIR}}/docs/html"
+ G_DOCS_VENV_DIR: "{{.G_BUILD_DIR}}/docs-venv"
+ G_NODE_DEPS_DIR: "{{.G_BUILD_DIR}}/docs-node"
+
+ # Target checksum files
+ G_DOCS_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/docs#docs-venv.md5"
+
+tasks:
+ clean:
+ cmds:
+ - "rm -rf '{{.G_DOCS_BUILD_DIR}}'"
+
+ serve:
+ deps:
+ - "http-server"
+ - "site"
+ cmds:
+ - "npm --prefix '{{.G_NODE_DEPS_DIR}}' exec http-server '{{.G_DOCS_BUILD_DIR}}' -c-1"
+
+ site:
+ vars:
+ CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5"
+ OUTPUT_DIR: "{{.G_DOCS_BUILD_DIR}}"
+ dir: "{{.TASKFILE_DIR}}"
+ deps:
+ - ":init"
+ - task: ":utils:validate-checksum"
+ vars:
+ CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ - "docs-venv"
+ cmds:
+ # Call `clean` before building since `sphinx-build --write-all --fresh-env` isn't always
+ # equivalent to building from scratch.
+ - task: "clean"
+ - "python3 '{{.ROOT_DIR}}/tools/scripts/find-broken-docs-links.py'"
+ - |-
+ . "{{.G_DOCS_VENV_DIR}}/bin/activate"
+ sphinx-build \
+ --write-all \
+ --fresh-env \
+ --conf-dir conf \
+ --nitpicky \
+ --fail-on-warning \
+ --keep-going \
+ --builder html \
+ src "{{.OUTPUT_DIR}}"
+ # This command must be last
+ - task: ":utils:compute-checksum"
+ vars:
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+ sources:
+ - "{{.G_DOCS_VENV_CHECKSUM_FILE}}"
+ - "{{.ROOT_DIR}}/taskfile.yaml"
+ - "{{.TASKFILE}}"
+ - "conf/**/*"
+ - "src/**/*"
+ generates: ["{{.CHECKSUM_FILE}}"]
+
+ docs-venv:
+ internal: true
+ vars:
+ CHECKSUM_FILE: "{{.G_DOCS_VENV_CHECKSUM_FILE}}"
+ OUTPUT_DIR: "{{.G_DOCS_VENV_DIR}}"
+ REQUIREMENTS_FILE: "docs/requirements.txt"
+ deps:
+ - ":init"
+ - task: ":utils:validate-checksum"
+ vars:
+ CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ cmds:
+ - task: ":utils:create-venv"
+ vars:
+ LABEL: "docs"
+ OUTPUT_DIR: "{{.OUTPUT_DIR}}"
+ REQUIREMENTS_FILE: "{{.REQUIREMENTS_FILE}}"
+ # This command must be last
+ - task: ":utils:compute-checksum"
+ vars:
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+ sources:
+ - "{{.REQUIREMENTS_FILE}}"
+ - "{{.ROOT_DIR}}/taskfile.yaml"
+ - "{{.TASKFILE}}"
+ generates: ["{{.CHECKSUM_FILE}}"]
+
+ http-server:
+ internal: true
+ vars:
+ CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5"
+ OUTPUT_DIR: "{{.G_NODE_DEPS_DIR}}"
+ deps:
+ - ":init"
+ - task: ":utils:validate-checksum"
+ vars:
+ CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ cmds:
+ - "rm -rf '{{.OUTPUT_DIR}}'"
+ - "npm --prefix '{{.OUTPUT_DIR}}' install http-server"
+ # This command must be last
+ - task: ":utils:compute-checksum"
+ vars:
+ DATA_DIR: "{{.OUTPUT_DIR}}"
+ OUTPUT_FILE: "{{.CHECKSUM_FILE}}"
+ sources:
+ - "{{.ROOT_DIR}}/taskfile.yaml"
+ - "{{.TASKFILE}}"
+ generates: ["{{.CHECKSUM_FILE}}"]
diff --git a/lint-tasks.yaml b/lint-tasks.yaml
index aa8804992..6897b7f1e 100644
--- a/lint-tasks.yaml
+++ b/lint-tasks.yaml
@@ -171,6 +171,7 @@ tasks:
.github/ \
build-tasks.yaml \
dep-tasks.yaml \
+ docs/tasks.yaml \
lint-tasks.yaml \
taskfile.yaml \
test-tasks.yaml
diff --git a/taskfile.yaml b/taskfile.yaml
index b398bfcf5..8b8778fde 100644
--- a/taskfile.yaml
+++ b/taskfile.yaml
@@ -1,8 +1,9 @@
version: "3"
includes:
- deps: "dep-tasks.yaml"
build: "build-tasks.yaml"
+ deps: "dep-tasks.yaml"
+ docs: "docs/tasks.yaml"
lint: "lint-tasks.yaml"
test: "test-tasks.yaml"
utils: "tools/yscope-dev-utils/taskfiles/utils.yml"
diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py
new file mode 100644
index 000000000..1a1d16806
--- /dev/null
+++ b/tools/scripts/find-broken-docs-links.py
@@ -0,0 +1,110 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+
+def main(argv: List[str] = None):
+ if argv is None:
+ argv = sys.argv
+
+ repo_root = _get_repo_root()
+
+ found_violation = False
+
+ # Check for docs.yscope.com links with ".md" suffixes
+ if _check_tracked_files(
+ r"docs\.yscope\.com/.+\.md",
+ repo_root,
+ repo_root,
+ 'docs.yscope.com links cannot have ".md" suffixes.',
+ ):
+ found_violation = True
+
+ # Check for sphinx :link: attributes that have ".md" suffixes
+ if _check_tracked_files(
+ r":link:[[:space:]]*.+\.md",
+ repo_root,
+ repo_root / "docs",
+ 'sphinx :link: attributes cannot have ".md" suffixes',
+ ):
+ found_violation = True
+
+ if found_violation:
+ return 1
+
+ return 0
+
+
+def _get_repo_root() -> Path:
+ path_str = subprocess.check_output(
+ ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True
+ )
+ return Path(path_str.strip())
+
+
+def _check_tracked_files(
+ pattern: str, repo_root: Path, dir_to_search: Path, error_msg: str
+) -> bool:
+ """
+ Check for a pattern in all tracked files in the repo (except this script).
+ :param pattern: The pattern to search for.
+ :param repo_root: The root of the repository.
+ :param dir_to_search: The directory to search in.
+ :param error_msg: Error message if the pattern is found.
+ :return: Whether the pattern was found in any file.
+ """
+ found_matches = False
+
+ # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0')
+ for path_str in subprocess.check_output(
+ [
+ "git",
+ "ls-files",
+ "--cached",
+ "--exclude-standard",
+ "-z",
+ str(dir_to_search.relative_to(repo_root)),
+ ],
+ cwd=repo_root,
+ text=True,
+ ).split("\0"):
+ path = Path(path_str)
+
+ # Skip directories and this script
+ if path == __file__ or (repo_root / path).is_dir():
+ continue
+
+ try:
+ for match in subprocess.check_output(
+ ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path],
+ cwd=repo_root,
+ text=True,
+ ).splitlines():
+ _parse_and_print_match(match, error_msg)
+ found_matches = True
+ except subprocess.CalledProcessError as ex:
+ if ex.returncode != 1:
+ print(f"Failed to grep '{path}' - exit status {ex.returncode}.", file=sys.stderr)
+
+ return found_matches
+
+
+def _parse_and_print_match(match: str, error_msg: str):
+ """
+ Parses and prints grep matches in a format relevant to the current environment.
+ :param match: The match to parse and print.
+ :param error_msg: Error message if the pattern is found.
+ """
+ if os.getenv("GITHUB_ACTIONS") == "true":
+ # Print a GitHub Actions error annotation
+ file, line, _ = match.split(":", 2)
+ print(f"::error file={file},line={line}::{error_msg}")
+ else:
+ print(error_msg, file=sys.stderr)
+ print(match, file=sys.stderr)
+
+
+if "__main__" == __name__:
+ sys.exit(main(sys.argv))