doc: Add an auto-generated dependency graph for internal crates

.github/workflows/dependencies.yml

@@ -66,4 +66,23 @@ jobs:
       - name: Install cargo-machete
         run: cargo install cargo-machete --version ^0.9 --locked
       - name: Detect unused dependencies
-        run: cargo machete --with-metadata
+        run: cargo machete --with-metadata
+
+  dependency-graph:
+    name: dependency graph up to date
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+    steps:
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8  # v6.0.1
+        with:
+          submodules: true
+          fetch-depth: 1
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Install cargo-depgraph
+        run: cargo install cargo-depgraph --version ^1.10 --locked
+      - name: Check workspace dependency graph
+        run: docs/scripts/generate_dependency_graph.sh --check

Cargo.toml

@@ -63,6 +63,7 @@ members = [
     "datafusion-examples/examples/ffi/ffi_module_loader",
     "test-utils",
     "benchmarks",
+    "datafusion/ci-dummy",
     "datafusion/macros",
     "datafusion/doc",
 ]

datafusion/ci-dummy/Cargo.toml

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-ci-dummy"
+description = "Internal dummy crate used to exercise CI checks"
+version = { workspace = true }
+edition = "2024"
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = { workspace = true }
+publish = false
+
+[lints]
+workspace = true
+
+[dependencies]

datafusion/ci-dummy/src/lib.rs

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Internal dummy crate to help validate CI wiring.
+
+/// A tiny helper that makes sure the crate compiles and can be used in tests.
+pub const fn ping() -> &'static str {
+    "pong"
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn responds_with_pong() {
+        assert_eq!(ping(), "pong");
+    }
+}

dev/rust_lint.sh

@@ -43,9 +43,16 @@ if ! command -v typos &> /dev/null; then
     cargo install typos-cli --locked
 fi
 
+# For dependency graph checks
+if ! cargo depgraph --help > /dev/null 2>&1; then
+    echo "Installing cargo-depgraph using cargo"
+    cargo install cargo-depgraph --version ^1.10 --locked
+fi
+
 ci/scripts/rust_fmt.sh
 ci/scripts/rust_clippy.sh
 ci/scripts/rust_toml_fmt.sh
 ci/scripts/rust_docs.sh
 ci/scripts/license_header.sh
 ci/scripts/typos_check.sh
+docs/scripts/generate_dependency_graph.sh --check

docs/scripts/generate_dependency_graph.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# See `usage()` for details about this script.
+#
+# The key commands to generate the dependency graph SVG in this script are:
+#   cargo depgraph ... | dot -Tsvg > deps.svg
+# See below for the exact command used.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+OUTPUT_DIR="${REPO_DIR}/docs/source/_static/data"
+DOT_OUTPUT="${OUTPUT_DIR}/deps.dot"
+SVG_OUTPUT="${OUTPUT_DIR}/deps.svg"
+
+usage() {
+  cat <<EOF
+Generate the workspace dependency graph SVG for the docs.
+
+'deps.svg' is embedded in the DataFusion docs (Contributor Guide → Architecture → Workspace Dependency Graph).
+
+'deps.dot' is the intermediate DOT used to render deps.svg; CI checks it to ensure the graph is up to date with the codebase.
+
+Outputs:
+  DOT: ${DOT_OUTPUT}
+  SVG: ${SVG_OUTPUT}
+
+Usage: $(basename "$0") [--check]
+
+Options:
+  --check     Validate the checked-in graph without modifying files.
+  -h, --help  Show this help message.
+EOF
+}
+
+# Default 'generate' mode creates/overrides 'deps.dot' and 'deps.svg', and the 'check'
+# mode verify if the existing 'deps.dot' is up-to-date
+mode="generate"
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --check)
+      mode="check"
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if ! command -v cargo >/dev/null 2>&1; then
+  echo "cargo is required to build the dependency graph." >&2
+  exit 1
+fi
+
+if ! cargo depgraph --help >/dev/null 2>&1; then
+  echo "cargo-depgraph is required (install with: cargo install cargo-depgraph)." >&2
+  exit 1
+fi
+
+if [[ "${mode}" != "check" ]] && ! command -v dot >/dev/null 2>&1; then
+  echo "Graphviz 'dot' is required to render the SVG." >&2
+  exit 1
+fi
+
+mkdir -p "${OUTPUT_DIR}"
+
+generate_dot() {
+  local output_file="$1"
+  (
+    cd "${REPO_DIR}"
+    cargo depgraph \
+      --workspace-only \
+      --all-deps \
+      --dedup-transitive-deps \
+      > "${output_file}"
+  )
+}
+
+if [[ "${mode}" == "check" ]]; then
+  if [[ ! -f "${DOT_OUTPUT}" ]]; then
+    echo "Expected graph ${DOT_OUTPUT} is missing; regenerate it with $(basename "$0")." >&2
+    exit 1
+  fi
+
+  temp_dot="$(mktemp)"
+  cleanup() { rm -f "${temp_dot}"; }
+  trap cleanup EXIT
+
+  generate_dot "${temp_dot}"
+
+  if ! cmp -s "${temp_dot}" "${DOT_OUTPUT}"; then
+    echo "Dependency graph is out of date. Re-run $(basename "$0") to refresh ${DOT_OUTPUT} and ${SVG_OUTPUT}." >&2
+    exit 1
+  fi
+
+  echo "Dependency graph is up to date."
+  exit 0
+fi
+
+generate_dot "${DOT_OUTPUT}"
+
+dot \
+  -Grankdir=TB \
+  -Gconcentrate=true \
+  -Goverlap=false \
+  -Tsvg "${DOT_OUTPUT}" \
+  > "${SVG_OUTPUT}"
+
+echo "Wrote dependency graph DOT to ${DOT_OUTPUT}"
+echo "Wrote dependency graph SVG to ${SVG_OUTPUT}"

docs/source/_static/data/deps.dot

@@ -0,0 +1,116 @@
+digraph {
+    0 [ label = "datafusion-common" shape = box]
+    1 [ label = "datafusion-common-runtime" shape = box]
+    2 [ label = "datafusion-catalog" shape = box]
+    3 [ label = "datafusion-datasource" shape = box]
+    4 [ label = "datafusion-execution" shape = box]
+    5 [ label = "datafusion-expr" shape = box]
+    6 [ label = "datafusion-doc" shape = box]
+    7 [ label = "datafusion-expr-common" shape = box]
+    8 [ label = "datafusion-functions-aggregate-common" shape = box]
+    9 [ label = "datafusion-physical-expr-common" shape = box]
+    10 [ label = "datafusion-functions-window-common" shape = box]
+    11 [ label = "datafusion-physical-expr" shape = box]
+    12 [ label = "datafusion-functions" shape = box]
+    13 [ label = "datafusion-macros" shape = box, color = green3]
+    14 [ label = "datafusion-physical-expr-adapter" shape = box]
+    15 [ label = "datafusion-physical-plan" shape = box]
+    16 [ label = "datafusion-functions-aggregate" shape = box]
+    17 [ label = "datafusion-functions-window" shape = box]
+    18 [ label = "datafusion-session" shape = box]
+    19 [ label = "datafusion-catalog-listing" shape = box]
+    20 [ label = "datafusion-datasource-parquet" shape = box]
+    21 [ label = "datafusion-pruning" shape = box]
+    22 [ label = "datafusion-functions-nested" shape = box]
+    23 [ label = "datafusion-datasource-arrow" shape = box]
+    24 [ label = "datafusion-datasource-avro" shape = box]
+    25 [ label = "datafusion-datasource-csv" shape = box]
+    26 [ label = "datafusion-datasource-json" shape = box]
+    27 [ label = "datafusion" shape = box]
+    28 [ label = "datafusion-functions-table" shape = box]
+    29 [ label = "datafusion-optimizer" shape = box]
+    30 [ label = "datafusion-sql" shape = box]
+    31 [ label = "datafusion-physical-optimizer" shape = box]
+    32 [ label = "test-utils" shape = box]
+    33 [ label = "datafusion-ffi" shape = box]
+    34 [ label = "datafusion-proto" shape = box]
+    35 [ label = "datafusion-proto-common" shape = box]
+    36 [ label = "gen" shape = box]
+    37 [ label = "gen-common" shape = box]
+    38 [ label = "datafusion-spark" shape = box]
+    39 [ label = "datafusion-sqllogictest" shape = box]
+    40 [ label = "datafusion-substrait" shape = box]
+    41 [ label = "datafusion-wasmtest" shape = box]
+    42 [ label = "datafusion-cli" shape = box]
+    43 [ label = "datafusion-examples" shape = box]
+    44 [ label = "ffi_example_table_provider" shape = box]
+    45 [ label = "ffi_module_interface" shape = box]
+    46 [ label = "ffi_module_loader" shape = box]
+    47 [ label = "datafusion-benchmarks" shape = box]
+    2 -> 3 [ ]
+    3 -> 14 [ ]
+    3 -> 18 [ ]
+    4 -> 5 [ ]
+    5 -> 6 [ ]
+    5 -> 8 [ ]
+    5 -> 10 [ ]
+    7 -> 0 [ ]
+    8 -> 9 [ ]
+    9 -> 7 [ ]
+    10 -> 9 [ ]
+    11 -> 12 [ color = blue]
+    12 -> 4 [ ]
+    12 -> 13 [ color = green3]
+    13 -> 6 [ color = green3]
+    14 -> 11 [ ]
+    15 -> 1 [ ]
+    15 -> 16 [ color = blue]
+    15 -> 17 [ color = blue]
+    16 -> 11 [ ]
+    17 -> 11 [ ]
+    18 -> 15 [ ]
+    19 -> 2 [ ]
+    19 -> 20 [ color = blue]
+    20 -> 21 [ ]
+    21 -> 3 [ ]
+    21 -> 22 [ color = blue]
+    22 -> 16 [ ]
+    23 -> 3 [ ]
+    24 -> 3 [ ]
+    25 -> 3 [ ]
+    26 -> 3 [ ]
+    27 -> 19 [ ]
+    27 -> 23 [ ]
+    27 -> 24 [ style = dotted]
+    27 -> 25 [ ]
+    27 -> 26 [ ]
+    27 -> 28 [ ]
+    27 -> 29 [ ]
+    27 -> 31 [ ]
+    27 -> 31 [ color = blue]
+    27 -> 32 [ color = blue]
+    28 -> 2 [ ]
+    29 -> 30 [ color = blue]
+    30 -> 22 [ color = blue]
+    30 -> 17 [ color = blue]
+    31 -> 21 [ ]
+    32 -> 0 [ ]
+    33 -> 34 [ ]
+    34 -> 27 [ color = blue]
+    34 -> 35 [ ]
+    35 -> 0 [ ]
+    38 -> 2 [ ]
+    38 -> 22 [ ]
+    39 -> 38 [ ]
+    39 -> 40 [ ]
+    40 -> 27 [ ]
+    40 -> 27 [ color = blue]
+    41 -> 27 [ ]
+    42 -> 27 [ ]
+    43 -> 34 [ color = blue]
+    44 -> 45 [ ]
+    45 -> 33 [ ]
+    46 -> 45 [ ]
+    47 -> 34 [ color = blue]
+}
+