feat(docker-compose): Add Docker Compose stack that mounts a local directory.

taskfiles/docker.yaml

@@ -38,3 +38,54 @@ tasks:
   compose:clean:
     dir: "{{.G_SPIDER_COMPOSE_DIR}}"
     cmd: "docker compose --file compose.yaml down --volumes --remove-orphans"
+
+  # Builds Spider and its images, then starts a local deployment.
+  #
+  # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries.
+  compose:local:build-and-up:
+    cmds:
+      - task: ":build:rust"
+      - task: "build"
+      - task: "compose:local:up"
+        vars:
+          SPIDER_STORAGE_IMAGE_REF:
+            sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-storage-image.id'"
+          SPIDER_SCHEDULER_IMAGE_REF:
+            sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-scheduler-image.id'"
+          SPIDER_WORKER_IMAGE_REF:
+            sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-worker-image.id'"
+          SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}"
+
+  # Starts a local Spider deployment with libraries mounted into every worker.
+  #
+  # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference.
+  # @param {string} SPIDER_SCHEDULER_IMAGE_REF The scheduler image reference.
+  # @param {string} SPIDER_WORKER_IMAGE_REF The worker image reference.
+  # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries.
+  compose:local:up:
+    env:
+      SPIDER_STORAGE_IMAGE_REF: "{{.SPIDER_STORAGE_IMAGE_REF}}"
+      SPIDER_SCHEDULER_IMAGE_REF: "{{.SPIDER_SCHEDULER_IMAGE_REF}}"
+      SPIDER_WORKER_IMAGE_REF: "{{.SPIDER_WORKER_IMAGE_REF}}"
+      SPIDER_WORKER_LIBRARY_DIR: >-
+        {{- if .SPIDER_WORKER_LIBRARY_DIR}}
+          {{- if osIsAbs .SPIDER_WORKER_LIBRARY_DIR}}
+            {{- osClean .SPIDER_WORKER_LIBRARY_DIR}}
+          {{- else}}
+            {{- joinPath .ROOT_DIR .SPIDER_WORKER_LIBRARY_DIR}}
+          {{- end}}
+        {{- end}}
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: >-
+      docker compose --project-name spider-local --file compose.yaml
+      --file compose.local.yaml up --detach --wait --wait-timeout 300
+
+  compose:local:down:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --project-name spider-local --file compose.yaml down"
+
+  compose:local:clean:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: >-
+      docker compose --project-name spider-local --file compose.yaml down
+      --volumes --remove-orphans

tools/deployment/spider-compose/.env.example

@@ -14,6 +14,8 @@ SPIDER_STORAGE_DB_PASSWORD=spider-password
 SPIDER_STORAGE_DB_USERNAME=spider-user
 
 # Storage settings.
+# A complete storage image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
+# SPIDER_STORAGE_IMAGE_REF=ghcr.io/y-scope/spider/storage:main
 SPIDER_STORAGE_LOG_LEVEL=INFO
 SPIDER_STORAGE_PORT=50051
 SPIDER_STORAGE_PUBLISHED_IP=127.0.0.1
@@ -29,6 +31,8 @@ SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC=30
 SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY=128
 
 # Scheduler settings.
+# A complete scheduler image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
+# SPIDER_SCHEDULER_IMAGE_REF=ghcr.io/y-scope/spider/scheduler:main
 SPIDER_SCHEDULER_LOG_LEVEL=INFO
 SPIDER_SCHEDULER_PORT=50052
 SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=8

tools/deployment/spider-compose/compose.local.yaml

@@ -0,0 +1,7 @@
+services:
+  spider-worker:
+    volumes:
+      - type: "bind"
+        source: "${SPIDER_WORKER_LIBRARY_DIR:?}"
+        target: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}"
+        read_only: true

tools/deployment/spider-compose/compose.yaml

@@ -51,7 +51,8 @@ services:
 
   spider-storage:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
+    image: >-
+      ${SPIDER_STORAGE_IMAGE_REF:-ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}}
     networks:
       - "default"
       - "spider-internal"
@@ -84,7 +85,8 @@ services:
 
   spider-scheduler:
     <<: *service_defaults
-    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
+    image: >-
+      ${SPIDER_SCHEDULER_IMAGE_REF:-ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}}
     networks:
       - "spider-internal"
     command: [