From da04bff203ec892157b1ed8eb73b8cd1fd0c0488 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Thu, 13 Aug 2026 21:04:50 -0400 Subject: [PATCH 1/9] Add the docker compose file and task to start a cluster and mount a folder --- taskfiles/docker.yaml | 23 +++++++++++++++++++ .../spider-compose/compose.local.yaml | 7 ++++++ 2 files changed, 30 insertions(+) create mode 100644 tools/deployment/spider-compose/compose.local.yaml diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 6c834fd4..8b570268 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -38,3 +38,26 @@ tasks: compose:clean: dir: "{{.G_SPIDER_COMPOSE_DIR}}" cmd: "docker compose --file compose.yaml down --volumes --remove-orphans" + + # Starts a local Spider deployment with libraries mounted into every worker. + # + # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. + compose:local:up: + env: + SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}" + requires: + vars: ["SPIDER_WORKER_LIBRARY_DIR"] + dir: "{{.G_SPIDER_COMPOSE_DIR}}" + cmd: >- + docker compose --project-name spider-local --file compose.yaml + --file compose.local.yaml up --detach --wait --wait-timeout 300 + + compose:local:down: + dir: "{{.G_SPIDER_COMPOSE_DIR}}" + cmd: "docker compose --project-name spider-local --file compose.yaml down" + + compose:local:clean: + dir: "{{.G_SPIDER_COMPOSE_DIR}}" + cmd: >- + docker compose --project-name spider-local --file compose.yaml down + --volumes --remove-orphans diff --git a/tools/deployment/spider-compose/compose.local.yaml b/tools/deployment/spider-compose/compose.local.yaml new file mode 100644 index 00000000..2a2a1f3b --- /dev/null +++ b/tools/deployment/spider-compose/compose.local.yaml @@ -0,0 +1,7 @@ +services: + spider-worker: + volumes: + - type: "bind" + source: "${SPIDER_WORKER_LIBRARY_DIR:?}" + target: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}" + read_only: true From d5f7d038804160188d3d1865fd4da0b09b070ca7 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Fri, 14 Aug 2026 13:48:02 -0400 Subject: [PATCH 2/9] Add env for image refs --- tools/deployment/spider-compose/.env.example | 4 ++++ tools/deployment/spider-compose/compose.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/deployment/spider-compose/.env.example b/tools/deployment/spider-compose/.env.example index 4ddb86d7..88a6387e 100644 --- a/tools/deployment/spider-compose/.env.example +++ b/tools/deployment/spider-compose/.env.example @@ -14,6 +14,8 @@ SPIDER_STORAGE_DB_PASSWORD=spider-password SPIDER_STORAGE_DB_USERNAME=spider-user # Storage settings. +# A complete storage image reference takes precedence over `SPIDER_IMAGE_TAG` when set. +# SPIDER_STORAGE_IMAGE_REF=ghcr.io/y-scope/spider/storage:main SPIDER_STORAGE_LOG_LEVEL=INFO SPIDER_STORAGE_PORT=50051 SPIDER_STORAGE_PUBLISHED_IP=127.0.0.1 @@ -29,6 +31,8 @@ SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC=30 SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY=128 # Scheduler settings. +# A complete scheduler image reference takes precedence over `SPIDER_IMAGE_TAG` when set. +# SPIDER_SCHEDULER_IMAGE_REF=ghcr.io/y-scope/spider/scheduler:main SPIDER_SCHEDULER_LOG_LEVEL=INFO SPIDER_SCHEDULER_PORT=50052 SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=8 diff --git a/tools/deployment/spider-compose/compose.yaml b/tools/deployment/spider-compose/compose.yaml index 6dcb3e67..de4ec976 100644 --- a/tools/deployment/spider-compose/compose.yaml +++ b/tools/deployment/spider-compose/compose.yaml @@ -51,7 +51,8 @@ services: spider-storage: <<: *service_defaults - image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}" + image: >- + ${SPIDER_STORAGE_IMAGE_REF:-ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}} networks: - "default" - "spider-internal" @@ -84,7 +85,8 @@ services: spider-scheduler: <<: *service_defaults - image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}" + image: >- + ${SPIDER_SCHEDULER_IMAGE_REF:-ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}} networks: - "spider-internal" command: [ From 8213b8b4c4cf4895ff0d3c01cb87cea84d3eced5 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Fri, 14 Aug 2026 17:00:03 -0400 Subject: [PATCH 3/9] Resolve relative path to full path --- taskfiles/docker.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 8b570268..61b94c55 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -44,9 +44,14 @@ tasks: # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. compose:local:up: env: - SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}" - requires: - vars: ["SPIDER_WORKER_LIBRARY_DIR"] + SPIDER_WORKER_LIBRARY_DIR: >- + {{- if .SPIDER_WORKER_LIBRARY_DIR}} + {{- if osIsAbs .SPIDER_WORKER_LIBRARY_DIR}} + {{- osClean .SPIDER_WORKER_LIBRARY_DIR}} + {{- else}} + {{- joinPath .ROOT_DIR .SPIDER_WORKER_LIBRARY_DIR}} + {{- end}} + {{- end}} dir: "{{.G_SPIDER_COMPOSE_DIR}}" cmd: >- docker compose --project-name spider-local --file compose.yaml From f0fb6894b9980a342f6e962931412ab84f183932 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 18 Aug 2026 00:10:28 -0400 Subject: [PATCH 4/9] Add image ref for env --- taskfiles/docker.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 61b94c55..e4a659fe 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -41,9 +41,15 @@ tasks: # Starts a local Spider deployment with libraries mounted into every worker. # + # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference. + # @param {string} SPIDER_SCHEDULER_IMAGE_REF The scheduler image reference. + # @param {string} SPIDER_WORKER_IMAGE_REF The worker image reference. # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. compose:local:up: env: + SPIDER_STORAGE_IMAGE_REF: "{{.SPIDER_STORAGE_IMAGE_REF}}" + SPIDER_SCHEDULER_IMAGE_REF: "{{.SPIDER_SCHEDULER_IMAGE_REF}}" + SPIDER_WORKER_IMAGE_REF: "{{.SPIDER_WORKER_IMAGE_REF}}" SPIDER_WORKER_LIBRARY_DIR: >- {{- if .SPIDER_WORKER_LIBRARY_DIR}} {{- if osIsAbs .SPIDER_WORKER_LIBRARY_DIR}} From ed735a318a6eb514d75e28f7b2017eff1f6f3f1a Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 18 Aug 2026 13:13:21 -0400 Subject: [PATCH 5/9] Add build and up task --- taskfiles/docker.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index e4a659fe..91918590 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -39,6 +39,23 @@ tasks: dir: "{{.G_SPIDER_COMPOSE_DIR}}" cmd: "docker compose --file compose.yaml down --volumes --remove-orphans" + # Builds Spider and its images, then starts a local deployment. + # + # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. + compose:local:build-and-up: + cmds: + - task: ":build:rust" + - task: "build" + - task: "compose:local:up" + vars: + SPIDER_STORAGE_IMAGE_REF: + sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-storage-image.id'" + SPIDER_SCHEDULER_IMAGE_REF: + sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-scheduler-image.id'" + SPIDER_WORKER_IMAGE_REF: + sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-worker-image.id'" + SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}" + # Starts a local Spider deployment with libraries mounted into every worker. # # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference. From 8daa1fd9889fcb15364dcfa46005a04bf18606e9 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 18 Aug 2026 22:21:10 -0400 Subject: [PATCH 6/9] Fix name --- taskfiles/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 53639494..67a4d546 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -58,7 +58,7 @@ tasks: compose:local:build-and-up: cmds: - task: ":build:rust" - - task: "build" + - task: ":docker:build" - task: "compose:local:up" vars: SPIDER_STORAGE_IMAGE_REF: From e7a6215d145bca192adc60d89fbe3717e951a529 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 19 Aug 2026 11:15:10 -0400 Subject: [PATCH 7/9] Apply suggestions from code review Co-authored-by: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> --- taskfiles/docker.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 67a4d546..77e5a601 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -52,7 +52,8 @@ tasks: dir: "{{.G_SPIDER_COMPOSE_DIR}}" cmd: "docker compose --file compose.yaml down --volumes --remove-orphans" - # Builds Spider and its images, then starts a local deployment. + # Builds Spider's Rust binaries and container images, then starts a local Docker Compose + # deployment that uses the freshly built images. # # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. compose:local:build-and-up: @@ -69,7 +70,11 @@ tasks: sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-worker-image.id'" SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}" - # Starts a local Spider deployment with libraries mounted into every worker. + # Starts a local Docker Compose deployment with the library directory mounted + # into every worker. + # + # NOTE: Users can override image refs for each service; when unset, each defaults to + # `ghcr.io/y-scope/spider/:${SPIDER_IMAGE_TAG:-main}`. # # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference. # @param {string} SPIDER_SCHEDULER_IMAGE_REF The scheduler image reference. From 65601b9ba5bc6075698ad2e0783e48d4bf413bce Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Wed, 19 Aug 2026 12:03:27 -0400 Subject: [PATCH 8/9] Fix image pull policy --- taskfiles/docker.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 77e5a601..fc6ee5b4 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -57,11 +57,13 @@ tasks: # # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. compose:local:build-and-up: + env: cmds: - task: ":build:rust" - task: ":docker:build" - task: "compose:local:up" vars: + SPIDER_PULL_POLICY: "never" SPIDER_STORAGE_IMAGE_REF: sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-storage-image.id'" SPIDER_SCHEDULER_IMAGE_REF: @@ -76,12 +78,14 @@ tasks: # NOTE: Users can override image refs for each service; when unset, each defaults to # `ghcr.io/y-scope/spider/:${SPIDER_IMAGE_TAG:-main}`. # + # @param {string} SPIDER_PULL_POLICY The image pull policy. # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference. # @param {string} SPIDER_SCHEDULER_IMAGE_REF The scheduler image reference. # @param {string} SPIDER_WORKER_IMAGE_REF The worker image reference. # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. compose:local:up: env: + SPIDER_PULL_POLICY: "{{.SPIDER_PULL_POLICY}}" SPIDER_STORAGE_IMAGE_REF: "{{.SPIDER_STORAGE_IMAGE_REF}}" SPIDER_SCHEDULER_IMAGE_REF: "{{.SPIDER_SCHEDULER_IMAGE_REF}}" SPIDER_WORKER_IMAGE_REF: "{{.SPIDER_WORKER_IMAGE_REF}}" From b640495beb8dca78562d3c543f46e4b2fa6be46f Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Thu, 20 Aug 2026 12:19:53 -0400 Subject: [PATCH 9/9] Remove build-and-up --- taskfiles/docker.yaml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/taskfiles/docker.yaml b/taskfiles/docker.yaml index 61efed41..a6751252 100644 --- a/taskfiles/docker.yaml +++ b/taskfiles/docker.yaml @@ -52,33 +52,13 @@ tasks: dir: "{{.G_SPIDER_COMPOSE_DIR}}" cmd: "docker compose --file compose.yaml down --volumes --remove-orphans" - # Builds Spider's Rust binaries and container images, then starts a local Docker Compose - # deployment that uses the freshly built images. - # - # @param {string} SPIDER_WORKER_LIBRARY_DIR The host directory containing the libraries. - compose:local:build-and-up: - env: - cmds: - - task: ":build:rust" - - task: ":docker:build" - - task: "compose:local:up" - vars: - SPIDER_PULL_POLICY: "never" - SPIDER_STORAGE_IMAGE_REF: - sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-storage-image.id'" - SPIDER_SCHEDULER_IMAGE_REF: - sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-scheduler-image.id'" - SPIDER_WORKER_IMAGE_REF: - sh: "sed -n '1p' '{{.G_BUILD_DIR}}/spider-worker-image.id'" - SPIDER_WORKER_LIBRARY_DIR: "{{.SPIDER_WORKER_LIBRARY_DIR}}" - # Starts a local Docker Compose deployment with the library directory mounted # into every worker. # # NOTE: Users can override image refs for each service; when unset, each defaults to # `ghcr.io/y-scope/spider/:${SPIDER_IMAGE_TAG:-main}`. # - # @param {string} SPIDER_PULL_POLICY The image pull policy. + # @param {string} SPIDER_PULL_POLICY The image pull policy. If not set, default to "always". # @param {string} SPIDER_STORAGE_IMAGE_REF The storage image reference. # @param {string} SPIDER_SCHEDULER_IMAGE_REF The scheduler image reference. # @param {string} SPIDER_WORKER_IMAGE_REF The worker image reference.