feat(docker-compose): Add Docker Compose deployment.

.gitignore

@@ -10,3 +10,6 @@ build
 # IDE-related directories and files
 .idea
 cmake-build-debug
+
+# Local Docker Compose credentials
+tools/deployment/spider-compose/.env

taskfiles/docker.yaml

@@ -2,6 +2,7 @@ version: "3"
 
 vars:
   G_BUILD_SCRIPT: "{{.ROOT_DIR}}/tools/docker/build.sh"
+  G_SPIDER_COMPOSE_DIR: "{{.ROOT_DIR}}/tools/deployment/spider-compose"
 
 tasks:
   build:
@@ -21,3 +22,19 @@ tasks:
   build-worker:
     deps: [":init"]
     cmd: "'{{.G_BUILD_SCRIPT}}' worker"
+
+  compose:validate:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --env-file .env.example --file compose.yaml config --quiet"
+
+  compose:up:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml up --detach --wait"
+
+  compose:down:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml down"
+
+  compose:clean:
+    dir: "{{.G_SPIDER_COMPOSE_DIR}}"
+    cmd: "docker compose --file compose.yaml down --volumes --remove-orphans"

taskfiles/lint.yaml

@@ -187,7 +187,8 @@ tasks:
           .github/ \
           docs/ \
           taskfile.yaml \
-          taskfiles/
+          taskfiles/ \
+          tools/deployment/spider-compose/
 
   toml-check:
     cmds:

tools/deployment/spider-compose/.env.example

@@ -0,0 +1,60 @@
+# Optionally copy this file to `.env` to override Spider's embedded defaults.
+
+# Deployment settings.
+SPIDER_IMAGE_TAG=main
+SPIDER_PULL_POLICY=always
+SPIDER_RESTART_POLICY=on-failure:3
+
+# Database settings.
+SPIDER_DATABASE_IMAGE_REF=mariadb:10.11.16
+SPIDER_DATABASE_NAME=spider-db
+SPIDER_DATABASE_PORT=3306
+SPIDER_DATABASE_ROOT_PASSWORD=spider-root-password
+SPIDER_STORAGE_DB_PASSWORD=spider-password
+SPIDER_STORAGE_DB_USERNAME=spider-user
+
+# Storage settings.
+SPIDER_STORAGE_LOG_LEVEL=INFO
+SPIDER_STORAGE_PORT=50051
+SPIDER_STORAGE_PUBLISHED_IP=127.0.0.1
+SPIDER_STORAGE_PUBLISHED_PORT=50051
+SPIDER_STORAGE_DB_MAX_CONNECTIONS=64
+SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY=256
+SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY=256
+SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY=1048576
+SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC=30
+SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC=300
+SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC=60
+SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC=30
+SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY=128
+
+# Scheduler settings.
+SPIDER_SCHEDULER_LOG_LEVEL=INFO
+SPIDER_SCHEDULER_PORT=50052
+SPIDER_SCHEDULER_CONNECTION_POOL_SIZE=8
+SPIDER_SCHEDULER_STOP_TIMEOUT_SEC=30
+SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC=30
+SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS=1000
+SPIDER_SCHEDULER_POLICY=round_robin
+SPIDER_SCHEDULER_ROUND_ROBIN_ACTIVE_JOB_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_ROUND_ROBIN_CLEANUP_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_ROUND_ROBIN_COMMIT_READY_TASK_CAPACITY=256
+SPIDER_SCHEDULER_ROUND_ROBIN_DISPATCH_QUEUE_CAPACITY=16
+SPIDER_SCHEDULER_ROUND_ROBIN_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC=300
+SPIDER_SCHEDULER_ROUND_ROBIN_READY_TASK_CAPACITY=1048576
+SPIDER_SCHEDULER_ROUND_ROBIN_STORAGE_POLL_TIMEOUT_MS=10
+SPIDER_SCHEDULER_ROUND_ROBIN_TICK_INTERVAL_MS=5
+
+# Worker settings.
+# A complete worker image reference takes precedence over `SPIDER_IMAGE_TAG` when set.
+# SPIDER_WORKER_IMAGE_REF=ghcr.io/y-scope/spider/worker:main
+SPIDER_WORKER_REPLICAS=4
+SPIDER_WORKER_LOG_LEVEL=INFO
+SPIDER_WORKER_CONNECTION_POOL_SIZE=4
+SPIDER_WORKER_LIVENESS_SCHEDULER_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS=1000
+SPIDER_WORKER_LIVENESS_STORAGE_HEARTBEAT_INTERVAL_SEC=1
+SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH=/usr/local/bin/spider-task-executor
+SPIDER_WORKER_INHERITED_ENV=[]
+SPIDER_WORKER_MAX_LOG_LINE_BYTES=65536
+SPIDER_WORKER_PACKAGE_DIR=/opt/spider/packages

tools/deployment/spider-compose/compose.yaml

@@ -0,0 +1,143 @@
+name: "spider"
+
+include:
+  - "./configs/scheduler.yaml"
+  - "./configs/storage.yaml"
+  - "./configs/worker.yaml"
+
+x-service-defaults: &service_defaults
+  logging:
+    driver: "local"
+  pull_policy: "${SPIDER_PULL_POLICY:-always}"
+  restart: "${SPIDER_RESTART_POLICY:-on-failure:3}"
+  stop_grace_period: "60s"
+
+x-healthcheck-defaults: &healthcheck_defaults
+  interval: "30s"
+  retries: 3
+  start_interval: "2s"
+  start_period: "60s"
+  timeout: "2s"
+
+services:
+  spider-database:
+    <<: *service_defaults
+    image: "${SPIDER_DATABASE_IMAGE_REF:-mariadb:10.11.16}"
+    networks:
+      - "spider-internal"
+    command: [
+      "--port=${SPIDER_DATABASE_PORT:-3306}",
+    ]
+    environment:
+      MYSQL_DATABASE: "${SPIDER_DATABASE_NAME:-spider-db}"
+      MYSQL_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:-spider-password}"
+      MYSQL_ROOT_PASSWORD: "${SPIDER_DATABASE_ROOT_PASSWORD:-spider-root-password}"
+      MYSQL_USER: "${SPIDER_STORAGE_DB_USERNAME:-spider-user}"
+    volumes:
+      - type: "volume"
+        source: "spider-database-data"
+        target: "/var/lib/mysql"
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "mysqladmin", "ping",
+        "--silent",
+        "-h", "127.0.0.1",
+        "-P", "${SPIDER_DATABASE_PORT:-3306}",
+        "-u", "${SPIDER_STORAGE_DB_USERNAME:-spider-user}",
+        "--password=${SPIDER_STORAGE_DB_PASSWORD:-spider-password}",
+      ]
+
+  spider-storage:
+    <<: *service_defaults
+    image: "ghcr.io/y-scope/spider/storage:${SPIDER_IMAGE_TAG:-main}"
+    networks:
+      - "default"
+      - "spider-internal"
+    command: [
+      "spider-storage",
+      "--config", "/etc/spider/storage.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_STORAGE_LOG_LEVEL:-INFO}"
+      SPIDER_STORAGE_DB_PASSWORD: "${SPIDER_STORAGE_DB_PASSWORD:-spider-password}"
+      SPIDER_STORAGE_DB_USERNAME: "${SPIDER_STORAGE_DB_USERNAME:-spider-user}"
+    ports:
+      - host_ip: "${SPIDER_STORAGE_PUBLISHED_IP:-127.0.0.1}"
+        published: "${SPIDER_STORAGE_PUBLISHED_PORT:-50051}"
+        target: "${SPIDER_STORAGE_PORT:-50051}"
+    configs:
+      - source: "spider-storage-config"
+        target: "/etc/spider/storage.yaml"
+    depends_on:
+      spider-database:
+        condition: "service_healthy"
+        restart: true
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "timeout", "2",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_STORAGE_PORT:-50051}",
+      ]
+
+  spider-scheduler:
+    <<: *service_defaults
+    image: "ghcr.io/y-scope/spider/scheduler:${SPIDER_IMAGE_TAG:-main}"
+    networks:
+      - "spider-internal"
+    command: [
+      "spider-scheduler",
+      "--config", "/etc/spider/scheduler.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_SCHEDULER_LOG_LEVEL:-INFO}"
+    configs:
+      - source: "spider-scheduler-config"
+        target: "/etc/spider/scheduler.yaml"
+    depends_on:
+      spider-storage:
+        condition: "service_healthy"
+        restart: true
+    healthcheck:
+      <<: *healthcheck_defaults
+      test: [
+        "CMD",
+        "timeout", "2",
+        "bash", "-c", "</dev/tcp/127.0.0.1/${SPIDER_SCHEDULER_PORT:-50052}",
+      ]
+
+  spider-worker:
+    <<: *service_defaults
+    image: >-
+      ${SPIDER_WORKER_IMAGE_REF:-ghcr.io/y-scope/spider/worker:${SPIDER_IMAGE_TAG:-main}}
+    networks:
+      - "default"
+      - "spider-internal"
+    command: [
+      "spider-execution-manager",
+      "--config", "/etc/spider/worker.yaml",
+    ]
+    environment:
+      RUST_LOG: "${SPIDER_WORKER_LOG_LEVEL:-INFO}"
+    configs:
+      - source: "spider-worker-config"
+        target: "/etc/spider/worker.yaml"
+    depends_on:
+      spider-scheduler:
+        condition: "service_healthy"
+        restart: true
+      spider-storage:
+        condition: "service_healthy"
+        restart: true
+    deploy:
+      replicas: "${SPIDER_WORKER_REPLICAS:-4}"
+
+networks:
+  default:
+  spider-internal:
+    internal: true
+
+volumes:
+  spider-database-data:

tools/deployment/spider-compose/configs/scheduler.yaml

@@ -0,0 +1,36 @@
+configs:
+  spider-scheduler-config:
+    content: |
+      connection_pool_size: ${SPIDER_SCHEDULER_CONNECTION_POOL_SIZE:-8}
+      host: "0.0.0.0"
+      port: ${SPIDER_SCHEDULER_PORT:-50052}
+      runtime:
+        advertised_endpoint:
+          host: "spider-scheduler"
+          port: ${SPIDER_SCHEDULER_PORT:-50052}
+        em_registry:
+          dead_em_cutoff_sec: ${SPIDER_SCHEDULER_EM_REGISTRY_DEAD_EM_CUTOFF_SEC:-30}
+          liveness_tracking_interval_ms:
+            ${SPIDER_SCHEDULER_EM_REGISTRY_LIVENESS_TRACKING_INTERVAL_MS:-1000}
+        scheduler:
+          policy: "${SPIDER_SCHEDULER_POLICY:-round_robin}"
+          config:
+            active_job_queue_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_ACTIVE_JOB_QUEUE_CAPACITY:-16}
+            cleanup_ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_CLEANUP_READY_TASK_CAPACITY:-256}
+            commit_ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_COMMIT_READY_TASK_CAPACITY:-256}
+            dispatch_queue_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_DISPATCH_QUEUE_CAPACITY:-16}
+            finalizing_job_expiration_timeout_sec:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_FINALIZING_JOB_EXPIRATION_TIMEOUT_SEC:-300}
+            ready_task_capacity:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_READY_TASK_CAPACITY:-1048576}
+            storage_poll_timeout_ms:
+              ${SPIDER_SCHEDULER_ROUND_ROBIN_STORAGE_POLL_TIMEOUT_MS:-10}
+            tick_interval_ms: ${SPIDER_SCHEDULER_ROUND_ROBIN_TICK_INTERVAL_MS:-5}
+        stop_timeout_sec: ${SPIDER_SCHEDULER_STOP_TIMEOUT_SEC:-30}
+      storage_endpoint:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:-50051}

tools/deployment/spider-compose/configs/storage.yaml

@@ -0,0 +1,25 @@
+configs:
+  spider-storage-config:
+    content: |
+      host: "0.0.0.0"
+      port: ${SPIDER_STORAGE_PORT:-50051}
+      runtime:
+        db:
+          host: "spider-database"
+          max_connections: ${SPIDER_STORAGE_DB_MAX_CONNECTIONS:-64}
+          name: "${SPIDER_DATABASE_NAME:-spider-db}"
+          port: ${SPIDER_DATABASE_PORT:-3306}
+        inbound_queue:
+          cleanup_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_CLEANUP_CAPACITY:-256}
+          commit_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_COMMIT_CAPACITY:-256}
+          task_capacity: ${SPIDER_STORAGE_INBOUND_QUEUE_TASK_CAPACITY:-1048576}
+        job_cache_gc:
+          gc_interval_sec: ${SPIDER_STORAGE_JOB_CACHE_GC_INTERVAL_SEC:-30}
+          terminated_job_retention_sec:
+            ${SPIDER_STORAGE_JOB_CACHE_TERMINATED_JOB_RETENTION_SEC:-300}
+        task_instance_pool:
+          execution_manager_stale_cutoff_sec:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_EXECUTION_MANAGER_STALE_CUTOFF_SEC:-60}
+          gc_interval_sec: ${SPIDER_STORAGE_TASK_INSTANCE_POOL_GC_INTERVAL_SEC:-30}
+          message_channel_capacity:
+            ${SPIDER_STORAGE_TASK_INSTANCE_POOL_MESSAGE_CHANNEL_CAPACITY:-128}

tools/deployment/spider-compose/configs/worker.yaml

@@ -0,0 +1,21 @@
+configs:
+  spider-worker-config:
+    content: |
+      connection_pool_size: ${SPIDER_WORKER_CONNECTION_POOL_SIZE:-4}
+      liveness:
+        scheduler_heartbeat_interval_sec:
+          ${SPIDER_WORKER_LIVENESS_SCHEDULER_HEARTBEAT_INTERVAL_SEC:-1}
+        storage_heartbeat_interval_sec:
+          ${SPIDER_WORKER_LIVENESS_STORAGE_HEARTBEAT_INTERVAL_SEC:-1}
+      scheduler:
+        host: "spider-scheduler"
+        port: ${SPIDER_SCHEDULER_PORT:-50052}
+      scheduler_poll_wait_ms: ${SPIDER_WORKER_SCHEDULER_POLL_WAIT_MS:-1000}
+      storage:
+        host: "spider-storage"
+        port: ${SPIDER_STORAGE_PORT:-50051}
+      task_executor:
+        bin_path: "${SPIDER_WORKER_TASK_EXECUTOR_BIN_PATH:-/usr/local/bin/spider-task-executor}"
+        inherited_env: ${SPIDER_WORKER_INHERITED_ENV:-[]}
+        max_log_line_bytes: ${SPIDER_WORKER_MAX_LOG_LINE_BYTES:-65536}
+        package_dir: "${SPIDER_WORKER_PACKAGE_DIR:-/opt/spider/packages}"