From 7091bafa1617d75a5d755a04a1b8e9dc9d46fdd1 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Mon, 17 Aug 2026 16:56:47 -0400 Subject: [PATCH 1/5] feat(deployment): add Spider Compose integration --- taskfile.yaml | 14 +++ .../package/compose.clp-spider.yaml | 14 +++ .../package/docker-compose-all.yaml | 21 +++++ .../package/docker-compose-spider.yaml | 88 +++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 tools/deployment/package/compose.clp-spider.yaml create mode 100644 tools/deployment/package/docker-compose-spider.yaml diff --git a/taskfile.yaml b/taskfile.yaml index 24db6a9e80..baf9f5d92e 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -37,6 +37,7 @@ vars: G_PYTHON_LIBS_DIR: "{{.G_BUILD_DIR}}/python-libs" G_PYTHON_WHEELS_BUILD_DIR: "{{.G_BUILD_DIR}}/python-wheels" G_RUST_BUILD_DIR: "{{.G_BUILD_DIR}}/rust-targets" + G_SPIDER_COMPOSE_TAR_FILE: "{{.G_BUILD_DIR}}/spider-compose-52817a78.tar.gz" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" # Taskfile paths @@ -44,6 +45,7 @@ vars: # Versions G_PACKAGE_VERSION: "0.13.1-dev" + G_SPIDER_COMPOSE_COMMIT: "52817a78c944efa478cfde94c77e19f585ff43ff" # Build parameters # NOTE: If CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK is set, it takes precedence. Otherwise, we cap @@ -62,6 +64,7 @@ vars: # Checksum files G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5" G_PYTHON_LIBS_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/python-libs.md5" + G_SPIDER_COMPOSE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/spider-compose-52817a78.md5" G_WEBUI_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui.md5" tasks: @@ -124,6 +127,7 @@ tasks: - "{{.G_BUILD_DIR}}/clp-package-image.id" - "{{.G_CONFIG_SCHEMA_BUILD_DIR}}/clp-config.schema.json" - "{{.G_WEBUI_BUILD_DIR}}/settings.json" + - "{{.G_SPIDER_COMPOSE_TAR_FILE}}" - "{{.TASKFILE}}" - "components/package-template/src/**/*" - "tools/deployment/package/**/*" @@ -137,6 +141,16 @@ tasks: INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] cmds: - "rm -rf '{{.OUTPUT_DIR}}'" + - task: "utils:remote:download-and-extract-tar" + vars: + CHECKSUM_FILE: "{{.G_SPIDER_COMPOSE_CHECKSUM_FILE}}" + FILE_SHA256: "84a666725baecb416eeba961694ab49ea65bd85f456792f839872846dd5de06e" + INCLUDE_PATTERNS: ["*/tools/deployment/spider-compose/**"] + NUM_COMPONENTS_TO_STRIP: 4 + OUTPUT_DIR: "{{.OUTPUT_DIR}}/spider-compose" + TAR_FILE: "{{.G_SPIDER_COMPOSE_TAR_FILE}}" + URL: >- + https://github.com/y-scope/spider/archive/{{.G_SPIDER_COMPOSE_COMMIT}}.tar.gz - >- rsync --archive "components/package-template/src/" diff --git a/tools/deployment/package/compose.clp-spider.yaml b/tools/deployment/package/compose.clp-spider.yaml new file mode 100644 index 0000000000..ff787608b1 --- /dev/null +++ b/tools/deployment/package/compose.clp-spider.yaml @@ -0,0 +1,14 @@ +services: + spider-worker: + image: "ghcr.io/y-scope/clp/clp-spider-worker:main" + user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" + environment: + CLP_CONFIG_PATH: "/etc/clp-config.yaml" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:-clp-user}" + volumes: + - type: "bind" + source: "${CLP_LOGS_DIR_HOST:-./var/log}/.clp-config.yaml" + target: "/etc/clp-config.yaml" + read_only: true + - "${CLP_STAGED_ARCHIVE_OUTPUT_DIR_HOST:-empty}:/var/data/staged-archives" diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index bb833e56c2..436be971ee 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -280,6 +280,27 @@ services: "--config", "/etc/clp-config.yaml" ] + compression-coordinator: + <<: *service_defaults + hostname: "compression_coordinator" + environment: + <<: *clp_telemetry_env_defaults + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:-clp-user}" + OTEL_SERVICE_NAME: "compression-coordinator" + RUST_LOG: "${CLP_COMPRESSION_COORDINATOR_LOGGING_LEVEL:-INFO}" + volumes: + - *volume_clp_config_readonly + depends_on: + db-table-creator: + condition: "service_completed_successfully" + spider-storage: + condition: "service_healthy" + command: [ + "/opt/clp/bin/compression-coordinator", + "--config", "/etc/clp-config.yaml" + ] + compression-worker: <<: *service_defaults hostname: "compression_worker" diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml new file mode 100644 index 0000000000..bf4fa16297 --- /dev/null +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -0,0 +1,88 @@ +name: "clp-package-spider" + +include: + - "docker-compose.utils.yaml" + - path: + - "spider-compose/compose.yaml" + - "compose.clp-spider.yaml" + +services: + database: + extends: + file: "docker-compose-all.yaml" + service: "database" + + db-table-creator: + extends: + file: "docker-compose-all.yaml" + service: "db-table-creator" + + queue: + extends: + file: "docker-compose-all.yaml" + service: "queue" + + redis: + extends: + file: "docker-compose-all.yaml" + service: "redis" + + results-cache: + extends: + file: "docker-compose-all.yaml" + service: "results-cache" + + results-cache-indices-creator: + extends: + file: "docker-compose-all.yaml" + service: "results-cache-indices-creator" + + compression-coordinator: + extends: + file: "docker-compose-all.yaml" + service: "compression-coordinator" + + webui: + extends: + file: "docker-compose-all.yaml" + service: "webui" + + garbage-collector: + extends: + file: "docker-compose-all.yaml" + service: "garbage-collector" + + api-server: + extends: + file: "docker-compose-all.yaml" + service: "api-server" + + query-scheduler: + extends: + file: "docker-compose-all.yaml" + service: "query-scheduler" + + query-worker: + extends: + file: "docker-compose-all.yaml" + service: "query-worker" + + reducer: + extends: + file: "docker-compose-all.yaml" + service: "reducer" + + mcp-server: + extends: + file: "docker-compose-all.yaml" + service: "mcp-server" + + log-ingestor: + extends: + file: "docker-compose-all.yaml" + service: "log-ingestor" + + otel-collector: + extends: + file: "docker-compose-all.yaml" + service: "otel-collector" From de6cf6c7e2d220c69eda636dfe40141f520ae3c4 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 18 Aug 2026 15:56:25 -0400 Subject: [PATCH 2/5] feat(deployment): Keep the Celery compression services in the Spider Compose entry point. `compression-coordinator` only handles S3 ingestion, so `compression-scheduler` and `compression-worker` are still needed for the other ingestion paths. Co-Authored-By: Claude Opus 5 (1M context) --- tools/deployment/package/docker-compose-spider.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml index bf4fa16297..330968abf1 100644 --- a/tools/deployment/package/docker-compose-spider.yaml +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -37,6 +37,16 @@ services: file: "docker-compose-all.yaml" service: "results-cache-indices-creator" + compression-scheduler: + extends: + file: "docker-compose-all.yaml" + service: "compression-scheduler" + + compression-worker: + extends: + file: "docker-compose-all.yaml" + service: "compression-worker" + compression-coordinator: extends: file: "docker-compose-all.yaml" From e4d78ac7d9974705808c0f0053999fdbc9bfa0e8 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 18 Aug 2026 15:56:25 -0400 Subject: [PATCH 3/5] feat(deployment): Make the CLP Spider worker image reference overridable. Follows the convention the rest of the package uses for `CLP_PACKAGE_CONTAINER_IMAGE_REF`, so a deployment can pin the worker to a digest without editing the Compose file. The publish workflow only ever tags this image with the branch ref, so `main` remains the default. Co-Authored-By: Claude Opus 5 (1M context) --- tools/deployment/package/compose.clp-spider.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/deployment/package/compose.clp-spider.yaml b/tools/deployment/package/compose.clp-spider.yaml index ff787608b1..3c6332177a 100644 --- a/tools/deployment/package/compose.clp-spider.yaml +++ b/tools/deployment/package/compose.clp-spider.yaml @@ -1,6 +1,7 @@ services: spider-worker: - image: "ghcr.io/y-scope/clp/clp-spider-worker:main" + image: >- + ${CLP_SPIDER_WORKER_CONTAINER_IMAGE_REF:-ghcr.io/y-scope/clp/clp-spider-worker:main} user: "${CLP_FIRST_PARTY_SERVICE_UID_GID:-1000:1000}" environment: CLP_CONFIG_PATH: "/etc/clp-config.yaml" From daada183bf620572a4ac2235a23c1dbe4e77131b Mon Sep 17 00:00:00 2001 From: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:41:39 -0400 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Junhao Liao --- taskfile.yaml | 2 +- tools/deployment/package/docker-compose-spider.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index baf9f5d92e..62fd4df888 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -147,7 +147,7 @@ tasks: FILE_SHA256: "84a666725baecb416eeba961694ab49ea65bd85f456792f839872846dd5de06e" INCLUDE_PATTERNS: ["*/tools/deployment/spider-compose/**"] NUM_COMPONENTS_TO_STRIP: 4 - OUTPUT_DIR: "{{.OUTPUT_DIR}}/spider-compose" + OUTPUT_DIR: "{{.OUTPUT_DIR}}/usr/share/spider-compose" TAR_FILE: "{{.G_SPIDER_COMPOSE_TAR_FILE}}" URL: >- https://github.com/y-scope/spider/archive/{{.G_SPIDER_COMPOSE_COMMIT}}.tar.gz diff --git a/tools/deployment/package/docker-compose-spider.yaml b/tools/deployment/package/docker-compose-spider.yaml index 330968abf1..f12c30fba8 100644 --- a/tools/deployment/package/docker-compose-spider.yaml +++ b/tools/deployment/package/docker-compose-spider.yaml @@ -3,7 +3,7 @@ name: "clp-package-spider" include: - "docker-compose.utils.yaml" - path: - - "spider-compose/compose.yaml" + - "usr/share/spider-compose/compose.yaml" - "compose.clp-spider.yaml" services: From 893d418770a4cbd33c6ed690d20eb5c9c886d6d0 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Wed, 19 Aug 2026 14:33:59 -0400 Subject: [PATCH 5/5] refactor(deployment): Drop the Spider Compose download from this PR. The download is handled by the `compose:download-spider` task, which places the deployment at `usr/share/spider-compose` in the package. This PR's entry point includes it from there. Co-Authored-By: Claude Opus 5 (1M context) --- taskfile.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 62fd4df888..24db6a9e80 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -37,7 +37,6 @@ vars: G_PYTHON_LIBS_DIR: "{{.G_BUILD_DIR}}/python-libs" G_PYTHON_WHEELS_BUILD_DIR: "{{.G_BUILD_DIR}}/python-wheels" G_RUST_BUILD_DIR: "{{.G_BUILD_DIR}}/rust-targets" - G_SPIDER_COMPOSE_TAR_FILE: "{{.G_BUILD_DIR}}/spider-compose-52817a78.tar.gz" G_WEBUI_BUILD_DIR: "{{.G_BUILD_DIR}}/webui" # Taskfile paths @@ -45,7 +44,6 @@ vars: # Versions G_PACKAGE_VERSION: "0.13.1-dev" - G_SPIDER_COMPOSE_COMMIT: "52817a78c944efa478cfde94c77e19f585ff43ff" # Build parameters # NOTE: If CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK is set, it takes precedence. Otherwise, we cap @@ -64,7 +62,6 @@ vars: # Checksum files G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5" G_PYTHON_LIBS_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/python-libs.md5" - G_SPIDER_COMPOSE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/spider-compose-52817a78.md5" G_WEBUI_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui.md5" tasks: @@ -127,7 +124,6 @@ tasks: - "{{.G_BUILD_DIR}}/clp-package-image.id" - "{{.G_CONFIG_SCHEMA_BUILD_DIR}}/clp-config.schema.json" - "{{.G_WEBUI_BUILD_DIR}}/settings.json" - - "{{.G_SPIDER_COMPOSE_TAR_FILE}}" - "{{.TASKFILE}}" - "components/package-template/src/**/*" - "tools/deployment/package/**/*" @@ -141,16 +137,6 @@ tasks: INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - - task: "utils:remote:download-and-extract-tar" - vars: - CHECKSUM_FILE: "{{.G_SPIDER_COMPOSE_CHECKSUM_FILE}}" - FILE_SHA256: "84a666725baecb416eeba961694ab49ea65bd85f456792f839872846dd5de06e" - INCLUDE_PATTERNS: ["*/tools/deployment/spider-compose/**"] - NUM_COMPONENTS_TO_STRIP: 4 - OUTPUT_DIR: "{{.OUTPUT_DIR}}/usr/share/spider-compose" - TAR_FILE: "{{.G_SPIDER_COMPOSE_TAR_FILE}}" - URL: >- - https://github.com/y-scope/spider/archive/{{.G_SPIDER_COMPOSE_COMMIT}}.tar.gz - >- rsync --archive "components/package-template/src/"