build(spider): Add Dockerfile and GitHub CI for CLP-tailored Spider Huntsman worker image.

.github/workflows/clp-artifact-build.yaml

@@ -778,3 +778,71 @@ jobs:
               "${image_base}:${tag}-arm64"
             docker manifest push "${image_base}:${tag}"
           done
+
+  spider-worker-image:
+    name: "spider-worker-image"
+    # Run if the ancestor jobs were successful/skipped and building clp was successful.
+    if: >-
+      !cancelled() && !failure() && (
+        needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
+        || needs.ubuntu-jammy-x86_64-deps-image.result == 'success'
+      )
+    needs:
+      - "filter-relevant-changes"
+      - "ubuntu-jammy-x86_64-deps-image"
+    runs-on: *runner
+    steps:
+      - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd"  # v6.0.2
+        with:
+          submodules: "recursive"
+
+      - name: "Workaround actions/runner-images/issues/6775"
+        shell: "bash"
+        run: "chown $(id -u):$(id -g) -R ."
+
+      - name: "Build the CLP artifacts required for Spider worker"
+        uses: "./.github/actions/run-on-image"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}ubuntu-jammy"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          run_command: >-
+            CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN)
+            HOME=/tmp
+            task core clp-tdl-package
+
+      - name: "Login to Image Registry"
+        uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121"  # v4.1.0
+        if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'"
+        with:
+          registry: "ghcr.io"
+          username: "${{github.actor}}"
+          password: "${{secrets.GITHUB_TOKEN}}"
+
+      - name: "Sanitize the repo's name"
+        id: "sanitize_repo_name"
+        run: |-
+          # Docker doesn't support repository names with uppercase characters, so we convert to
+          # lowercase here.
+          lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')
+          echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT"
+        shell: "bash"
+
+      - id: "spider_worker_image_meta"
+        uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"  # v5.7.0
+        with:
+          images: >-
+            ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-spider-worker
+          tags: "type=raw,value=${{github.ref_name}}"
+
+      # Only publish the image if this workflow was triggered by a push to `main`.
+      # NOTE: We run the rest of the job to test that the docker image build were successful.
+      - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"  # v6.15.0
+        with:
+          context: "."
+          file: "tools/docker-images/clp-spider-worker/Dockerfile"
+          provenance: false
+          push: "${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}"
+          tags: "${{steps.spider_worker_image_meta.outputs.tags}}"
+          labels: "${{steps.spider_worker_image_meta.outputs.labels}}"

taskfile.yaml

@@ -223,6 +223,13 @@ tasks:
       cargo build --release --bins
       cargo build --release -p clp-tdl-package
 
+  clp-tdl-package:
+    deps: ["toolchains:rust"]
+    dir: "{{.ROOT_DIR}}"
+    cmd: |-
+      . "{{.G_RUST_TOOLCHAIN_ENV_FILE}}"
+      cargo build --release -p clp-tdl-package
+
   clp-mcp-server:
     - task: "uv-component"
       vars:

taskfiles/docker-images.yaml

@@ -7,3 +7,11 @@ tasks:
       - ":package-build-deps"
     cmds:
       - "./build.sh"
+
+  spider-worker:
+    dir: "{{.ROOT_DIR}}/tools/docker-images/clp-spider-worker"
+    deps:
+      - ":clp-tdl-package"
+      - ":core"
+    cmds:
+      - "./build.sh"

tools/docker-images/clp-package/build.sh

@@ -4,37 +4,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-remove_temp_file_and_prev_image() {
-    rm -f "$temp_iid_file"
-
-    [[ -z "$prev_image_id" || "$prev_image_id" == "$new_image_id" ]] && return
-
-    docker image inspect "$prev_image_id" >/dev/null 2>&1 || return
-
-    echo "Removing previous image $prev_image_id."
-    docker image remove "$prev_image_id" || true
-}
-trap remove_temp_file_and_prev_image EXIT
-
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-repo_root="${script_dir}/../../../"
-iid_file="${repo_root}/build/clp-package-image.id"
-
-prev_image_id=""
-if [[ -f "$iid_file" ]]; then
-    prev_image_id=$(<"$iid_file")
-fi
+repo_root="${script_dir}/../../.."
 
-temp_iid_file="$(mktemp)"
-new_image_id=""
-
-build_cmd=(
-    docker build
-    --pull
-    --iidfile "$temp_iid_file"
-    "$repo_root"
-    --file "${script_dir}/Dockerfile"
-)
+. "${script_dir}/../utils.sh"
 
 ubuntu_version_codename="jammy"
 if [[ -f /etc/os-release ]]; then
@@ -43,27 +16,9 @@ if [[ -f /etc/os-release ]]; then
         ubuntu_version_codename="$host_codename"
     fi
 fi
-build_cmd+=(--build-arg "UBUNTU_VERSION_CODENAME=${ubuntu_version_codename}")
-
-if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null;
-then
-    build_cmd+=(
-        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
-        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
-    )
-fi
 
-"${build_cmd[@]}"
-
-if [[ -s "$temp_iid_file" ]]; then
-    new_image_id="$(<"$temp_iid_file")"
-    echo "$new_image_id" > "$iid_file"
-
-    user="${USER:-$(whoami 2>/dev/null)}" \
-        || user=$(id -un 2>/dev/null) \
-        || user=$(id -u 2>/dev/null) \
-        || user="unknown";
-    short_id="${new_image_id#sha256:}"
-    short_id="${short_id:0:4}"
-    docker tag "$new_image_id" "clp-package:dev-${user}-${short_id}"
-fi
+build_clp_image \
+    "clp-package" \
+    "${repo_root}" \
+    "${script_dir}/Dockerfile" \
+    --build-arg "UBUNTU_VERSION_CODENAME=${ubuntu_version_codename}"

tools/docker-images/clp-spider-worker/Dockerfile

@@ -0,0 +1,26 @@
+# syntax=docker/dockerfile:1
+
+ARG SPIDER_WORKER_IMAGE=ghcr.io/y-scope/spider/worker:main
+FROM ${SPIDER_WORKER_IMAGE}
+
+ARG SPIDER_TDL_PACKAGE_DIR=/opt/clp/lib/spider-tdl-packages
+ARG CLP_HOME=/opt/clp
+
+USER root
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \
+        ca-certificates \
+        libcurl4 \
+        libmariadb3 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --link ./build/rust-targets/release/libclp_tdl_package.so \
+    ${SPIDER_TDL_PACKAGE_DIR}/clp/libclp.so
+COPY --link ./build/core/clp-s ./build/core/indexer ${CLP_HOME}/bin/
+
+ENV SPIDER_TDL_PACKAGE_DIR=${SPIDER_TDL_PACKAGE_DIR} \
+    CLP_HOME=${CLP_HOME}
+
+USER spider-user

tools/docker-images/clp-spider-worker/build.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+repo_root="${script_dir}/../../.."
+
+. "${script_dir}/../utils.sh"
+
+build_clp_image "clp-spider-worker" "${repo_root}" "${script_dir}/Dockerfile"

tools/docker-images/utils.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+remove_temp_file_and_prev_image() {
+    rm -f "$temp_iid_file"
+
+    if [[ -z "$new_image_id" ]]; then
+        rm -f "$iid_file"
+    elif [[ "$prev_image_id" == "$new_image_id" ]]; then
+        return
+    fi
+
+    [[ -z "$prev_image_id" ]] && return
+
+    docker image inspect "$prev_image_id" >/dev/null 2>&1 || return
+
+    echo "Removing previous image $prev_image_id."
+    docker image remove "$prev_image_id" || true
+}
+
+build_clp_image() {
+    local image_name="$1"
+    local repo_root="$2"
+    local dockerfile="$3"
+    shift 3
+
+    iid_file="${repo_root}/build/${image_name}-image.id"
+
+    prev_image_id=""
+    if [[ -f "$iid_file" ]]; then
+        prev_image_id=$(<"$iid_file")
+    fi
+
+    temp_iid_file="$(mktemp)"
+    new_image_id=""
+    trap remove_temp_file_and_prev_image EXIT
+
+    build_cmd=(
+        docker build
+        --pull
+        --iidfile "$temp_iid_file"
+        "$repo_root"
+        --file "$dockerfile"
+        "$@"
+    )
+
+    if command -v git >/dev/null && git -C "$repo_root" rev-parse --is-inside-work-tree >/dev/null;
+    then
+        build_cmd+=(
+            --label "org.opencontainers.image.revision=$(git -C "$repo_root" rev-parse HEAD)"
+            --label "org.opencontainers.image.source=$(git -C "$repo_root" remote get-url origin)"
+        )
+    fi
+
+    "${build_cmd[@]}"
+
+    if [[ -s "$temp_iid_file" ]]; then
+        new_image_id="$(<"$temp_iid_file")"
+        echo "$new_image_id" > "$iid_file"
+
+        user="${USER:-$(whoami 2>/dev/null)}" \
+            || user=$(id -un 2>/dev/null) \
+            || user=$(id -u 2>/dev/null) \
+            || user="unknown";
+        short_id="${new_image_id#sha256:}"
+        short_id="${short_id:0:4}"
+        docker tag "$new_image_id" "${image_name}:dev-${user}-${short_id}"
+    fi
+}