diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 2ce47c7595..235be77650 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -825,3 +825,78 @@ jobs: "${image_base}:${tag}-arm64" docker manifest push "${image_base}:${tag}" done + + spider-worker-image: + name: "spider-worker-image" + # Run if the dependency-image job succeeded or was skipped because the image is unchanged. + if: >- + !cancelled() && !failure() && ( + needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || needs.ubuntu-jammy-x86_64-deps-image.result == 'success' + ) + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-x86_64-deps-image" + runs-on: *runner + permissions: + # To check out the repository. + contents: "read" + # To publish the Spider worker image to GHCR. + packages: "write" + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 + with: + persist-credentials: false + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - name: "Build the CLP artifacts required for Spider worker" + uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}ubuntu-jammy" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + HOME=/tmp + task core clp-tdl-package + + - name: "Login to Image Registry" + uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121" # v4.1.0 + if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "spider_worker_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" # v5.7.0 + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-spider-worker + tags: "type=raw,value=${{github.ref_name}}" + + # Publish on non-PR runs of `main`, including scheduled and manually dispatched runs. + # NOTE: Pull requests still build the image to validate the Dockerfile. + - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" # v6.15.0 + with: + context: "." + file: "tools/docker-images/clp-spider-worker/Dockerfile" + provenance: false + pull: true + push: "${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}" + tags: "${{steps.spider_worker_image_meta.outputs.tags}}" + labels: "${{steps.spider_worker_image_meta.outputs.labels}}" diff --git a/taskfile.yaml b/taskfile.yaml index 6aea1f6452..5482833b46 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -223,6 +223,13 @@ tasks: cargo build --release --bins cargo build --release -p clp-tdl-package + clp-tdl-package: + deps: ["toolchains:rust"] + dir: "{{.ROOT_DIR}}" + cmd: |- + . "{{.G_RUST_TOOLCHAIN_ENV_FILE}}" + cargo build --release -p clp-tdl-package + clp-mcp-server: - task: "uv-component" vars: diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 1177075d06..ffb291fdcf 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -7,3 +7,11 @@ tasks: - ":package-build-deps" cmds: - "./build.sh" + + spider-worker: + dir: "{{.ROOT_DIR}}/tools/docker-images/clp-spider-worker" + deps: + - ":clp-tdl-package" + - ":core" + cmds: + - "./build.sh" diff --git a/tools/docker-images/clp-package/build.sh b/tools/docker-images/clp-package/build.sh index 8bb9136bb5..69bd19df32 100755 --- a/tools/docker-images/clp-package/build.sh +++ b/tools/docker-images/clp-package/build.sh @@ -4,37 +4,10 @@ set -o errexit set -o nounset set -o pipefail -remove_temp_file_and_prev_image() { - rm -f "$temp_iid_file" - - [[ -z "$prev_image_id" || "$prev_image_id" == "$new_image_id" ]] && return - - docker image inspect "$prev_image_id" >/dev/null 2>&1 || return - - echo "Removing previous image $prev_image_id." - docker image remove "$prev_image_id" || true -} -trap remove_temp_file_and_prev_image EXIT - script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -repo_root="${script_dir}/../../../" -iid_file="${repo_root}/build/clp-package-image.id" - -prev_image_id="" -if [[ -f "$iid_file" ]]; then - prev_image_id=$(<"$iid_file") -fi +repo_root="${script_dir}/../../.." -temp_iid_file="$(mktemp)" -new_image_id="" - -build_cmd=( - docker build - --pull - --iidfile "$temp_iid_file" - "$repo_root" - --file "${script_dir}/Dockerfile" -) +. "${script_dir}/../utils.sh" ubuntu_version_codename="jammy" if [[ -f /etc/os-release ]]; then @@ -43,27 +16,9 @@ if [[ -f /etc/os-release ]]; then ubuntu_version_codename="$host_codename" fi fi -build_cmd+=(--build-arg "UBUNTU_VERSION_CODENAME=${ubuntu_version_codename}") - -if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null; -then - build_cmd+=( - --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" - --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" - ) -fi -"${build_cmd[@]}" - -if [[ -s "$temp_iid_file" ]]; then - new_image_id="$(<"$temp_iid_file")" - echo "$new_image_id" > "$iid_file" - - user="${USER:-$(whoami 2>/dev/null)}" \ - || user=$(id -un 2>/dev/null) \ - || user=$(id -u 2>/dev/null) \ - || user="unknown"; - short_id="${new_image_id#sha256:}" - short_id="${short_id:0:4}" - docker tag "$new_image_id" "clp-package:dev-${user}-${short_id}" -fi +build_clp_image \ + "clp-package" \ + "${repo_root}" \ + "${script_dir}/Dockerfile" \ + --build-arg "UBUNTU_VERSION_CODENAME=${ubuntu_version_codename}" diff --git a/tools/docker-images/clp-spider-worker/Dockerfile b/tools/docker-images/clp-spider-worker/Dockerfile new file mode 100644 index 0000000000..1816346156 --- /dev/null +++ b/tools/docker-images/clp-spider-worker/Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +ARG SPIDER_WORKER_IMAGE=ghcr.io/y-scope/spider/worker:main +FROM ${SPIDER_WORKER_IMAGE} + +ARG SPIDER_TDL_PACKAGE_DIR=/opt/spider/packages +ARG CLP_HOME=/opt/clp + +USER root + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \ + ca-certificates \ + libcurl4 \ + libmariadb3 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --link ./build/rust-targets/release/libclp_tdl_package.so \ + ${SPIDER_TDL_PACKAGE_DIR}/clp/libclp.so +COPY --link ./build/core/clp-s ./build/core/indexer ./build/core/log-converter ${CLP_HOME}/bin/ + +ENV SPIDER_TDL_PACKAGE_DIR=${SPIDER_TDL_PACKAGE_DIR} \ + CLP_HOME=${CLP_HOME} + +USER spider-user diff --git a/tools/docker-images/clp-spider-worker/build.sh b/tools/docker-images/clp-spider-worker/build.sh new file mode 100755 index 0000000000..d4566a038a --- /dev/null +++ b/tools/docker-images/clp-spider-worker/build.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +repo_root="${script_dir}/../../.." + +. "${script_dir}/../utils.sh" + +build_clp_image "clp-spider-worker" "${repo_root}" "${script_dir}/Dockerfile" diff --git a/tools/docker-images/utils.sh b/tools/docker-images/utils.sh new file mode 100644 index 0000000000..d1ad99029b --- /dev/null +++ b/tools/docker-images/utils.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +remove_temp_file_and_prev_image() { + rm -f "$temp_iid_file" + + if [[ -z "$new_image_id" ]]; then + rm -f "$iid_file" + elif [[ "$prev_image_id" == "$new_image_id" ]]; then + return + fi + + [[ -z "$prev_image_id" ]] && return + + docker image inspect "$prev_image_id" >/dev/null 2>&1 || return + + echo "Removing previous image $prev_image_id." + docker image remove "$prev_image_id" || true +} + +build_clp_image() { + local image_name="$1" + local repo_root="$2" + local dockerfile="$3" + shift 3 + + iid_file="${repo_root}/build/${image_name}-image.id" + + prev_image_id="" + if [[ -f "$iid_file" ]]; then + prev_image_id=$(<"$iid_file") + fi + + temp_iid_file="$(mktemp)" + new_image_id="" + trap remove_temp_file_and_prev_image EXIT + + build_cmd=( + docker build + --pull + --iidfile "$temp_iid_file" + "$repo_root" + --file "$dockerfile" + "$@" + ) + + if command -v git >/dev/null && git -C "$repo_root" rev-parse --is-inside-work-tree >/dev/null; + then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$repo_root" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$repo_root" remote get-url origin)" + ) + fi + + "${build_cmd[@]}" + + if [[ -s "$temp_iid_file" ]]; then + new_image_id="$(<"$temp_iid_file")" + echo "$new_image_id" > "$iid_file" + + user="${USER:-$(whoami 2>/dev/null)}" \ + || user=$(id -un 2>/dev/null) \ + || user=$(id -u 2>/dev/null) \ + || user="unknown"; + short_id="${new_image_id#sha256:}" + short_id="${short_id:0:4}" + docker tag "$new_image_id" "${image_name}:dev-${user}-${short_id}" + fi +}