From 3e01ce0ad6a2c8efc8b0c94192a6ed6c16ebb15a Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 13 Jul 2026 10:55:56 -0400 Subject: [PATCH 01/16] feat(packaging): Add cached local package builds --- .dockerignore | 1 + .gitignore | 1 + README.md | 2 +- taskfiles/velox-connector/main.yaml | 3 + tools/build-packages/README.md | 30 ++++- tools/build-packages/build-packages.sh | 123 ++++++++++++++++++ .../dependency-image/Dockerfile | 2 +- .../internal/build-cache/README.md | 45 +++++++ .../internal/build-cache/container.sh | 22 ++++ .../internal/build-cache/host.sh | 25 ++++ .../internal/container/build-artifacts.sh | 17 ++- 11 files changed, 264 insertions(+), 7 deletions(-) create mode 100755 tools/build-packages/build-packages.sh create mode 100644 tools/build-packages/internal/build-cache/README.md create mode 100644 tools/build-packages/internal/build-cache/container.sh create mode 100644 tools/build-packages/internal/build-cache/host.sh diff --git a/.dockerignore b/.dockerignore index 6e98d69..a9bcbf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .git .task +.cache build **/target packages diff --git a/.gitignore b/.gitignore index 1902b9e..cead14b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /.task/ +/.cache/ /build/ /packages/ diff --git a/README.md b/README.md index ad14dec..e7862da 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,6 @@ one of the tasks in the table below. ## Building installable packages -For details about the `.deb`, `.rpm`, and `.tar.gz` artifacts built in CI, see [tools/build-packages/README.md](tools/build-packages/README.md). +To build `.deb`, `.rpm`, and `.tar.gz` artifacts of the plugin locally, see [tools/build-packages/README.md](tools/build-packages/README.md). [Task]: https://taskfile.dev diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml index 09c9d87..1790906 100644 --- a/taskfiles/velox-connector/main.yaml +++ b/taskfiles/velox-connector/main.yaml @@ -9,6 +9,8 @@ includes: vars: G_VELOX_CONNECTOR_BUILD_DIR: "{{.G_BUILD_DIR}}/velox-connector" G_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" + DEFAULT_FETCHCONTENT_BASE_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/_deps" + G_FETCHCONTENT_BASE_DIR: '{{env "FETCHCONTENT_BASE_DIR" | default .DEFAULT_FETCHCONTENT_BASE_DIR}}' tasks: build: @@ -41,5 +43,6 @@ tasks: EXTRA_ARGS: - "-DLIBCLP_PLUGIN_VELOX_CONNECTOR_DEPS_CMAKE_SETTINGS=\ {{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake" + - "-DFETCHCONTENT_BASE_DIR={{.G_FETCHCONTENT_BASE_DIR}}" - "-DPRESTO_GIT_TAG={{.G_PRESTO_GIT_TAG}}" SOURCE_DIR: "{{.ROOT_DIR}}/velox-connector" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 84d02b6..9d9e249 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -4,8 +4,9 @@ This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts for th Presto connector (coordinator + worker) on `amd64` and `arm64`. CI packaging runs `tools/build-packages/internal/container/build-artifacts.sh` -through `.github/workflows/build-packages.yaml`. A follow-up adds a local -entrypoint that reuses the same container implementation. +through `.github/workflows/build-packages.yaml`. Local builds use +`build-packages.sh`, which resolves the build-env image and invokes the same +container-side script. Supported package version format: must start with a digit and use only `[0-9A-Za-z.+~-]`. @@ -15,3 +16,28 @@ For command options, run `--help` on the relevant entry point. Default outputs are written to `./packages`. `coordinator/` contains `clp-plugin-presto-connector.jar`; `worker/` contains `libclp-plugin-velox-connector.so` and bundled non-system runtime `.so` files. + +## Local usage + +```bash +./tools/build-packages/build-packages.sh +``` + +Use `--output DIR` and `--version VER` to override the output directory and +package version (otherwise derived from `presto-connector/pom.xml`). + +The build runs inside a hash-tagged **build-env image** (`env-`) based on +`manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker +cache, this repository's GHCR package, or a local build, reusing the cached +image on later runs. + +Build state is cached under `.cache/` (`maven/`, `ccache/`, and +`fetchcontent//`), shared across build-env revisions; see +[internal/build-cache/README.md](internal/build-cache/README.md). The container +runs as root, so `.cache/`, `build/`, and `presto-connector/target/` may hold +root-owned files, while `packages/` is owned by the invoking user. + +### Prerequisites + +Docker with buildx (usable without `sudo`), git, `sha256sum`, and ~10 GB free +disk for the build-env image. diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh new file mode 100755 index 0000000..7612a18 --- /dev/null +++ b/tools/build-packages/build-packages.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +# User-facing entry point for packaging. Resolves the build-env image, then runs +# internal/container/build-artifacts.sh inside it. +# +# Requires: docker (with buildx), git, sha256sum. + +set -o errexit +set -o nounset +set -o pipefail + +src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" +# shellcheck source=tools/build-packages/internal/build-cache/host.sh +source "${src}/tools/build-packages/internal/build-cache/host.sh" + +show_help() { + cat <<'EOF' +Usage: ./tools/build-packages/build-packages.sh [OPTIONS] + +User-facing entry point for packaging. Resolves the build-env image, then runs +internal/container/build-artifacts.sh inside it. + +Options are forwarded to internal/container/build-artifacts.sh: + --output DIR Output directory for built packages (default: ./packages) + --version VER Override package version + (default: derived from presto-connector/pom.xml) + VER must start with a digit and use [0-9A-Za-z.+~-] + --help Show this help + +See tools/build-packages/README.md for details. +EOF +} + +# Resolve the output directory on the host before copying completed artifacts +# into it. Other arguments are forwarded unchanged to build-artifacts.sh. +output_dir="${src}/packages" +build_args=() +while [[ $# -gt 0 ]]; do + case $1 in + --output) + [[ -n "${2:-}" ]] || { echo >&2 "ERROR: --output requires a value"; exit 1; } + output_dir="$2" + shift 2 + ;; + --help) + show_help + exit 0 + ;; + *) + build_args+=("$1") + shift + ;; + esac +done + +# Run the wrapper as the intended artifact owner. Using sudo would make the +# staging directories and copied artifacts root-owned. +if (( EUID == 0 )); then + echo >&2 "ERROR: build-packages.sh must run as a non-root user; do not invoke it with sudo." + exit 1 +fi + +if [[ "${output_dir}" != /* ]]; then + output_dir="${src}/${output_dir}" +fi +mkdir -p "${output_dir}" +output_dir="$(cd "${output_dir}" && pwd)" + +# Initialize submodules on the host so the source tree is complete before it is +# bind-mounted into the build container. +echo "==> Initializing submodules..." +git -C "${src}" submodule update --init --recursive + +echo "==> Resolving build-env image..." +image=$("${src}/tools/build-packages/build-dependency-image.sh") +# FetchContent build state is compatible only with the image inputs identified +# by this hash. +image_hash="${image##*:env-}" +if [[ "${image_hash}" == "${image}" ]]; then + echo >&2 "ERROR: build-env image lacks an env- tag: ${image}" + exit 1 +fi + +# Keep root-owned container output in temporary staging, then copy it to the +# requested directory as the invoking host user. +stage_dir=$(mktemp -d) +trap 'rm -rf "${stage_dir}"' EXIT +artifact_stage="${stage_dir}/artifacts" +mkdir -p "${artifact_stage}" +prepare_build_cache "${src}/.cache" "${image_hash}" + +# Use a stable container checkout path so cached CMake state does not embed the +# host checkout path. HOME and Task scratch data remain in the disposable +# container. +echo "==> Running internal/container/build-artifacts.sh inside ${image}..." +docker run --rm \ + --mount "type=bind,src=${src},dst=/repo" \ + --mount "type=bind,src=${artifact_stage},dst=/output" \ + --env "BUILD_CACHE_KEY=${image_hash}" \ + -w /repo \ + "${image}" \ + bash -c ' + set -o errexit + set -o nounset + set -o pipefail + export BUILD_CACHE_DIR=/repo/.cache + export HOME=/tmp/clp-plugin-presto-connector-home + export TASK_TEMP_DIR=/tmp/clp-plugin-presto-connector-task + source tools/build-packages/internal/build-cache/container.sh + mkdir -p "${HOME}" "${TASK_TEMP_DIR}" + umask 0022 + echo "==> Running the package build as root..." + exec bash tools/build-packages/internal/container/build-artifacts.sh "$@" + ' bash --output /output ${build_args[@]+"${build_args[@]}"} + +echo "==> Copying package artifacts to ${output_dir}..." +# Fail clearly when the build produced no artifacts, instead of passing a +# literal '*' to cp (which happens when the glob has no matches). +if ! compgen -G "${artifact_stage}/*" > /dev/null; then + echo >&2 "ERROR: no package artifacts were produced under ${artifact_stage}" + exit 1 +fi +cp --remove-destination "${artifact_stage}"/* "${output_dir}/" diff --git a/tools/build-packages/dependency-image/Dockerfile b/tools/build-packages/dependency-image/Dockerfile index b08cec2..eb74e93 100644 --- a/tools/build-packages/dependency-image/Dockerfile +++ b/tools/build-packages/dependency-image/Dockerfile @@ -20,7 +20,7 @@ RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ dnf_sslcacert_opt="--setopt=sslcacert=${HOST_CA_BUNDLE}"; \ fi \ && dnf install -y --setopt=install_weak_deps=False ${dnf_sslcacert_opt} \ - dpkg gettext git java-17-openjdk-devel libcurl-devel libevent-devel \ + ccache dpkg gettext git java-17-openjdk-devel libcurl-devel libevent-devel \ libunwind-devel ninja-build openssl-devel patchelf python3-pip rpm-build \ && dnf clean all diff --git a/tools/build-packages/internal/build-cache/README.md b/tools/build-packages/internal/build-cache/README.md new file mode 100644 index 0000000..ac48dbc --- /dev/null +++ b/tools/build-packages/internal/build-cache/README.md @@ -0,0 +1,45 @@ +# Build cache + +This directory provides reusable host and container helpers for persistent build caches. The helpers keep cache layout and tool configuration consistent while leaving cache location, cache identity, and container mounting under the caller's control. + +## Layout + +Given a cache directory and key, the helpers use: + +```text +/ +├── ccache/ +├── fetchcontent// +└── maven/ +``` + +Maven and ccache content is shared across cache keys. FetchContent contains generated CMake state and is isolated by a caller-provided key, such as a build-environment hash. + +## Host API + +Source `host.sh`, then prepare the cache before mounting it: + +```bash +source tools/build-packages/internal/build-cache/host.sh +prepare_build_cache ./.cache "${build_env_hash}" +``` + +The caller is responsible for choosing a nonempty cache key that is safe to use as a path component. + +## Container API + +Mount the prepared cache, set its container path and key, then source `container.sh`: + +```bash +BUILD_CACHE_DIR=/var/cache/build +BUILD_CACHE_KEY="${build_env_hash}" +source tools/build-packages/internal/build-cache/container.sh +``` + +The container helper creates missing subdirectories and supplies defaults for Maven, ccache, CMake compiler launchers, and `FETCHCONTENT_BASE_DIR`. Existing tool-specific environment variables take precedence over those defaults. + +`FETCHCONTENT_BASE_DIR` is an integration variable: projects must pass it to CMake as `-DFETCHCONTENT_BASE_DIR=...`. The helper does not modify project CMake files or command lines. + +## Lifecycle + +Build caches are persistent and may contain downloaded source, compiled objects, and generated build state. They are not temporary staging material and are not removed automatically. The caller decides when to delete or rotate the cache. diff --git a/tools/build-packages/internal/build-cache/container.sh b/tools/build-packages/internal/build-cache/container.sh new file mode 100644 index 0000000..c175378 --- /dev/null +++ b/tools/build-packages/internal/build-cache/container.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +# Container-side configuration for consuming a prepared build cache. + +if [ -n "${BUILD_CACHE_DIR:-}" ]; then + if [ -z "${BUILD_CACHE_KEY:-}" ]; then + echo >&2 "ERROR: BUILD_CACHE_KEY must be set when BUILD_CACHE_DIR is set" + return 1 + fi + + export CCACHE_DIR="${CCACHE_DIR:-${BUILD_CACHE_DIR}/ccache}" + export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-1G}" + export CMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER:-ccache}" + export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-ccache}" + export FETCHCONTENT_BASE_DIR="${FETCHCONTENT_BASE_DIR:-${BUILD_CACHE_DIR}/fetchcontent/${BUILD_CACHE_KEY}}" + export MAVEN_USER_HOME="${MAVEN_USER_HOME:-${BUILD_CACHE_DIR}/maven}" + + mkdir -p \ + "${CCACHE_DIR}" \ + "${FETCHCONTENT_BASE_DIR}" \ + "${MAVEN_USER_HOME}" +fi diff --git a/tools/build-packages/internal/build-cache/host.sh b/tools/build-packages/internal/build-cache/host.sh new file mode 100644 index 0000000..985da70 --- /dev/null +++ b/tools/build-packages/internal/build-cache/host.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Host-side preparation for persistent container build caches. + +if [[ "${_BUILD_CACHE_HOST_SH_LOADED:-}" == "1" ]]; then + return 0 +fi +readonly _BUILD_CACHE_HOST_SH_LOADED=1 + +# Creates the shared tool caches and a namespaced FetchContent cache. +# +# Args: +prepare_build_cache() { + if (( $# != 2 )) || [[ -z "$1" || -z "$2" ]]; then + echo >&2 "ERROR: prepare_build_cache requires a cache directory and key" + return 2 + fi + + local cache_dir="$1" + local cache_key="$2" + mkdir -p \ + "${cache_dir}/ccache" \ + "${cache_dir}/fetchcontent/${cache_key}" \ + "${cache_dir}/maven" +} diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index 337220c..bd98f3f 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -3,8 +3,8 @@ # Build the Java coordinator and C++ worker plugins, assemble one install tree, # and emit the same files as .deb, .rpm, and relocatable .tar.gz artifacts. # -# CI runs this container-side implementation inside the build-env image. See -# tools/build-packages/README.md for the overall packaging flow. +# This is the container-side implementation used by local and CI builds. See +# tools/build-packages/README.md for the entry points and overall flow. set -o errexit set -o nounset @@ -34,7 +34,7 @@ Options: VER must start with a digit and use [0-9A-Za-z.+~-] --help Show this help -See tools/build-packages/README.md for CI usage and package-build details. +See tools/build-packages/README.md for the recommended local entry point. EOF } @@ -91,6 +91,15 @@ if [[ -z "${JAVA_HOME:-}" ]]; then export JAVA_HOME="${javac_path%/bin/javac}" fi +maven_opts="${MAVEN_OPTS:-}" +if [[ -n "${MAVEN_USER_HOME:-}" ]]; then + # MAVEN_USER_HOME also caches the Maven Wrapper distribution, which is + # separate from Maven's local artifact repository. + mkdir -p "${MAVEN_USER_HOME}/repository" + [[ -n "${maven_opts}" ]] && maven_opts+=" " + maven_opts+="-Dmaven.repo.local=${MAVEN_USER_HOME}/repository" +fi + # ── Resolve architecture ────────────────────────────────────────────────────── # Debian and tarball names use amd64/arm64; RPM uses x86_64/aarch64. @@ -128,6 +137,7 @@ echo " -> ${so_file}" if [[ -z "${version}" ]]; then echo "==> Deriving version from presto-connector/pom.xml via mvnw..." version=$( + MAVEN_OPTS="${maven_opts}" \ "${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ --quiet help:evaluate \ @@ -149,6 +159,7 @@ echo "" # ── Build the Java coordinator plugin ───────────────────────────────────────── echo "==> Building presto-connector .jar via module mvnw..." +MAVEN_OPTS="${maven_opts}" \ "${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ clean package -DskipTests -B From 2a7d7684d98b61ac169d1aaae85ac7e7cda57f94 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 16 Jul 2026 17:33:33 -0700 Subject: [PATCH 02/16] fix(packaging): Use POSIX cp -f instead of GNU --remove-destination --remove-destination is a GNU cp extension absent on BSD/macOS cp; -f is POSIX-specified and behaves equivalently for copying staged artifacts into the user-owned output directory. --- tools/build-packages/build-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 7612a18..8354447 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -120,4 +120,4 @@ if ! compgen -G "${artifact_stage}/*" > /dev/null; then echo >&2 "ERROR: no package artifacts were produced under ${artifact_stage}" exit 1 fi -cp --remove-destination "${artifact_stage}"/* "${output_dir}/" +cp -f "${artifact_stage}"/* "${output_dir}/" From 08810866a26f5eb0842450bf6c703d9efbd351b9 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 16 Jul 2026 17:37:42 -0700 Subject: [PATCH 03/16] docs(packaging): Keep CI artifact reference alongside local build path The top-level README replaced the CI-build reference with a local-build one; both paths exist, so restore the CI mention and add the local one. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e7862da..7e80ba4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,6 @@ one of the tasks in the table below. ## Building installable packages -To build `.deb`, `.rpm`, and `.tar.gz` artifacts of the plugin locally, see [tools/build-packages/README.md](tools/build-packages/README.md). +For details about the `.deb`, `.rpm`, and `.tar.gz` artifacts built in CI, and to build them locally, see [tools/build-packages/README.md](tools/build-packages/README.md). [Task]: https://taskfile.dev From 524be08641dd224795a2add7f7d681776dc3a049 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 16 Jul 2026 17:51:36 -0700 Subject: [PATCH 04/16] docs(packaging): Trim build-cache README Tighten prose (intro, Container API, Lifecycle) while keeping every contract and integration note. Use one prose paragraph per line instead of manual hard wraps. --- .../internal/build-cache/README.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tools/build-packages/internal/build-cache/README.md b/tools/build-packages/internal/build-cache/README.md index ac48dbc..623f375 100644 --- a/tools/build-packages/internal/build-cache/README.md +++ b/tools/build-packages/internal/build-cache/README.md @@ -1,10 +1,10 @@ # Build cache -This directory provides reusable host and container helpers for persistent build caches. The helpers keep cache layout and tool configuration consistent while leaving cache location, cache identity, and container mounting under the caller's control. +Host and container helpers for persistent build caches. They fix cache layout and tool configuration; the caller controls cache location, identity, and mounting. ## Layout -Given a cache directory and key, the helpers use: +Given a cache directory and key: ```text / @@ -13,33 +13,27 @@ Given a cache directory and key, the helpers use: └── maven/ ``` -Maven and ccache content is shared across cache keys. FetchContent contains generated CMake state and is isolated by a caller-provided key, such as a build-environment hash. +Maven and ccache are shared across keys. FetchContent holds generated CMake state and is isolated by a caller-provided key, such as a build-env hash. ## Host API -Source `host.sh`, then prepare the cache before mounting it: - ```bash source tools/build-packages/internal/build-cache/host.sh prepare_build_cache ./.cache "${build_env_hash}" ``` -The caller is responsible for choosing a nonempty cache key that is safe to use as a path component. +The cache key must be nonempty and safe as a path component. ## Container API -Mount the prepared cache, set its container path and key, then source `container.sh`: - ```bash BUILD_CACHE_DIR=/var/cache/build BUILD_CACHE_KEY="${build_env_hash}" source tools/build-packages/internal/build-cache/container.sh ``` -The container helper creates missing subdirectories and supplies defaults for Maven, ccache, CMake compiler launchers, and `FETCHCONTENT_BASE_DIR`. Existing tool-specific environment variables take precedence over those defaults. - -`FETCHCONTENT_BASE_DIR` is an integration variable: projects must pass it to CMake as `-DFETCHCONTENT_BASE_DIR=...`. The helper does not modify project CMake files or command lines. +Creates missing subdirectories and exports defaults for ccache, Maven, CMake compiler launchers, and `FETCHCONTENT_BASE_DIR`; existing env vars take precedence. Projects must pass `FETCHCONTENT_BASE_DIR` to CMake themselves — the helper only exports it. ## Lifecycle -Build caches are persistent and may contain downloaded source, compiled objects, and generated build state. They are not temporary staging material and are not removed automatically. The caller decides when to delete or rotate the cache. +Caches are persistent (sources, objects, generated state) and are not removed automatically; the caller decides when to delete or rotate them. \ No newline at end of file From 681a34561f13c13a4c84e12a6c4a4d708a90a5c3 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 16 Jul 2026 21:11:25 -0400 Subject: [PATCH 05/16] feat(packaging): Run local package builds as the host user with cached build tree Local builds previously ran the container build as root and wrote the project build tree to the disposable container layer, so every invocation recompiled the C++ connector from scratch and left root-owned files in the repo. Changes: * build-packages.sh: run `docker run` as the host uid/gid and set CLP_PLUGIN_BUILD_DIR to a path under the persistent .cache/build/ cache, so the CMake build tree survives across runs (incremental rebuilds). * dependency-image/Dockerfile: pin CLP_PLUGIN_DEPS_CPP_DIR to the image's installed deps so the deps dir is not derived from the overridden build dir. * taskfiles/velox-connector/main.yaml: make G_DEPS_CPP_DIR env-overridable via CLP_PLUGIN_DEPS_CPP_DIR (mirrors the existing FETCHCONTENT_BASE_DIR pattern). * internal/build-cache/{host,container}.sh: create the build/ cache subdir; document it in internal/build-cache/README.md. * internal/container/build-artifacts.sh: place packaging staging under project_build_dir so it lives in the cache for local builds (CI still resolves to the image's /opt). Result: a no-op rebuild completes in under 30 seconds (C++ compile is a cache hit) and produces host-owned .deb/.rpm/.tar.gz with no root-owned files in the repo. --- .github/workflows/build-packages.yaml | 3 +-- taskfiles/velox-connector/main.yaml | 5 +++-- tools/build-packages/build-packages.sh | 12 ++++++++---- tools/build-packages/dependency-image/Dockerfile | 1 + .../build-packages/internal/build-cache/README.md | 3 ++- .../internal/build-cache/container.sh | 1 + tools/build-packages/internal/build-cache/host.sh | 1 + .../internal/container/build-artifacts.sh | 14 ++++++-------- 8 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index 549e51d..04187ed 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -121,8 +121,7 @@ jobs: for extension in deb rpm tar.gz; do artifact_files=(packages/clp-plugin-presto-connector*."${extension}") if (( ${#artifact_files[@]} != 1 )); then - echo "::error::Expected exactly one .${extension} artifact," \ - "found ${#artifact_files[@]}" + echo "::error::Expected exactly one .${extension} artifact, found ${#artifact_files[@]}" exit 1 fi printf '%s_filename=%s\n' \ diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml index 1790906..1bfe53a 100644 --- a/taskfiles/velox-connector/main.yaml +++ b/taskfiles/velox-connector/main.yaml @@ -8,7 +8,8 @@ includes: vars: G_VELOX_CONNECTOR_BUILD_DIR: "{{.G_BUILD_DIR}}/velox-connector" - G_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" + DEFAULT_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" + G_DEPS_CPP_DIR: '{{env "CLP_PLUGIN_DEPS_CPP_DIR" | default .DEFAULT_DEPS_CPP_DIR}}' DEFAULT_FETCHCONTENT_BASE_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/_deps" G_FETCHCONTENT_BASE_DIR: '{{env "FETCHCONTENT_BASE_DIR" | default .DEFAULT_FETCHCONTENT_BASE_DIR}}' @@ -20,7 +21,7 @@ tasks: - task: "build-with-installed-deps" build-with-installed-deps: - desc: "Builds the connector using dependencies already installed in G_DEPS_CPP_DIR." + desc: Builds the connector using dependencies already installed in G_DEPS_CPP_DIR. preconditions: - sh: "test -f '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake'" msg: "Installed dependency settings not found; run velox-connector:deps:install-all first." diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 8354447..59fb79f 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -81,22 +81,26 @@ if [[ "${image_hash}" == "${image}" ]]; then exit 1 fi -# Keep root-owned container output in temporary staging, then copy it to the -# requested directory as the invoking host user. +# Keep container output in temporary staging, then copy it to the requested +# directory after the build succeeds. stage_dir=$(mktemp -d) trap 'rm -rf "${stage_dir}"' EXIT artifact_stage="${stage_dir}/artifacts" mkdir -p "${artifact_stage}" prepare_build_cache "${src}/.cache" "${image_hash}" +host_uid=$(id -u) +host_gid=$(id -g) # Use a stable container checkout path so cached CMake state does not embed the # host checkout path. HOME and Task scratch data remain in the disposable # container. echo "==> Running internal/container/build-artifacts.sh inside ${image}..." docker run --rm \ + --user "${host_uid}:${host_gid}" \ --mount "type=bind,src=${src},dst=/repo" \ --mount "type=bind,src=${artifact_stage},dst=/output" \ --env "BUILD_CACHE_KEY=${image_hash}" \ + --env "CLP_PLUGIN_BUILD_DIR=/repo/.cache/build/${image_hash}" \ -w /repo \ "${image}" \ bash -c ' @@ -109,9 +113,9 @@ docker run --rm \ source tools/build-packages/internal/build-cache/container.sh mkdir -p "${HOME}" "${TASK_TEMP_DIR}" umask 0022 - echo "==> Running the package build as root..." + echo "==> Running the package build as host user $(id -u):$(id -g)..." exec bash tools/build-packages/internal/container/build-artifacts.sh "$@" - ' bash --output /output ${build_args[@]+"${build_args[@]}"} + ' bash --output /output "${build_args[@]}" echo "==> Copying package artifacts to ${output_dir}..." # Fail clearly when the build produced no artifacts, instead of passing a diff --git a/tools/build-packages/dependency-image/Dockerfile b/tools/build-packages/dependency-image/Dockerfile index eb74e93..3feb42e 100644 --- a/tools/build-packages/dependency-image/Dockerfile +++ b/tools/build-packages/dependency-image/Dockerfile @@ -41,6 +41,7 @@ RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ ENV PATH=/opt/go-task/bin:${PATH} ENV CLP_PLUGIN_BUILD_DIR=/opt/clp-plugin-presto-connector/build +ENV CLP_PLUGIN_DEPS_CPP_DIR=/opt/clp-plugin-presto-connector/build/velox-connector/deps/cpp ENV TASK_TEMP_DIR=/opt/clp-plugin-presto-connector/.task # Build one dependency at a time to avoid excessive memory usage from parallel builds. diff --git a/tools/build-packages/internal/build-cache/README.md b/tools/build-packages/internal/build-cache/README.md index 623f375..655e410 100644 --- a/tools/build-packages/internal/build-cache/README.md +++ b/tools/build-packages/internal/build-cache/README.md @@ -8,12 +8,13 @@ Given a cache directory and key: ```text / +├── build// ├── ccache/ ├── fetchcontent// └── maven/ ``` -Maven and ccache are shared across keys. FetchContent holds generated CMake state and is isolated by a caller-provided key, such as a build-env hash. +Maven and ccache are shared across keys. The project build and FetchContent hold generated state and are isolated by a caller-provided key, such as a build-env hash. ## Host API diff --git a/tools/build-packages/internal/build-cache/container.sh b/tools/build-packages/internal/build-cache/container.sh index c175378..e8832dc 100644 --- a/tools/build-packages/internal/build-cache/container.sh +++ b/tools/build-packages/internal/build-cache/container.sh @@ -16,6 +16,7 @@ if [ -n "${BUILD_CACHE_DIR:-}" ]; then export MAVEN_USER_HOME="${MAVEN_USER_HOME:-${BUILD_CACHE_DIR}/maven}" mkdir -p \ + "${BUILD_CACHE_DIR}/build/${BUILD_CACHE_KEY}" \ "${CCACHE_DIR}" \ "${FETCHCONTENT_BASE_DIR}" \ "${MAVEN_USER_HOME}" diff --git a/tools/build-packages/internal/build-cache/host.sh b/tools/build-packages/internal/build-cache/host.sh index 985da70..01362cb 100644 --- a/tools/build-packages/internal/build-cache/host.sh +++ b/tools/build-packages/internal/build-cache/host.sh @@ -19,6 +19,7 @@ prepare_build_cache() { local cache_dir="$1" local cache_key="$2" mkdir -p \ + "${cache_dir}/build/${cache_key}" \ "${cache_dir}/ccache" \ "${cache_dir}/fetchcontent/${cache_key}" \ "${cache_dir}/maven" diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index bd98f3f..59a5241 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -112,7 +112,7 @@ esac pkg_specs_dir="${src}/tools/build-packages/package-specs" project_build_dir="${CLP_PLUGIN_BUILD_DIR:-${src}/build}" velox_build_dir="${project_build_dir}/velox-connector" -build_root="${src}/build/packaging" +build_root="${project_build_dir}/packaging" payload="${build_root}/payload" artifacts=() @@ -120,9 +120,9 @@ prepare_paths # ── Build the C++ worker plugin ─────────────────────────────────────────────── -# The CI workflow initializes submodules before invoking this script. Reuse the -# dependency installations and CMake settings already present in the build-env -# image instead of rebuilding them. +# The local entrypoint and CI workflow initialize submodules before invoking +# this script. Reuse the dependency installations and CMake settings already +# present in the build-env image instead of rebuilding them. echo "==> Building velox-connector .so with image-installed dependencies..." task -d "${src}" velox-connector:build-with-installed-deps so_file="${velox_build_dir}/libclp-plugin-velox-connector.so" @@ -137,8 +137,7 @@ echo " -> ${so_file}" if [[ -z "${version}" ]]; then echo "==> Deriving version from presto-connector/pom.xml via mvnw..." version=$( - MAVEN_OPTS="${maven_opts}" \ - "${src}/presto-connector/mvnw" \ + MAVEN_OPTS="${maven_opts}" "${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ --quiet help:evaluate \ -Dexpression=project.version \ @@ -159,8 +158,7 @@ echo "" # ── Build the Java coordinator plugin ───────────────────────────────────────── echo "==> Building presto-connector .jar via module mvnw..." -MAVEN_OPTS="${maven_opts}" \ -"${src}/presto-connector/mvnw" \ +MAVEN_OPTS="${maven_opts}" "${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ clean package -DskipTests -B From d79bb83e7a88d10b9f6d644da032ff376b48fc5c Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 16 Jul 2026 23:23:27 -0400 Subject: [PATCH 06/16] revert(packaging): Drop cosmetic-only reformats from the host-user port Reverts the behavior-neutral formatting changes that came over with the host-user/cached-build port so the diff is purely functional: * .github/workflows/build-packages.yaml: restore the two-line ::error:: echo. * taskfiles/velox-connector/main.yaml: restore the quoted `desc:`. * tools/build-packages/internal/container/build-artifacts.sh: restore the MAVEN_OPTS/mvnw line wrapping and the submodule-init comment wording. * tools/build-packages/build-packages.sh: restore the defensive ${build_args[@]+"${build_args[@]}"} quoting. Functional changes (host-user run, CLP_PLUGIN_BUILD_DIR env, CLP_PLUGIN_DEPS_CPP_DIR pin, cache build/ helpers + README, build_root relocation) are unchanged. --- .github/workflows/build-packages.yaml | 3 ++- taskfiles/velox-connector/main.yaml | 2 +- tools/build-packages/build-packages.sh | 2 +- .../internal/container/build-artifacts.sh | 12 +++++++----- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index 04187ed..549e51d 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -121,7 +121,8 @@ jobs: for extension in deb rpm tar.gz; do artifact_files=(packages/clp-plugin-presto-connector*."${extension}") if (( ${#artifact_files[@]} != 1 )); then - echo "::error::Expected exactly one .${extension} artifact, found ${#artifact_files[@]}" + echo "::error::Expected exactly one .${extension} artifact," \ + "found ${#artifact_files[@]}" exit 1 fi printf '%s_filename=%s\n' \ diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml index 1bfe53a..aeb293e 100644 --- a/taskfiles/velox-connector/main.yaml +++ b/taskfiles/velox-connector/main.yaml @@ -21,7 +21,7 @@ tasks: - task: "build-with-installed-deps" build-with-installed-deps: - desc: Builds the connector using dependencies already installed in G_DEPS_CPP_DIR. + desc: "Builds the connector using dependencies already installed in G_DEPS_CPP_DIR." preconditions: - sh: "test -f '{{.G_DEPS_CPP_CMAKE_SETTINGS_DIR}}/all-deps.cmake'" msg: "Installed dependency settings not found; run velox-connector:deps:install-all first." diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 59fb79f..48d029a 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -115,7 +115,7 @@ docker run --rm \ umask 0022 echo "==> Running the package build as host user $(id -u):$(id -g)..." exec bash tools/build-packages/internal/container/build-artifacts.sh "$@" - ' bash --output /output "${build_args[@]}" + ' bash --output /output ${build_args[@]+"${build_args[@]}"} echo "==> Copying package artifacts to ${output_dir}..." # Fail clearly when the build produced no artifacts, instead of passing a diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index 59a5241..51bd699 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -120,9 +120,9 @@ prepare_paths # ── Build the C++ worker plugin ─────────────────────────────────────────────── -# The local entrypoint and CI workflow initialize submodules before invoking -# this script. Reuse the dependency installations and CMake settings already -# present in the build-env image instead of rebuilding them. +# The CI workflow initializes submodules before invoking this script. Reuse the +# dependency installations and CMake settings already present in the build-env +# image instead of rebuilding them. echo "==> Building velox-connector .so with image-installed dependencies..." task -d "${src}" velox-connector:build-with-installed-deps so_file="${velox_build_dir}/libclp-plugin-velox-connector.so" @@ -137,7 +137,8 @@ echo " -> ${so_file}" if [[ -z "${version}" ]]; then echo "==> Deriving version from presto-connector/pom.xml via mvnw..." version=$( - MAVEN_OPTS="${maven_opts}" "${src}/presto-connector/mvnw" \ + MAVEN_OPTS="${maven_opts}" \ + "${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ --quiet help:evaluate \ -Dexpression=project.version \ @@ -158,7 +159,8 @@ echo "" # ── Build the Java coordinator plugin ───────────────────────────────────────── echo "==> Building presto-connector .jar via module mvnw..." -MAVEN_OPTS="${maven_opts}" "${src}/presto-connector/mvnw" \ +MAVEN_OPTS="${maven_opts}" \ +"${src}/presto-connector/mvnw" \ --file "${src}/presto-connector/pom.xml" \ clean package -DskipTests -B From 344e37c9c64e4b0f4756bb63a2f880f0c41e2655 Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 16 Jul 2026 23:39:44 -0400 Subject: [PATCH 07/16] refactor(packaging): Move local container setup into build-artifacts.sh Fold the inline bash setup (cache wiring, HOME/TASK_TEMP_DIR scratch dirs, umask) that build-packages.sh ran before build-artifacts.sh into a BUILD_CACHE_DIR-guarded block at the top of build-artifacts.sh itself. Local builds set BUILD_CACHE_DIR (via build-packages.sh's --env flags), so they run that setup. CI invokes build-artifacts.sh directly without it, so the block is skipped and CI inherits the image's /opt layout unchanged. Removes the inline 'bash -c' block from build-packages.sh in favor of a direct 'bash /repo/.../build-artifacts.sh' invocation. No new file needed. --- tools/build-packages/build-packages.sh | 18 +++++------------- .../internal/container/build-artifacts.sh | 10 ++++++++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 48d029a..4bd534a 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -100,22 +100,14 @@ docker run --rm \ --mount "type=bind,src=${src},dst=/repo" \ --mount "type=bind,src=${artifact_stage},dst=/output" \ --env "BUILD_CACHE_KEY=${image_hash}" \ + --env "BUILD_CACHE_DIR=/repo/.cache" \ --env "CLP_PLUGIN_BUILD_DIR=/repo/.cache/build/${image_hash}" \ + --env "HOME=/tmp/clp-plugin-presto-connector-home" \ + --env "TASK_TEMP_DIR=/tmp/clp-plugin-presto-connector-task" \ -w /repo \ "${image}" \ - bash -c ' - set -o errexit - set -o nounset - set -o pipefail - export BUILD_CACHE_DIR=/repo/.cache - export HOME=/tmp/clp-plugin-presto-connector-home - export TASK_TEMP_DIR=/tmp/clp-plugin-presto-connector-task - source tools/build-packages/internal/build-cache/container.sh - mkdir -p "${HOME}" "${TASK_TEMP_DIR}" - umask 0022 - echo "==> Running the package build as host user $(id -u):$(id -g)..." - exec bash tools/build-packages/internal/container/build-artifacts.sh "$@" - ' bash --output /output ${build_args[@]+"${build_args[@]}"} + bash /repo/tools/build-packages/internal/container/build-artifacts.sh \ + --output /output ${build_args[@]+"${build_args[@]}"} echo "==> Copying package artifacts to ${output_dir}..." # Fail clearly when the build produced no artifacts, instead of passing a diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index 51bd699..464ac7e 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -12,6 +12,16 @@ set -o pipefail src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." &>/dev/null && pwd)" +# Local builds (invoked via build-packages.sh) set BUILD_CACHE_DIR so the build +# cache is wired up and HOME/TASK_TEMP_DIR point at writable scratch space for +# the non-root container user. CI invokes this script directly without those, +# so the cache and scratch setup here is local-only. +if [[ -n "${BUILD_CACHE_DIR:-}" ]]; then + umask 0022 + mkdir -p "${HOME}" "${TASK_TEMP_DIR}" + source "${src}/tools/build-packages/internal/build-cache/container.sh" +fi + # Destination paths used by .deb and .rpm. Environment overrides support a # non-default install layout; the tarball remains relocatable. readonly PLUGIN_ROOT="${PLUGIN_ROOT:-/opt/clp-plugin-presto-connector}" From 8f245028f0d5759fc987672c4da8899fa21d6d5e Mon Sep 17 00:00:00 2001 From: Jack Date: Thu, 16 Jul 2026 23:55:09 -0400 Subject: [PATCH 08/16] docs(packaging): Clarify env contract between build-packages.sh and build-artifacts.sh Comment-only: name the BUILD_CACHE_DIR/HOME/TASK_TEMP_DIR env contract in build-artifacts.sh and reword the docker-run comment in build-packages.sh so the --env flags aren't orphaned from the setup they drive. --- tools/build-packages/build-packages.sh | 9 ++++++--- .../build-packages/internal/container/build-artifacts.sh | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 4bd534a..f89f632 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -91,9 +91,12 @@ prepare_build_cache "${src}/.cache" "${image_hash}" host_uid=$(id -u) host_gid=$(id -g) -# Use a stable container checkout path so cached CMake state does not embed the -# host checkout path. HOME and Task scratch data remain in the disposable -# container. +# Run as the host user so staged files and artifacts aren't root-owned. Bind the +# repo at a stable /repo path so cached CMake state doesn't embed the host +# checkout path. The --env flags below wire the build cache and point HOME / +# TASK_TEMP_DIR at disposable in-container scratch (the non-root user can't +# write to the image's defaults); build-artifacts.sh activates that setup only +# when BUILD_CACHE_DIR is present, so CI (which calls it directly) is unaffected. echo "==> Running internal/container/build-artifacts.sh inside ${image}..." docker run --rm \ --user "${host_uid}:${host_gid}" \ diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index 464ac7e..10b9564 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -16,6 +16,10 @@ src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." &>/dev/null && pwd)" # cache is wired up and HOME/TASK_TEMP_DIR point at writable scratch space for # the non-root container user. CI invokes this script directly without those, # so the cache and scratch setup here is local-only. +# +# Contract: build-packages.sh passes BUILD_CACHE_DIR, BUILD_CACHE_KEY, +# CLP_PLUGIN_BUILD_DIR, HOME, and TASK_TEMP_DIR as --env flags. This block +# wires up the cache + scratch dirs only when BUILD_CACHE_DIR is present. if [[ -n "${BUILD_CACHE_DIR:-}" ]]; then umask 0022 mkdir -p "${HOME}" "${TASK_TEMP_DIR}" From ad633806dcd14ee4694b6359f38aa478f30e9636 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 17 Jul 2026 01:23:35 -0400 Subject: [PATCH 09/16] fix(packaging): Use shasum fallback for build-env hash on macOS derive_build_env_hash called sha256sum directly, which stock macOS doesn't ship (it provides shasum -a 256). Resolve a sha256_cmd that falls back to shasum -a 256 and use it in both stages of the hash pipeline. Both tools emit the same format, so the resulting tag is identical on either host. Also update the Requires: header in build-packages.sh and build-dependency-image.sh and the README prerequisites to list both. --- tools/build-packages/README.md | 4 ++-- tools/build-packages/build-dependency-image.sh | 2 +- tools/build-packages/build-packages.sh | 2 +- tools/build-packages/dependency-image/utils.sh | 14 +++++++++++--- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 9d9e249..9a0782a 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -39,5 +39,5 @@ root-owned files, while `packages/` is owned by the invoking user. ### Prerequisites -Docker with buildx (usable without `sudo`), git, `sha256sum`, and ~10 GB free -disk for the build-env image. +Docker with buildx (usable without `sudo`), git, `sha256sum` or `shasum`, and +~10 GB free disk for the build-env image. diff --git a/tools/build-packages/build-dependency-image.sh b/tools/build-packages/build-dependency-image.sh index 80f7e98..13c36d4 100755 --- a/tools/build-packages/build-dependency-image.sh +++ b/tools/build-packages/build-dependency-image.sh @@ -10,7 +10,7 @@ # docker run --rm -v "$(pwd):/src" -w /src "${image}" \ # task velox-connector:build-with-installed-deps # -# Requires: docker (with buildx), git, sha256sum. +# Requires: docker (with buildx), git, and sha256sum or shasum. set -o errexit set -o nounset diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index f89f632..995c3b1 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -3,7 +3,7 @@ # User-facing entry point for packaging. Resolves the build-env image, then runs # internal/container/build-artifacts.sh inside it. # -# Requires: docker (with buildx), git, sha256sum. +# Requires: docker (with buildx), git, and sha256sum or shasum. set -o errexit set -o nounset diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh index 3423057..15cf825 100644 --- a/tools/build-packages/dependency-image/utils.sh +++ b/tools/build-packages/dependency-image/utils.sh @@ -32,15 +32,23 @@ _BUILD_ENV_HASH_INPUTS=( # Computes the 16-hex-char hash used in the image tag. # -# Requires: git, sha256sum +# Requires: git, and either sha256sum (Linux) or shasum -a 256 (macOS). derive_build_env_hash() { ( cd "${_REPO_ROOT}" || exit ensure_yscope_dev_utils_submodule >&2 + + # macOS ships `shasum` rather than `sha256sum`; pick whichever exists. + # Both emit the same ` ` format, so the pipeline is unchanged. + local sha256_cmd=(sha256sum) + if ! command -v sha256sum &>/dev/null; then + sha256_cmd=(shasum -a 256) + fi + git ls-files -z --recurse-submodules -- "${_BUILD_ENV_HASH_INPUTS[@]}" \ | LC_ALL=C sort -z \ - | xargs -0 sha256sum \ - | sha256sum \ + | xargs -0 "${sha256_cmd[@]}" \ + | "${sha256_cmd[@]}" \ | cut -c1-16 ) } From d0b24dd971126bf03d3f71b82d8faf8d50294960 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 17 Jul 2026 02:04:57 -0400 Subject: [PATCH 10/16] feat(packaging): Add 'task package' wrapper for build-packages.sh Add a 'package' task to the root taskfile as a thin wrapper over tools/build-packages/build-packages.sh, so local package builds use the same 'task ...' ergonomics as 'task clean' / 'task velox-connector:build'. Args forward via CLI_ARGS (use '--' to pass flags). Document it as the recommended local entry point in tools/build-packages/README.md. --- taskfile.yaml | 8 ++++++++ tools/build-packages/README.md | 9 ++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index b6dbf50..04f1089 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -22,3 +22,11 @@ tasks: run: "once" cmds: - "mkdir -p '{{.G_BUILD_DIR}}'" + + package: + desc: "Build .deb/.rpm/.tar.gz packages." + preconditions: + - sh: "command -v docker" + msg: "docker (with buildx) is required for local package builds." + cmds: + - "{{.ROOT_DIR}}/tools/build-packages/build-packages.sh {{.CLI_ARGS}}" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 9a0782a..fa3a1ba 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -20,11 +20,14 @@ Default outputs are written to `./packages`. ## Local usage ```bash -./tools/build-packages/build-packages.sh +task package ``` -Use `--output DIR` and `--version VER` to override the output directory and -package version (otherwise derived from `presto-connector/pom.xml`). +This is a thin wrapper over `./tools/build-packages/build-packages.sh`, which +resolves the build-env image and runs the build inside it. Forward flags with +`--`, e.g. `task package -- --output DIR --version VER` (or pass them directly +to the script). `--output DIR` and `--version VER` override the output directory +and package version (otherwise derived from `presto-connector/pom.xml`). The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker From 43cf513c8adbcf5f798cb0db05ef4746575544b3 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 17 Jul 2026 02:11:00 -0400 Subject: [PATCH 11/16] docs(packaging): Mention 'task package' in local usage, keep script as fallback --- tools/build-packages/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index fa3a1ba..e970be5 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -23,11 +23,10 @@ Default outputs are written to `./packages`. task package ``` -This is a thin wrapper over `./tools/build-packages/build-packages.sh`, which -resolves the build-env image and runs the build inside it. Forward flags with -`--`, e.g. `task package -- --output DIR --version VER` (or pass them directly -to the script). `--output DIR` and `--version VER` override the output directory -and package version (otherwise derived from `presto-connector/pom.xml`). +A thin wrapper over `./tools/build-packages/build-packages.sh` (call that +directly if `go-task` isn't installed). Both accept `--output DIR` and +`--version VER`; with the task, put `--` before the flags: +`task package -- --output DIR`. The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker From 29c97cee90425bd3f4636943b5b69e98ec1b1392 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Sat, 18 Jul 2026 09:44:15 -0700 Subject: [PATCH 12/16] docs(packaging): Remove build-cache README Per reviewer request. Drop the internal build-cache README and the link to it from tools/build-packages/README.md. --- tools/build-packages/README.md | 3 +- .../internal/build-cache/README.md | 40 ------------------- 2 files changed, 1 insertion(+), 42 deletions(-) delete mode 100644 tools/build-packages/internal/build-cache/README.md diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index e970be5..80fc3e0 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -34,8 +34,7 @@ cache, this repository's GHCR package, or a local build, reusing the cached image on later runs. Build state is cached under `.cache/` (`maven/`, `ccache/`, and -`fetchcontent//`), shared across build-env revisions; see -[internal/build-cache/README.md](internal/build-cache/README.md). The container +`fetchcontent//`), shared across build-env revisions. The container runs as root, so `.cache/`, `build/`, and `presto-connector/target/` may hold root-owned files, while `packages/` is owned by the invoking user. diff --git a/tools/build-packages/internal/build-cache/README.md b/tools/build-packages/internal/build-cache/README.md deleted file mode 100644 index 655e410..0000000 --- a/tools/build-packages/internal/build-cache/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Build cache - -Host and container helpers for persistent build caches. They fix cache layout and tool configuration; the caller controls cache location, identity, and mounting. - -## Layout - -Given a cache directory and key: - -```text -/ -├── build// -├── ccache/ -├── fetchcontent// -└── maven/ -``` - -Maven and ccache are shared across keys. The project build and FetchContent hold generated state and are isolated by a caller-provided key, such as a build-env hash. - -## Host API - -```bash -source tools/build-packages/internal/build-cache/host.sh -prepare_build_cache ./.cache "${build_env_hash}" -``` - -The cache key must be nonempty and safe as a path component. - -## Container API - -```bash -BUILD_CACHE_DIR=/var/cache/build -BUILD_CACHE_KEY="${build_env_hash}" -source tools/build-packages/internal/build-cache/container.sh -``` - -Creates missing subdirectories and exports defaults for ccache, Maven, CMake compiler launchers, and `FETCHCONTENT_BASE_DIR`; existing env vars take precedence. Projects must pass `FETCHCONTENT_BASE_DIR` to CMake themselves — the helper only exports it. - -## Lifecycle - -Caches are persistent (sources, objects, generated state) and are not removed automatically; the caller decides when to delete or rotate them. \ No newline at end of file From f28f5efc0d8224a55d5daa88fc4ed7fb0a33be53 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Sat, 18 Jul 2026 12:04:55 -0700 Subject: [PATCH 13/16] build(packaging): Validate buildx and quote script path in 'task package' Precondition now runs 'docker buildx version' instead of 'command -v docker' so it actually verifies buildx, not just docker. Quote the ROOT_DIR-based build-packages.sh path so workspace paths containing spaces invoke the script correctly; CLI_ARGS stays unquoted to preserve argument forwarding. --- taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 04f1089..156fde1 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -26,7 +26,7 @@ tasks: package: desc: "Build .deb/.rpm/.tar.gz packages." preconditions: - - sh: "command -v docker" + - sh: "docker buildx version >/dev/null 2>&1" msg: "docker (with buildx) is required for local package builds." cmds: - - "{{.ROOT_DIR}}/tools/build-packages/build-packages.sh {{.CLI_ARGS}}" + - "'{{.ROOT_DIR}}/tools/build-packages/build-packages.sh' {{.CLI_ARGS}}" From 882ce7b33f325b613b6d4651b2befd862691f880 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Mon, 20 Jul 2026 07:05:42 -0700 Subject: [PATCH 14/16] docs(packaging): Document .cache/build// and distinguish from repo build/ Add build// to the cache layout list in the packaging README and clarify that it is the container's build output dir, distinct from the repo's top-level build/ used by non-container local dev builds. --- tools/build-packages/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 80fc3e0..bb8b91d 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -33,8 +33,11 @@ The build runs inside a hash-tagged **build-env image** (`env-`) based on cache, this repository's GHCR package, or a local build, reusing the cached image on later runs. -Build state is cached under `.cache/` (`maven/`, `ccache/`, and -`fetchcontent//`), shared across build-env revisions. The container +Build state is cached under `.cache/` (`maven/`, `ccache/`, +`fetchcontent//`, and `build//` for persisted CMake/build state), +shared across build-env revisions. `.cache/build//` is the container's +build output directory and is distinct from the repository's separate top-level +`build/` directory used by non-container local dev builds. The container runs as root, so `.cache/`, `build/`, and `presto-connector/target/` may hold root-owned files, while `packages/` is owned by the invoking user. From 323378ccdec520a167d60e0ce094e90b0da636a7 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Mon, 20 Jul 2026 07:11:22 -0700 Subject: [PATCH 15/16] docs(packaging): Correct container UID/GID guidance in README The local wrapper runs the build container with the invoking host UID/GID (--user flag) and refuses sudo, so it does not create root-owned files. Reword the ownership note to reflect that: any root-owned files in .cache/, build/, or presto-connector/target/ are leftovers from earlier privileged or CI builds, while packages/ stays owned by the invoking user. --- tools/build-packages/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index bb8b91d..51bacba 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -37,9 +37,11 @@ Build state is cached under `.cache/` (`maven/`, `ccache/`, `fetchcontent//`, and `build//` for persisted CMake/build state), shared across build-env revisions. `.cache/build//` is the container's build output directory and is distinct from the repository's separate top-level -`build/` directory used by non-container local dev builds. The container -runs as root, so `.cache/`, `build/`, and `presto-connector/target/` may hold -root-owned files, while `packages/` is owned by the invoking user. +`build/` directory used by non-container local dev builds. The local wrapper +runs the container with the invoking host UID/GID, so it does not create +root-owned files; any root-owned files in `.cache/`, `build/`, or +`presto-connector/target/` are leftovers from earlier privileged or CI builds, +while `packages/` is owned by the invoking user. ### Prerequisites From d4a3f4247787cf969224830431c1bc8a90f2c28b Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Mon, 20 Jul 2026 07:58:31 -0700 Subject: [PATCH 16/16] refactor(taskfile): Use {{.VAR | default}} idiom for env-sourced build dirs Switch G_BUILD_DIR, G_DEPS_CPP_DIR, and G_FETCHCONTENT_BASE_DIR from {{env "VAR"}} to {{.VAR | default ...}}. In Task v3 both forms read the shell environment ({{.VAR}} falls back to env as its last lookup step), and since no Taskfile scope defines these names they behave identically. Adopted for consistency with the Taskfile docs idiom. Validated via task --dry: G_BUILD_DIR and G_DEPS_CPP_DIR fallback and env-override both resolve correctly in the real Taskfiles, and the G_FETCHCONTENT_BASE_DIR override resolves in an isolated probe using the same pattern. --- taskfile.yaml | 2 +- taskfiles/velox-connector/main.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 156fde1..c455d4b 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -7,7 +7,7 @@ includes: vars: DEFAULT_CLP_PLUGIN_BUILD_DIR: "{{.ROOT_DIR}}/build" - G_BUILD_DIR: '{{env "CLP_PLUGIN_BUILD_DIR" | default .DEFAULT_CLP_PLUGIN_BUILD_DIR}}' + G_BUILD_DIR: '{{.CLP_PLUGIN_BUILD_DIR | default .DEFAULT_CLP_PLUGIN_BUILD_DIR}}' tasks: clean: diff --git a/taskfiles/velox-connector/main.yaml b/taskfiles/velox-connector/main.yaml index aeb293e..dba5b3b 100644 --- a/taskfiles/velox-connector/main.yaml +++ b/taskfiles/velox-connector/main.yaml @@ -9,9 +9,9 @@ includes: vars: G_VELOX_CONNECTOR_BUILD_DIR: "{{.G_BUILD_DIR}}/velox-connector" DEFAULT_DEPS_CPP_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/deps/cpp" - G_DEPS_CPP_DIR: '{{env "CLP_PLUGIN_DEPS_CPP_DIR" | default .DEFAULT_DEPS_CPP_DIR}}' + G_DEPS_CPP_DIR: '{{.CLP_PLUGIN_DEPS_CPP_DIR | default .DEFAULT_DEPS_CPP_DIR}}' DEFAULT_FETCHCONTENT_BASE_DIR: "{{.G_VELOX_CONNECTOR_BUILD_DIR}}/_deps" - G_FETCHCONTENT_BASE_DIR: '{{env "FETCHCONTENT_BASE_DIR" | default .DEFAULT_FETCHCONTENT_BASE_DIR}}' + G_FETCHCONTENT_BASE_DIR: '{{.FETCHCONTENT_BASE_DIR | default .DEFAULT_FETCHCONTENT_BASE_DIR}}' tasks: build: